From 06a44bb0abbb72f15664bcf5e4d1b2c511ed34ae Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 25 Oct 2026 16:01:18 -0400 Subject: [PATCH] docs: add point queries blog to good reads and tuning guide Co-Authored-By: Claude Fable 5.1 --- dev/update_config_docs.sh | 32 +++++++++++++++++++ .../user-guide/concepts-readings-events.md | 2 ++ docs/source/user-guide/configs.md | 32 +++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/dev/update_config_docs.sh b/dev/update_config_docs.sh index 5bba39625f606..3b6291eb7cda6 100755 --- a/dev/update_config_docs.sh +++ b/dev/update_config_docs.sh @@ -198,6 +198,38 @@ SET datafusion.execution.target_partitions = '1'; [`ListingTable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html +## Point Queries (Single Row Lookups) + +Point queries look up a small number of rows by key (for example, +`SELECT * FROM t WHERE id = 42`). For these queries the fixed costs of planning, +opening files, and coordinating partitions often dominate the actual work, so +the goal is to read as little data as possible and to avoid unnecessary +parallelism. + +For Parquet data, the most effective techniques are: + +- Sort the data by the lookup key so min/max statistics prune most row groups + and pages, and use relatively small row groups and pages (for example, ~128k + rows per row group and ~1k rows per page) so that pruning is fine grained. +- Set `datafusion.execution.target_partitions` to `1` to avoid the overhead of + parallelism, as described in [Short Queries](#short-queries). +- Enable `datafusion.execution.parquet.pushdown_filters` so filters are + evaluated while decoding Parquet rather than after the whole row group is read. +- Keep Parquet metadata caching enabled (the default) and size the cache with + `datafusion.runtime.metadata_cache_limit` so it holds the metadata for the + files you query repeatedly. +- Reuse prepared statements to avoid re-planning each query. + +```sql +SET datafusion.execution.target_partitions = '1'; +SET datafusion.execution.parquet.pushdown_filters = true; +``` + +See the [Best Practices for Single Row Lookups (Point Queries)] blog post for +benchmarks and a detailed walk through of each of these settings. + +[Best Practices for Single Row Lookups (Point Queries)]: https://datafusion.apache.org/blog/2026/10/24/point-queries/ + ## Memory-limited Queries When executing a memory-consuming query under a tight memory limit, DataFusion diff --git a/docs/source/user-guide/concepts-readings-events.md b/docs/source/user-guide/concepts-readings-events.md index 2024dc6474caf..5d7a066b3e658 100644 --- a/docs/source/user-guide/concepts-readings-events.md +++ b/docs/source/user-guide/concepts-readings-events.md @@ -37,6 +37,8 @@ This is a list of DataFusion related blog posts, articles, and other resources. Please open a PR to add any new resources you create or find +- **2026-10-24** [Blog: Best Practices for Single Row Lookups (Point Queries)](https://datafusion.apache.org/blog/2026/10/24/point-queries/) + - **2026-10-01** [Blog: How we extended Apache DataFusion to execute one query across many machines](https://www.datadoghq.com/blog/engineering/distributed-datafusion/) - **2026-07-29** [Blog: Bringing DuckLake to DataFusion](https://ducklake.select/2026/07/29/bringing-ducklake-to-datafusion/) diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 27356ab8c1b77..3a161b1bb8899 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -284,6 +284,38 @@ SET datafusion.execution.target_partitions = '1'; [`listingtable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html +## Point Queries (Single Row Lookups) + +Point queries look up a small number of rows by key (for example, +`SELECT * FROM t WHERE id = 42`). For these queries the fixed costs of planning, +opening files, and coordinating partitions often dominate the actual work, so +the goal is to read as little data as possible and to avoid unnecessary +parallelism. + +For Parquet data, the most effective techniques are: + +- Sort the data by the lookup key so min/max statistics prune most row groups + and pages, and use relatively small row groups and pages (for example, ~128k + rows per row group and ~1k rows per page) so that pruning is fine grained. +- Set `datafusion.execution.target_partitions` to `1` to avoid the overhead of + parallelism, as described in [Short Queries](#short-queries). +- Enable `datafusion.execution.parquet.pushdown_filters` so filters are + evaluated while decoding Parquet rather than after the whole row group is read. +- Keep Parquet metadata caching enabled (the default) and size the cache with + `datafusion.runtime.metadata_cache_limit` so it holds the metadata for the + files you query repeatedly. +- Reuse prepared statements to avoid re-planning each query. + +```sql +SET datafusion.execution.target_partitions = '1'; +SET datafusion.execution.parquet.pushdown_filters = true; +``` + +See the [Best Practices for Single Row Lookups (Point Queries)] blog post for +benchmarks and a detailed walk through of each of these settings. + +[best practices for single row lookups (point queries)]: https://datafusion.apache.org/blog/2026/10/24/point-queries/ + ## Memory-limited Queries When executing a memory-consuming query under a tight memory limit, DataFusion