From 852c5ef16b2851cac0e58a99f3c7791696f5d3cf Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 04:36:00 +0700 Subject: [PATCH 01/12] feat(drive): boolean HAVING range queries on ranked index axes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serve a grouped aggregate carrying exactly one HAVING clause on the selected aggregate (GROUP BY p HAVING LIMIT n) as a value-bounded range read of the covering ranked index's axis secondary — the same grovedb trees the PV14 ranked top-k surface walks — with a completeness-attesting proof. - rs-drive: drive_document_having_query (versioned grammar, bounds translation, executors) + document_having verifier; prover and verifier share one bounds-to-Merk-query translation and path builder - rs-drive-abci: compute_aggregate_mode_and_check_limit v2 routes the shape to dispatch_having_v1; response reuses RankedEntries with skipped unset, so zero proto changes - rs-platform-version: PV14 selects DRIVE_ABCI_QUERY_VERSIONS_V3; detect_having_mode / verify_having_range_proof slots dormant at 0 in all tables; v13 and earlier keep rejecting every non-empty HAVING - rs-drive-proof-verifier / rs-sdk: DocumentHavingEntries with FromProof/Fetch, binding the proof to the quorum-signed app hash Co-Authored-By: Claude Fable 5 --- .../mod.rs | 41 +- .../v2/mod.rs | 63 + .../src/query/document_query/v1/mod.rs | 176 ++- .../src/query/document_query/v1/tests.rs | 445 ++++++- packages/rs-drive-proof-verifier/src/lib.rs | 8 + packages/rs-drive-proof-verifier/src/proof.rs | 5 + .../src/proof/document_having.rs | 333 +++++ .../src/proof/document_ranked.rs | 2 +- .../drive_dispatcher.rs | 147 +++ .../execute_range.rs | 176 +++ .../drive_document_having_query/executors.rs | 117 ++ .../query/drive_document_having_query/mod.rs | 273 ++++ .../mode_detection.rs | 527 ++++++++ .../drive_document_having_query/tests.rs | 1112 +++++++++++++++++ .../query/drive_document_ranked_query/path.rs | 50 +- packages/rs-drive/src/query/mod.rs | 23 + .../src/verify/document_having/mod.rs | 19 + .../verify_having_range_proof/mod.rs | 52 + .../verify_having_range_proof/v0/mod.rs | 112 ++ packages/rs-drive/src/verify/mod.rs | 4 + .../drive_abci_query_versions/mod.rs | 1 + .../drive_abci_query_versions/v3.rs | 31 + .../drive_document_method_versions/mod.rs | 8 + .../drive_document_method_versions/v1.rs | 1 + .../drive_document_method_versions/v2.rs | 1 + .../drive_document_method_versions/v3.rs | 1 + .../drive_document_method_versions/v4.rs | 1 + .../drive_verify_method_versions/mod.rs | 10 +- .../drive_verify_method_versions/v1.rs | 1 + .../rs-platform-version/src/version/v14.rs | 56 +- packages/rs-sdk/src/mock/requests.rs | 24 + .../documents/document_having_entries.rs | 319 +++++ .../src/platform/documents/document_query.rs | 22 +- .../documents/having_proof_helpers.rs | 162 +++ packages/rs-sdk/src/platform/documents/mod.rs | 7 + 35 files changed, 4239 insertions(+), 91 deletions(-) create mode 100644 packages/rs-drive-abci/src/query/document_query/v1/compute_aggregate_mode_and_check_limit/v2/mod.rs create mode 100644 packages/rs-drive-proof-verifier/src/proof/document_having.rs create mode 100644 packages/rs-drive/src/query/drive_document_having_query/drive_dispatcher.rs create mode 100644 packages/rs-drive/src/query/drive_document_having_query/execute_range.rs create mode 100644 packages/rs-drive/src/query/drive_document_having_query/executors.rs create mode 100644 packages/rs-drive/src/query/drive_document_having_query/mod.rs create mode 100644 packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs create mode 100644 packages/rs-drive/src/query/drive_document_having_query/tests.rs create mode 100644 packages/rs-drive/src/verify/document_having/mod.rs create mode 100644 packages/rs-drive/src/verify/document_having/verify_having_range_proof/mod.rs create mode 100644 packages/rs-drive/src/verify/document_having/verify_having_range_proof/v0/mod.rs create mode 100644 packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/v3.rs create mode 100644 packages/rs-sdk/src/platform/documents/document_having_entries.rs create mode 100644 packages/rs-sdk/src/platform/documents/having_proof_helpers.rs diff --git a/packages/rs-drive-abci/src/query/document_query/v1/compute_aggregate_mode_and_check_limit/mod.rs b/packages/rs-drive-abci/src/query/document_query/v1/compute_aggregate_mode_and_check_limit/mod.rs index 8fe31b340b8..4b5eb65758b 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/compute_aggregate_mode_and_check_limit/mod.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/compute_aggregate_mode_and_check_limit/mod.rs @@ -5,11 +5,13 @@ //! against the v1 query surface is executed: as one of the four //! `(group_by × where)` grouped modes, or — from feature version 1 — //! as a *ranked* request when the request orders by the aggregate it -//! selects (`GROUP BY p ORDER BY `). Routing -//! here only asks whether the `order_by` names that aggregate; the -//! direction, the limit's bounds and the offset are drive's call and -//! are made in `detect_ranked_mode`. It also enforces the per-mode -//! `accepts_limit()` contract on the grouped path. +//! selects (`GROUP BY p ORDER BY `), or — from +//! feature version 2 — as a *having-range* request when a grouped +//! aggregate carries exactly one `having` clause. Routing here only +//! asks which shape the request has; the direction, the bounds, the +//! limit's contract and the offset are drive's call and are made in +//! `detect_ranked_mode` / `detect_having_mode`. It also enforces the +//! per-mode `accepts_limit()` contract on the grouped path. //! //! The routing rules it embeds are part of the query contract clients //! see on the wire — a change to which `(group_by × where_clauses × @@ -17,10 +19,10 @@ //! dispatcher runs on every v1 query request. Versioning it lets later //! protocol bumps adjust the routing table without breaking older //! nodes' replay of historical traffic, and is what keeps a -//! mixed-version network in agreement across the ranked-query -//! activation: protocol version 13 and earlier select v0, which has no -//! ranked path at all, while protocol version 14 selects v1 and answers -//! ranked queries. +//! mixed-version network in agreement across the ranked-query and +//! having-range activations: protocol version 13 and earlier select v0, +//! which has neither path, while protocol version 14 selects v2 and +//! answers both. //! //! Lives next to the v1 query handler (the only call site today) and //! is dispatched via the `DriveAbciDocumentQueryHelperVersions` slot @@ -28,6 +30,7 @@ mod v0; mod v1; +mod v2; use crate::error::query::QueryError; use dpp::version::PlatformVersion; @@ -56,6 +59,15 @@ pub(super) enum AggregateRouting { Grouped(CountMode), /// Ranked aggregate: execute through the ranked (top-k) surface. Ranked, + /// Boolean-`HAVING` range: a grouped aggregate carrying exactly one + /// `having` clause, executed through + /// `Drive::execute_document_having_request` as a value-bounded range + /// read of the covering ranked index's axis secondary. The + /// feature-version-2 addition. Carries no data for the same reason + /// `Ranked` carries none: drive owns the having grammar + /// (`detect_having_mode`), and routing only decides *where* the + /// request goes. + HavingRange, } /// Decide how a `SELECT COUNT` / `SUM` / `AVG` request executes. @@ -113,10 +125,19 @@ pub(super) fn compute_aggregate_mode_and_check_limit( having, function_name, ), + 2 => v2::compute_aggregate_mode_and_check_limit_v2( + select, + group_by, + where_clauses, + order_by, + limit, + having, + function_name, + ), version => Err(QueryError::Drive(drive::error::Error::Drive( drive::error::drive::DriveError::UnknownVersionMismatch { method: "compute_aggregate_mode_and_check_limit".to_string(), - known_versions: vec![0, 1], + known_versions: vec![0, 1, 2], received: version, }, ))), diff --git a/packages/rs-drive-abci/src/query/document_query/v1/compute_aggregate_mode_and_check_limit/v2/mod.rs b/packages/rs-drive-abci/src/query/document_query/v1/compute_aggregate_mode_and_check_limit/v2/mod.rs new file mode 100644 index 00000000000..39151e09f28 --- /dev/null +++ b/packages/rs-drive-abci/src/query/document_query/v1/compute_aggregate_mode_and_check_limit/v2/mod.rs @@ -0,0 +1,63 @@ +//! Feature version 2 of the aggregate routing helper: the boolean-`HAVING` +//! range activation. +//! +//! Differs from v1 in exactly one branch: a grouped aggregate carrying a +//! **single** `having` clause routes to the having-range executor +//! ([`AggregateRouting::HavingRange`]) instead of being refused. Routing +//! here only asks "is there a grouped select with exactly one having +//! clause?" — whether the clause bounds the selected aggregate, whether +//! its operator translates to a contiguous range, whether an `order_by` +//! is compatible, and the limit's bounds are all drive's call, made in +//! `detect_having_mode`; a routing-layer copy of that grammar could +//! disagree with drive's after a version bump. +//! +//! Multi-clause `having` (implicit AND) keeps the `not_yet_implemented` +//! contract: each extra clause needs a per-candidate post-check against +//! the primary that no executor performs yet. And `having` without +//! `group_by` still falls through to the v1 → v0 blanket rejection — a +//! global aggregate produces one row, and bounding it is a client-side +//! comparison, not a query. + +use super::v1::compute_aggregate_mode_and_check_limit_v1; +use super::AggregateRouting; +use crate::error::query::QueryError; +use crate::query::document_query::v1::not_yet_implemented; +use drive::query::{HavingClause, OrderClause, SelectProjection, WhereClause}; + +#[allow(clippy::too_many_arguments)] +pub(super) fn compute_aggregate_mode_and_check_limit_v2( + select: &SelectProjection, + group_by: &[String], + where_clauses: &[WhereClause], + order_by: &[OrderClause], + limit: Option, + having: &[HavingClause], + function_name: &str, +) -> Result { + if !having.is_empty() && !group_by.is_empty() { + return match having { + [_single] => Ok(AggregateRouting::HavingRange), + many => Err(not_yet_implemented(&format!( + "multiple HAVING clauses (implicit AND): got {}. One clause on the \ + selected {function_name} aggregate is served as a single contiguous \ + range read of the covering ranked index's axis secondary; additional \ + clauses would need a per-candidate post-check that is not implemented. \ + Narrow to a single clause", + many.len() + ))), + }; + } + + // No having (or no group_by, where a having still dies in v0's + // blanket rejection): identical routing to v1, including its ranked + // detection and its delegation to v0 for non-ranked shapes. + compute_aggregate_mode_and_check_limit_v1( + select, + group_by, + where_clauses, + order_by, + limit, + having, + function_name, + ) +} diff --git a/packages/rs-drive-abci/src/query/document_query/v1/mod.rs b/packages/rs-drive-abci/src/query/document_query/v1/mod.rs index def61a80a91..e79ee540a3e 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/mod.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/mod.rs @@ -58,10 +58,11 @@ use dpp::version::PlatformVersion; use drive::error::query::QuerySyntaxError; use drive::query::{ AverageEntry as DriveAverageEntry, AverageMode, CountMode, DocumentAverageRequest, - DocumentAverageResponse, DocumentCountRequest, DocumentCountResponse, DocumentRankedRequest, - DocumentRankedResponse, DocumentSumRequest, DocumentSumResponse, HavingClause, OrderClause, - RankedEntry as DriveRankedEntry, RankedEntryValue, SelectFunction, SelectProjection, - SplitCountEntry, SumEntry as DriveSumEntry, SumMode, WhereClause, + DocumentAverageResponse, DocumentCountRequest, DocumentCountResponse, DocumentHavingRequest, + DocumentHavingResponse, DocumentRankedRequest, DocumentRankedResponse, DocumentSumRequest, + DocumentSumResponse, HavingClause, OrderClause, RankedEntry as DriveRankedEntry, + RankedEntryValue, SelectFunction, SelectProjection, SplitCountEntry, SumEntry as DriveSumEntry, + SumMode, WhereClause, }; use drive::util::grove_operations::GroveDBToUse; @@ -132,8 +133,9 @@ fn validate_and_route( // it is a boolean predicate over the groups a `COUNT` / `SUM` / // `AVG` produces. Those three functions route through the // versioned `compute_aggregate_mode_and_check_limit` helper below, - // which rejects non-empty HAVING on both tables (evaluation is not - // implemented) but with wording that depends on whether the + // whose v2 table routes a single grouped clause to the + // having-range executor and whose older tables reject every + // non-empty HAVING, with wording that depends on whether the // request is otherwise a ranked one. // // For every other SELECT there is no aggregate for a HAVING to @@ -225,6 +227,7 @@ fn validate_and_route( mode, }), AggregateRouting::Ranked => Ok(RoutingDecision::Ranked), + AggregateRouting::HavingRange => Ok(RoutingDecision::HavingRange), } } SelectFunction::Avg => { @@ -269,6 +272,7 @@ fn validate_and_route( mode, }), AggregateRouting::Ranked => Ok(RoutingDecision::Ranked), + AggregateRouting::HavingRange => Ok(RoutingDecision::HavingRange), } } SelectFunction::Min => Err(not_yet_implemented( @@ -326,6 +330,7 @@ fn validate_and_route( )? { AggregateRouting::Grouped(mode) => Ok(RoutingDecision::Count(mode)), AggregateRouting::Ranked => Ok(RoutingDecision::Ranked), + AggregateRouting::HavingRange => Ok(RoutingDecision::HavingRange), } } } @@ -378,6 +383,18 @@ enum RoutingDecision { /// there is nothing to carry and no opportunity for the routing /// layer's reading of the ranking to drift from drive's. Ranked, + /// Boolean-`HAVING` range routing: a `COUNT` / `SUM` / `AVG` select + /// with a `GROUP BY` carrying exactly one `having` clause. Routing + /// does not read the clause — whether it bounds the selected + /// aggregate, whether its operator translates to a contiguous + /// range, and the limit's contract are drive's to resolve and to + /// refuse (`detect_having_mode`). Dispatches to + /// [`Self::dispatch_having_v1`] → + /// `Drive::execute_document_having_request` and emits the + /// `RankedEntries` proto message with `skipped` unset (a range page + /// has no rank base). Carries nothing, for the same reason `Ranked` + /// carries nothing. + HavingRange, } /// The `OFFSET` gate, applied **after** routing. @@ -511,6 +528,9 @@ pub(super) fn validate_and_route_for_tests( // breakdown is drive's to resolve, not routing's, so there // is no sub-mode to report here. RoutingDecision::Ranked => "ranked", + // Having-range surface — single label for the same reason as + // ranked: the bounds / direction / limit breakdown is drive's. + RoutingDecision::HavingRange => "having_range", }) } @@ -678,6 +698,21 @@ impl Platform { platform_state, platform_version, ), + RoutingDecision::HavingRange => self.dispatch_having_v1( + data_contract_id, + document_type, + select, + group_by, + having_clauses, + where_clauses, + order_by_clauses, + limit, + offset, + start, + prove, + platform_state, + platform_version, + ), } } @@ -1393,6 +1428,135 @@ impl Platform { Ok(QueryValidationResult::new_with_data(response)) } + + /// Dispatch a boolean-`HAVING` range request + /// (`GROUP BY p HAVING LIMIT n`) to + /// [`Drive::execute_document_having_request`] and map the response + /// onto the wire. + /// + /// Parallels [`Self::dispatch_ranked_v1`] line-for-line — same + /// contract/doctype resolution, same error → typed-rejection + /// mapping, same prove split — because the two surfaces read the + /// same indexed tree. The response reuses the `RankedEntries` + /// message (a having page is the same "group key + aggregate value" + /// entry list), with one deliberate difference: `skipped` is left + /// unset. Its published contract is "the page's starting rank", and + /// a value-bounded page has no rank base — the entries are simply + /// every matching group in axis order, cut at `limit`. + #[allow(clippy::too_many_arguments)] + fn dispatch_having_v1( + &self, + data_contract_id: Vec, + document_type_name: String, + select: SelectProjection, + group_by: Vec, + having: Vec, + where_clauses: Vec, + order_clauses: Vec, + limit: Option, + offset: Option, + start: Option, + prove: bool, + platform_state: &PlatformState, + platform_version: &PlatformVersion, + ) -> Result, Error> { + let contract_id: Identifier = + check_validation_result_with_data!(data_contract_id.try_into().map_err(|_| { + QueryError::InvalidArgument( + "id must be a valid identifier (32 bytes long)".to_string(), + ) + })); + + let (_, contract_fetch_info) = self.drive.get_contract_with_fetch_info_and_fee( + contract_id.to_buffer(), + None, + true, + None, + platform_version, + )?; + let contract_fetch_info = check_validation_result_with_data!(contract_fetch_info.ok_or( + QueryError::Query(QuerySyntaxError::DataContractNotFound( + "contract not found when querying from value with contract info", + )) + )); + let contract_ref = &contract_fetch_info.contract; + let document_type = check_validation_result_with_data!(contract_ref + .document_type_for_name(document_type_name.as_str()) + .map_err(|_| QueryError::InvalidArgument(format!( + "document type {} not found for contract {}", + document_type_name, contract_id + )))); + + let drive_request = DocumentHavingRequest { + contract: contract_ref, + document_type, + group_by: &group_by, + select, + having: &having, + order_by: &order_clauses, + where_clauses: &where_clauses, + limit, + offset, + has_start_at: start.is_some(), + prove, + }; + + let drive_response = + match self + .drive + .execute_document_having_request(drive_request, None, platform_version) + { + Ok(r) => r, + Err(drive::error::Error::Query(qe)) => { + return Ok(QueryValidationResult::new_with_error(QueryError::Query(qe))); + } + // Same empty-tree backstop as the ranked path: the + // range prover emits a guaranteed-empty range against + // an empty secondary, so this should be unreachable, + // but the failure class is merk-level and could + // surface from anywhere in the ancestor chain. + Err(e) => match empty_ranking_proof_rejection(&e) { + Some(rejection) => { + return Ok(QueryValidationResult::new_with_error(rejection)); + } + None => return Err(e.into()), + }, + }; + + let response = match drive_response { + DocumentHavingResponse::Entries(entries) => GetDocumentsResponseV1 { + result: Some(get_documents_response_v1::Result::Data(ResultData { + // Always a list, never an aggregate collapse — same + // rationale as ranked: even one matching group is + // an entry, because the caller needs to know which + // group matched, not only that one did. + variant: Some(result_data::Variant::Ranked(RankedEntries { + // Order preserved verbatim: axis order in the + // walk direction, and drive already asserted + // the list is no longer than the limit. + entries: entries.into_iter().map(into_v1_ranked_entry).collect(), + // Deliberately unset. `skipped`'s published + // contract is rank-based ("entry i is the + // group at rank skipped + i"), and a + // value-bounded page has no rank base — there + // is nothing the field could truthfully say. + skipped: None, + })), + })), + metadata: Some(self.response_metadata_v0(platform_state, CheckpointUsed::Current)), + }, + DocumentHavingResponse::Proof(proof_bytes) => { + let (grovedb_used, proof) = + self.response_proof_v0(platform_state, proof_bytes, GroveDBToUse::Current)?; + GetDocumentsResponseV1 { + result: Some(get_documents_response_v1::Result::Proof(proof)), + metadata: Some(self.response_metadata_v0(platform_state, grovedb_used)), + } + } + }; + + Ok(QueryValidationResult::new_with_data(response)) + } } /// Translate an rs-drive `RankedEntry` into the wire `RankedEntry`. diff --git a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs index b3aeb70b843..1d5e4850e4b 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs @@ -165,10 +165,12 @@ fn assert_not_yet_implemented(result: Result<&'static str, QueryError>, expected #[test] fn reject_having_non_empty() { - // Non-empty `having` is rejected wholesale until the server - // gains HAVING-evaluation capability. The clause shape itself - // doesn't matter (server doesn't decode it past the `is_empty()` - // check), so a single placeholder clause is sufficient. + // Non-empty `having` on a **non-aggregate** select stays rejected + // by the unversioned gate: this request carries no `selects`, so it + // defaults to `SELECT DOCUMENTS`, and there is no aggregate for a + // HAVING to talk about. (The aggregate selects route through the + // versioned helper, whose v2 table serves a single grouped clause — + // see `having_range_tests`.) let request = GetDocumentsRequestV1 { having: vec![hc( having_aggregate::Function::Count, @@ -1964,17 +1966,17 @@ mod ranked_tests { /// Shared with rs-drive's ranked suite — see the module docs for /// why this is a cross-crate path and not a copy. - const RESTAURANTS_CONTRACT_PATH: &str = + pub(super) const RESTAURANTS_CONTRACT_PATH: &str = "../rs-drive/tests/supporting_files/contract/restaurants/restaurants-contract.json"; /// The one property every fixture doctype groups by. - const GROUP_PROPERTY: &str = "restaurantId"; + pub(super) const GROUP_PROPERTY: &str = "restaurantId"; /// The last protocol version whose query table (v0) has no ranked /// path at all. Ranked routing activates at 14. - const PROTOCOL_VERSION_V13: u32 = 13; + pub(super) const PROTOCOL_VERSION_V13: u32 = 13; - fn register_restaurants( + pub(super) fn register_restaurants( platform: &Platform, platform_version: &PlatformVersion, ) -> DataContract { @@ -1992,7 +1994,7 @@ mod ranked_tests { /// `first_seed` seeds the random-document generator, which derives /// each document's id; two calls in one test need disjoint seed /// ranges or the second collides on an existing id. - fn insert_docs( + pub(super) fn insert_docs( platform: &Platform, contract: &DataContract, document_type_name: &str, @@ -2025,7 +2027,7 @@ mod ranked_tests { } } - fn select(function: v1_select::Function, field: &str) -> Vec { + pub(super) fn select(function: v1_select::Function, field: &str) -> Vec { vec![V1Select { function: function as i32, field: field.to_string(), @@ -2091,7 +2093,7 @@ mod ranked_tests { /// **page** — entries plus the `skipped` rank base — asserting the /// response landed on the `ranked` variant of `ResultData` rather /// than on `counts` / `sums` / `averages`. - fn ranked_page( + pub(super) fn ranked_page( platform: &Platform, state: &PlatformState, request: GetDocumentsRequestV1, @@ -2119,7 +2121,7 @@ mod ranked_tests { /// [`ranked_page`] for the majority of tests, which only care about /// the entries. - fn ranked_entries( + pub(super) fn ranked_entries( platform: &Platform, state: &PlatformState, request: GetDocumentsRequestV1, @@ -2133,7 +2135,7 @@ mod ranked_tests { /// validation result, which the gRPC layer turns into /// `invalid_argument` — and never as the `Err` arm, which becomes /// an opaque internal error. - fn ranked_error( + pub(super) fn ranked_error( platform: &Platform, state: &PlatformState, request: GetDocumentsRequestV1, @@ -2153,7 +2155,7 @@ mod ranked_tests { result.errors.into_iter().next().expect("checked non-empty") } - fn group_keys(entries: &[RankedEntry]) -> Vec { + pub(super) fn group_keys(entries: &[RankedEntry]) -> Vec { entries .iter() .map(|entry| String::from_utf8(entry.key.clone()).expect("fixture keys are utf-8")) @@ -2706,14 +2708,28 @@ mod ranked_tests { assert_eq!(sums(&bottom_one), vec![10]); } - /// A boolean `HAVING` alongside an aggregate ordering is refused - /// with the `not_yet_implemented` contract — a client can leave the - /// request in place and it starts working when the capability - /// lands. + /// A single boolean `HAVING` clause alongside an aggregate ordering + /// no longer rides the ranked path at all: the v2 routing helper + /// sends any grouped single-clause `having` to the having-range + /// surface, where an `ORDER BY` naming the selected aggregate is + /// legal and sets the walk direction. This request — a client left + /// in place across the capability landing, exactly as the old + /// `not_yet_implemented` contract promised — now answers. Full + /// having-range behaviour is pinned in [`super::having_range_tests`]; + /// this test pins the routing handoff from the ranked shape. #[test] - fn having_alongside_an_aggregate_ordering_is_still_unsupported() { + fn having_alongside_an_aggregate_ordering_now_routes_to_having_range() { let (platform, state, version) = setup_platform(None, Network::Testnet, None); let contract = register_restaurants(&platform, version); + insert_docs( + &platform, + &contract, + "review", + "grade", + 9_000, + &[("alpha", 90), ("beta", 30), ("gamma", 60)], + version, + ); let mut request = ranked_desc( &contract, @@ -2726,18 +2742,19 @@ mod ranked_tests { having_aggregate::Function::Avg, "grade", having_clause::Operator::GreaterThan, - Value::U64(4), + Value::U64(40), )]; - match ranked_error(&platform, &state, request, version) { - QueryError::Query(QuerySyntaxError::Unsupported(message)) => { - assert!( - message.contains("not yet implemented") && message.contains("HAVING"), - "expected the HAVING-with-ordering rejection, got: {message}" - ); - } - other => panic!("expected Unsupported, got {other:?}"), - } + let page = ranked_page(&platform, &state, request, version); + assert_eq!( + page.skipped, None, + "a having-range page has no rank base and must leave `skipped` unset" + ); + assert_eq!( + group_keys(&page.entries), + vec!["alpha", "gamma"], + "descending walk over averages above 40: alpha (90) then gamma (60)" + ); } /// Ordering by the selected aggregate **without** a `GROUP BY` is @@ -3032,3 +3049,373 @@ mod ranked_tests { } } } + +mod having_range_tests { + //! End-to-end coverage of the having-range + //! (`GROUP BY p HAVING LIMIT n`) + //! surface through the real v1 handler: wire request in, + //! `ResultData.ranked` with `skipped` unset (or a `Proof`) out. + //! + //! Shares the `restaurants` fixture with [`super::ranked_tests`] — + //! same cross-crate path, same doctype → axis table — because the + //! having-range surface reads the very same indexed trees; only the + //! addressing (value bound instead of rank) differs. Value-level + //! behaviour (bounds translation, proof round-trips, tamper + //! rejection) is pinned in rs-drive's + //! `drive_document_having_query::tests`; this suite pins the wire + //! encoding, the routing, and the rejection contracts. + + use super::ranked_tests::{ + group_keys, insert_docs, ranked_error, ranked_page, register_restaurants, select, + GROUP_PROPERTY, PROTOCOL_VERSION_V13, + }; + use super::*; + + /// The canonical having-range request: one aggregate select, one + /// `group_by`, one `having` clause on the selected aggregate, a + /// `limit`, and optionally an `order_by` naming the selected + /// aggregate. Everything else at its "unset" wire value. + #[allow(clippy::too_many_arguments)] + fn having_request( + contract: &dpp::prelude::DataContract, + document_type: &str, + selects: Vec, + clause: ProtoHavingClause, + order_by: Vec, + limit: Option, + prove: bool, + ) -> GetDocumentsRequestV1 { + GetDocumentsRequestV1 { + data_contract_id: contract.id().to_vec(), + document_type: document_type.to_string(), + where_clauses: Vec::new(), + order_by, + limit, + start: None, + prove, + selects, + group_by: vec![GROUP_PROPERTY.to_string()], + having: vec![clause], + offset: None, + } + } + + /// `SELECT COUNT(*) GROUP BY restaurantId HAVING $count > 2 + /// LIMIT 10` — the headline spam-resistant-discovery shape. No + /// `order_by`: ascending by count is the default, and `skipped` + /// must be unset because a value-bounded page has no rank base. + #[test] + fn count_threshold_returns_matching_entries_with_no_rank_base() { + let (platform, state, version) = setup_platform(None, Network::Testnet, None); + let contract = register_restaurants(&platform, version); + insert_docs( + &platform, + &contract, + "visit", + "guests", + 10_000, + &[ + ("alpha", 1), + ("beta", 1), + ("beta", 2), + ("beta", 3), + ("gamma", 1), + ("gamma", 2), + ("delta", 1), + ("delta", 2), + ("delta", 3), + ("delta", 4), + ], + version, + ); + + let request = having_request( + &contract, + "visit", + select(v1_select::Function::Count, ""), + hc( + having_aggregate::Function::Count, + "", + having_clause::Operator::GreaterThan, + Value::U64(2), + ), + Vec::new(), + Some(10), + false, + ); + + let page = ranked_page(&platform, &state, request, version); + assert_eq!( + page.skipped, None, + "a having-range page must leave the rank-based `skipped` field unset" + ); + assert_eq!( + group_keys(&page.entries), + vec!["beta", "delta"], + "ascending count order: beta (3 visits) before delta (4)" + ); + } + + /// `prove = true` answers with a `Proof` payload, exactly like the + /// ranked path. The proof's verifiability is pinned in rs-drive's + /// suite; here only the wire shape is asserted. + #[test] + fn a_having_request_with_prove_returns_a_proof() { + let (platform, state, version) = setup_platform(None, Network::Testnet, None); + let contract = register_restaurants(&platform, version); + insert_docs( + &platform, + &contract, + "visit", + "guests", + 11_000, + &[("alpha", 1), ("beta", 1), ("beta", 2), ("beta", 3)], + version, + ); + + let request = having_request( + &contract, + "visit", + select(v1_select::Function::Count, ""), + hc( + having_aggregate::Function::Count, + "", + having_clause::Operator::GreaterThan, + Value::U64(2), + ), + Vec::new(), + Some(10), + true, + ); + + let result = platform + .query_documents_v1(request, &state, version) + .expect("query call should not error at the transport layer"); + assert!( + result.errors.is_empty(), + "expected no validation errors, got {:?}", + result.errors + ); + match result.data { + Some(GetDocumentsResponseV1 { + result: Some(get_documents_response_v1::Result::Proof(_)), + metadata: Some(_), + }) => {} + other => panic!("expected a Proof result, got {:?}", other), + } + } + + /// Two clauses (implicit AND) keep the `not_yet_implemented` + /// contract, with a message that names the restriction rather than + /// the blanket "HAVING clause". + #[test] + fn multiple_clauses_are_still_not_implemented() { + let (platform, state, version) = setup_platform(None, Network::Testnet, None); + let contract = register_restaurants(&platform, version); + + let clause = hc( + having_aggregate::Function::Count, + "", + having_clause::Operator::GreaterThan, + Value::U64(2), + ); + let mut request = having_request( + &contract, + "visit", + select(v1_select::Function::Count, ""), + clause.clone(), + Vec::new(), + Some(10), + false, + ); + request.having.push(clause); + + match ranked_error(&platform, &state, request, version) { + QueryError::Query(QuerySyntaxError::Unsupported(message)) => { + assert!( + message.contains("multiple HAVING clauses") + && message.contains("not yet implemented"), + "expected the multi-clause rejection, got: {message}" + ); + } + other => panic!("expected Unsupported, got {other:?}"), + } + } + + /// `OFFSET` stays ranked-only: the having-range walk has no skip, + /// so the post-routing offset gate fires with its long-standing + /// message. + #[test] + fn offset_is_rejected_on_the_having_path() { + let (platform, state, version) = setup_platform(None, Network::Testnet, None); + let contract = register_restaurants(&platform, version); + + let mut request = having_request( + &contract, + "visit", + select(v1_select::Function::Count, ""), + hc( + having_aggregate::Function::Count, + "", + having_clause::Operator::GreaterThan, + Value::U64(2), + ), + Vec::new(), + Some(10), + false, + ); + request.offset = Some(1); + + match ranked_error(&platform, &state, request, version) { + QueryError::Query(QuerySyntaxError::Unsupported(message)) => { + assert!( + message.contains("OFFSET pagination"), + "expected the offset gate's message, got: {message}" + ); + } + other => panic!("expected Unsupported, got {other:?}"), + } + } + + /// A bound on an axis the index does not declare surfaces as a + /// query error naming the missing contract keyword. The `review` + /// doctype's index is `rankedAverageable` only. + #[test] + fn a_bound_on_an_undeclared_axis_names_the_missing_keyword() { + let (platform, state, version) = setup_platform(None, Network::Testnet, None); + let contract = register_restaurants(&platform, version); + + let request = having_request( + &contract, + "review", + select(v1_select::Function::Count, ""), + hc( + having_aggregate::Function::Count, + "", + having_clause::Operator::GreaterThan, + Value::U64(2), + ), + Vec::new(), + Some(10), + false, + ); + + let error = ranked_error(&platform, &state, request, version); + assert!( + format!("{error}").contains("rankedCountable"), + "the rejection must name the missing keyword, got: {error}" + ); + } + + /// Non-contiguous operators (`!=`, `IN`) reach drive and are + /// refused there with a message explaining the contiguity + /// requirement — as a query error, never an internal one. + #[test] + fn non_contiguous_operators_surface_as_query_errors() { + let (platform, state, version) = setup_platform(None, Network::Testnet, None); + let contract = register_restaurants(&platform, version); + + for operator in [ + having_clause::Operator::NotEqual, + having_clause::Operator::In, + ] { + let request = having_request( + &contract, + "visit", + select(v1_select::Function::Count, ""), + hc( + having_aggregate::Function::Count, + "", + operator, + Value::U64(2), + ), + Vec::new(), + Some(10), + false, + ); + match ranked_error(&platform, &state, request, version) { + QueryError::Query(QuerySyntaxError::Unsupported(message)) => { + assert!( + message.contains("contiguous"), + "expected the contiguity rejection for {operator:?}, got: {message}" + ); + } + other => panic!("expected Unsupported for {operator:?}, got {other:?}"), + } + } + } + + /// Protocol version 13's query table (v0 helper) has no having + /// path: the same request a v14 node answers is refused with the + /// blanket rejection. The routing gate fires before any contract + /// fetch, so no ranked contract is needed (v13's meta-schema could + /// not register one anyway). + #[test] + fn protocol_version_13_still_rejects_having() { + let (platform, state, version) = + setup_platform(None, Network::Testnet, Some(PROTOCOL_VERSION_V13)); + + let request = GetDocumentsRequestV1 { + data_contract_id: vec![0u8; 32], + document_type: "visit".to_string(), + where_clauses: Vec::new(), + order_by: Vec::new(), + limit: Some(10), + start: None, + prove: false, + selects: select(v1_select::Function::Count, ""), + group_by: vec![GROUP_PROPERTY.to_string()], + having: vec![hc( + having_aggregate::Function::Count, + "", + having_clause::Operator::GreaterThan, + Value::U64(2), + )], + offset: None, + }; + + match ranked_error(&platform, &state, request, version) { + QueryError::Query(QuerySyntaxError::Unsupported(message)) => { + assert!( + message.contains("HAVING clause") && message.contains("not yet implemented"), + "expected v13's blanket rejection, got: {message}" + ); + } + other => panic!("expected Unsupported, got {other:?}"), + } + } + + /// The routing label: a grouped aggregate with one having clause + /// routes to `having_range`, and without the group_by it stays on + /// the blanket-rejection path. + #[test] + fn routing_picks_having_range_only_for_grouped_single_clause_having() { + let clause = hc( + having_aggregate::Function::Count, + "", + having_clause::Operator::GreaterThan, + Value::U64(2), + ); + + let grouped = GetDocumentsRequestV1 { + selects: select_count_star(), + group_by: vec![GROUP_PROPERTY.to_string()], + having: vec![clause.clone()], + limit: Some(10), + ..empty_v1_request() + }; + let label = validate_and_route_for_tests(&grouped, &[], PlatformVersion::latest()) + .expect("a grouped single-clause having is a supported shape"); + assert_eq!(label, "having_range"); + + // No group_by → the v0 blanket rejection still owns it. + let ungrouped = GetDocumentsRequestV1 { + group_by: Vec::new(), + ..grouped + }; + assert_not_yet_implemented( + validate_and_route_for_tests(&ungrouped, &[], PlatformVersion::latest()), + "HAVING clause", + ); + } +} diff --git a/packages/rs-drive-proof-verifier/src/lib.rs b/packages/rs-drive-proof-verifier/src/lib.rs index 0a5bf872d42..f91801de6e1 100644 --- a/packages/rs-drive-proof-verifier/src/lib.rs +++ b/packages/rs-drive-proof-verifier/src/lib.rs @@ -14,6 +14,14 @@ pub use proof::document_count::{ verify_distinct_count_proof, verify_point_lookup_count_proof, verify_primary_key_count_tree_proof, DocumentCount, }; +/// Verified having-range (`GROUP BY … HAVING +/// LIMIT n`) result types. `DocumentHavingEntries` carries one entry +/// per matching group **in axis order**; +/// [`verify_having_range_proof`] is the tenderdash-composition wrapper +/// that binds the proof's reconstructed root hash to the signed app +/// hash and returns the verified entry list — including its +/// completeness: an in-range group the node omitted fails verification. +pub use proof::document_having::{verify_having_range_proof, DocumentHavingEntries}; /// Verified ranked (`GROUP BY … ORDER BY LIMIT n /// [OFFSET m]`) result types. `DocumentRankedEntries` carries one entry /// per returned group **in ranking order**, plus the `starting_rank` diff --git a/packages/rs-drive-proof-verifier/src/proof.rs b/packages/rs-drive-proof-verifier/src/proof.rs index eb47bdab25c..ffcd66171da 100644 --- a/packages/rs-drive-proof-verifier/src/proof.rs +++ b/packages/rs-drive-proof-verifier/src/proof.rs @@ -4,6 +4,11 @@ /// `AggregateCountAndSumOnRange` primitive. pub mod document_average; pub mod document_count; +/// Verified having-range (`GROUP BY … HAVING +/// LIMIT n`) result. One entry per matching group, in axis order, read +/// as a value-bounded range of an indexed tree's per-axis secondary +/// (grovedb PR 657); see the file's docs. +pub mod document_having; /// Verified ranked (`GROUP BY … ORDER BY LIMIT n /// [OFFSET m]`) result. One entry per returned group, in ranking order, /// plus the attested rank the page starts at, read from an indexed diff --git a/packages/rs-drive-proof-verifier/src/proof/document_having.rs b/packages/rs-drive-proof-verifier/src/proof/document_having.rs new file mode 100644 index 00000000000..9120440e1e5 --- /dev/null +++ b/packages/rs-drive-proof-verifier/src/proof/document_having.rs @@ -0,0 +1,333 @@ +//! Verified **having-range** +//! (`GROUP BY … HAVING LIMIT n`) document +//! results. +//! +//! A having-range query answers "which groups' aggregate falls inside a +//! value bound?" — `SELECT COUNT(*) GROUP BY hashtag HAVING $count > 100 +//! LIMIT 100`. The answer is a value-bounded range read of the same +//! per-axis *secondary* Merk the ranked query walks, so it costs +//! `O(log n + k)` and comes with a proof that commits to exactly the +//! returned `(aggregate, group key)` pairs **and their completeness**: +//! the Merk range proof commits its boundaries, so an in-range group the +//! node omitted fails verification. +//! +//! This module holds the client-facing result type +//! ([`DocumentHavingEntries`]), the tenderdash-composition wrapper +//! around rs-drive's merk-level verifier +//! ([`verify_having_range_proof`]), and the decoder for the unproven +//! wire payload ([`DocumentHavingEntries::from_unproved_response`]) — +//! which rides the same `ResultData.ranked` variant the ranked surface +//! uses, since a having page is the same "group key + aggregate value" +//! entry list. +//! +//! Per-shape routing (which index covers the axis, which bounds the +//! clause translates to) lives in rs-sdk's `having_proof_helpers`, +//! exactly as the ranked equivalents live in `ranked_proof_helpers` — +//! it needs the data contract, which this crate does not carry. + +use crate::error::MapGroveDbError; +use crate::proof::document_ranked::ranked_entry_from_proto; +use crate::verify::verify_tenderdash_proof; +use crate::{ContextProvider, Error, FromProof}; +use dapi_grpc::platform::v0::get_documents_response::get_documents_response_v1::{ + result_data, ResultData, +}; +use dapi_grpc::platform::v0::get_documents_response::{ + get_documents_response_v1, Version as ResponseVersion, +}; +use dapi_grpc::platform::v0::{GetDocumentsResponse, Proof, ResponseMetadata}; +use dpp::dashcore::Network; +use dpp::version::PlatformVersion; +use drive::query::{DriveDocumentHavingQuery, DriveDocumentQuery, RankedEntry}; +use drive::verify::RootHash; + +/// One page of a `GROUP BY … HAVING LIMIT n` +/// query: the groups whose aggregate falls inside the bound. +/// +/// **Entry order is axis order in the walk direction** — ascending by +/// default, descending when the request ordered by the aggregate +/// descending. Callers must not re-sort; ties (groups with equal +/// aggregates) come back in group-key order in the direction of the +/// walk, same as on the ranked surface. +/// +/// Fewer than `n` entries means fewer groups matched — not an error. +/// **Exactly `n` entries may mean the match set was cut at the limit**; +/// nothing in the page marks the cut, so a caller that needs the full +/// set tightens the bound (moves the threshold past the last aggregate +/// value already seen) and asks again. +/// +/// Entry semantics ([`RankedEntry`]) are identical to the ranked +/// surface's, including the fixed-point average scaling and the +/// exact-on-the-proved-path-only caveat. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct DocumentHavingEntries { + /// The matching groups, in axis order in the walk direction. + pub entries: Vec, +} + +impl DocumentHavingEntries { + /// Build a [`DocumentHavingEntries`] from the verifier-side entry + /// list — the shape rs-drive's merk-level verifier returns. + pub fn from_verified(entries: Vec) -> Self { + DocumentHavingEntries { entries } + } + + /// Decode the **unproven** having-range payload of a `getDocuments` + /// response — the `ResultData.ranked` variant a node returns for a + /// having-range request sent with `prove = false`. (The wire reuses + /// the ranked entries message; a having page leaves its `skipped` + /// field unset, and this decoder ignores it either way, because a + /// value-bounded page has no rank base for it to describe.) + /// + /// Order is preserved verbatim. This is a plain wire decode with + /// **no cryptographic guarantee whatsoever** — and unlike the ranked + /// surface the missing guarantee here includes *completeness*: an + /// unproven page is free to omit matching groups, which for a + /// spam-resistance query is precisely the interesting attack. Prefer + /// [`verify_having_range_proof`] (via rs-sdk's + /// `DocumentHavingEntries::fetch`) unless you deliberately trust the + /// node. + /// + /// # Errors + /// + /// - [`Error::EmptyVersion`] when the response carries no version. + /// - [`Error::ResponseDecodeError`] when the response is a V0 + /// response, carries a proof rather than data, carries a + /// non-ranked `ResultData` variant, or an entry's `value` oneof is + /// unset / out of domain. + pub fn from_unproved_response( + response: &GetDocumentsResponse, + ) -> Result<(Self, ResponseMetadata), Error> { + let version = response.version.as_ref().ok_or(Error::EmptyVersion)?; + let ResponseVersion::V1(v1) = version else { + return Err(Error::ResponseDecodeError { + error: "having-range results are a V1-only response shape; got a V0 \ + getDocuments response. Having-range queries require protocol \ + version 14+." + .to_string(), + }); + }; + let metadata = v1.metadata.clone().ok_or(Error::EmptyResponseMetadata)?; + let entries = match v1.result.as_ref() { + Some(get_documents_response_v1::Result::Data(ResultData { + variant: Some(result_data::Variant::Ranked(ranked)), + })) => ranked + .entries + .iter() + .map(ranked_entry_from_proto) + .collect::, _>>()?, + Some(get_documents_response_v1::Result::Proof(_)) => { + return Err(Error::ResponseDecodeError { + error: "the response carries a proof, not unproven having-range entries; \ + verify it with `verify_having_range_proof` instead of decoding it" + .to_string(), + }); + } + other => { + return Err(Error::ResponseDecodeError { + error: format!( + "expected a `ResultData.ranked` payload for a having-range request, \ + got {other:?}. A response on another variant means the node routed \ + the request to a different executor — check that the request \ + carries a `group_by` and exactly one `having` clause bounding the \ + single `select`'s aggregate." + ), + }); + } + }; + Ok((DocumentHavingEntries { entries }, metadata)) + } +} + +/// Verify a grovedb indexed-axis range proof **and the surrounding +/// tenderdash commit**, returning the reconstructed root hash and the +/// matching groups it commits to. +/// +/// Thin tenderdash-composition wrapper over +/// [`DriveDocumentHavingQuery::verify_having_range_proof`] in rs-drive +/// (which does the merk-level verification). Both sides derive the +/// proved subtree from the same +/// `DriveDocumentHavingQuery::indexed_property_name_tree_path` and the +/// secondary query from the same `AxisRangeBounds::merk_query`, so +/// prover and verifier cannot drift on *which bound over which tree* is +/// being checked, and grovedb re-checks the echoed query and limit — a +/// proof of one bound does not verify as another. +/// +/// ## The root hash is the whole point +/// +/// Same as on the ranked surface: the merk-level verifier returning +/// `Ok` is not by itself evidence of anything — the binding to the +/// quorum-signed app hash in [`verify_tenderdash_proof`] is what makes +/// the entries (and their completeness) attested facts. This function +/// exists so that composition can never be skipped by accident. +pub fn verify_having_range_proof( + query: &DriveDocumentHavingQuery, + proof: &Proof, + mtd: &ResponseMetadata, + platform_version: &PlatformVersion, + provider: &dyn ContextProvider, +) -> Result<(RootHash, Vec), Error> { + let (root_hash, entries) = query + .verify_having_range_proof(&proof.grovedb_proof, platform_version) + .map_drive_error(proof, mtd)?; + + verify_tenderdash_proof(proof, mtd, &root_hash, provider)?; + + Ok((root_hash, entries)) +} + +/// Reject the generic [`FromProof`] entry point for +/// [`DocumentHavingEntries`] — same guard rail, same rationale as the +/// [`crate::DocumentRankedEntries`] blanket impl: the generic +/// `FromProof>` path carries neither the +/// bounds nor the covering index, so it errors out explicitly rather +/// than verifying the wrong thing. +impl<'dq, Q> FromProof for DocumentHavingEntries +where + Q: TryInto> + Clone + 'dq, + Q::Error: std::fmt::Display, +{ + type Request = Q; + type Response = GetDocumentsResponse; + + fn maybe_from_proof_with_metadata<'a, I: Into, O: Into>( + _request: I, + _response: O, + _network: Network, + _platform_version: &PlatformVersion, + _provider: &'a dyn ContextProvider, + ) -> Result<(Option, ResponseMetadata, Proof), Error> + where + Self: 'a, + { + Err(Error::RequestError { + error: "DocumentHavingEntries can't be verified via the generic FromProof path; \ + call DocumentHavingEntries::fetch on a DocumentQuery carrying \ + .with_select(), .with_group_by(), \ + .with_having() and \ + .with_limit(n), which resolves the bounds and the covering index from \ + the data contract" + .to_string(), + }) + } +} + +#[cfg(test)] +mod tests { + //! Offline tests for the unproven decode and the response-shape + //! rejections. Proof verification itself is exercised end-to-end by + //! rs-drive's `drive_document_having_query::tests` (prover and + //! verifier against a real Drive) and rs-drive-abci's + //! `having_range_tests` (wire encoding of the same values). + use super::*; + use dapi_grpc::platform::v0::get_documents_response::get_documents_response_v1::{ + ranked_entry, Documents, RankedEntries, RankedEntry as ProtoRankedEntry, + }; + use dapi_grpc::platform::v0::get_documents_response::GetDocumentsResponseV1; + use drive::query::RankedEntryValue; + + fn count_entry(key: &str, count: u64) -> ProtoRankedEntry { + ProtoRankedEntry { + key: key.as_bytes().to_vec(), + value: Some(ranked_entry::Value::Count(count)), + } + } + + fn response_with(result: get_documents_response_v1::Result) -> GetDocumentsResponse { + GetDocumentsResponse { + version: Some(ResponseVersion::V1(GetDocumentsResponseV1 { + result: Some(result), + metadata: Some(ResponseMetadata { + height: 42, + ..Default::default() + }), + })), + } + } + + fn having_response( + entries: Vec, + skipped: Option, + ) -> GetDocumentsResponse { + response_with(get_documents_response_v1::Result::Data(ResultData { + variant: Some(result_data::Variant::Ranked(RankedEntries { + entries, + skipped, + })), + })) + } + + /// The headline decode: `HAVING $count > 100`-shaped entries come + /// back in axis order, untouched, with `skipped` (unset on a + /// having page) ignored. + #[test] + fn decodes_entries_preserving_axis_order() { + let response = having_response( + vec![count_entry("dash", 101), count_entry("evo", 250)], + None, + ); + let (decoded, metadata) = DocumentHavingEntries::from_unproved_response(&response) + .expect("a well-formed having payload decodes"); + assert_eq!(metadata.height, 42); + assert_eq!( + decoded.entries.iter().map(|e| e.value).collect::>(), + vec![RankedEntryValue::Count(101), RankedEntryValue::Count(250)] + ); + } + + /// A stray `skipped` from a non-conforming node is ignored, not a + /// decode failure: the field cannot describe anything on a + /// value-bounded page, and failing on it would break against a + /// node that reused its ranked encoder wholesale. + #[test] + fn a_stray_skipped_field_is_ignored() { + let response = having_response(vec![count_entry("dash", 101)], Some(7)); + let (decoded, _) = DocumentHavingEntries::from_unproved_response(&response) + .expect("a stray skipped is not a decode failure"); + assert_eq!(decoded.entries.len(), 1); + } + + /// No groups matching the bound is a legitimate answer. + #[test] + fn decodes_an_empty_match_set() { + let (decoded, _) = + DocumentHavingEntries::from_unproved_response(&having_response(vec![], None)) + .expect("an empty match set is well-formed"); + assert!(decoded.entries.is_empty()); + } + + /// Same caller-mistake guard as the ranked decoder: a proof must + /// be verified, not decoded. + #[test] + fn rejects_a_proof_response() { + let response = response_with(get_documents_response_v1::Result::Proof(Proof::default())); + let err = DocumentHavingEntries::from_unproved_response(&response) + .expect_err("a proof is not an unproven having payload"); + assert!(format!("{err}").contains("verify_having_range_proof")); + } + + /// A response on another variant means the node routed the request + /// somewhere else entirely. + #[test] + fn rejects_a_non_ranked_result_variant() { + let response = response_with(get_documents_response_v1::Result::Data(ResultData { + variant: Some(result_data::Variant::Documents(Documents { + documents: Vec::new(), + })), + })); + let err = DocumentHavingEntries::from_unproved_response(&response) + .expect_err("a documents payload is not a having one"); + assert!(format!("{err}").contains("ResultData.ranked")); + } + + /// V0 predates the SQL-shaped surface entirely. + #[test] + fn rejects_a_v0_response() { + let response = GetDocumentsResponse { + version: Some(ResponseVersion::V0(Default::default())), + }; + let err = DocumentHavingEntries::from_unproved_response(&response) + .expect_err("V0 has no having shape"); + assert!(format!("{err}").contains("V1-only")); + } +} diff --git a/packages/rs-drive-proof-verifier/src/proof/document_ranked.rs b/packages/rs-drive-proof-verifier/src/proof/document_ranked.rs index 6239eca48e9..d562a7dcb11 100644 --- a/packages/rs-drive-proof-verifier/src/proof/document_ranked.rs +++ b/packages/rs-drive-proof-verifier/src/proof/document_ranked.rs @@ -224,7 +224,7 @@ impl DocumentRankedEntries { /// out-of-range double into `i128::MIN`/`MAX`. Every legitimate value /// fits comfortably, since `|sum| ≤ i64::MAX` bounds the true fixed /// point at `i64::MAX * 10^19 ≈ 9.2e37 < i128::MAX`. -fn ranked_entry_from_proto(entry: &ProtoRankedEntry) -> Result { +pub(crate) fn ranked_entry_from_proto(entry: &ProtoRankedEntry) -> Result { let value = match entry.value.as_ref() { Some(ranked_entry::Value::Count(count)) => RankedEntryValue::Count(*count), Some(ranked_entry::Value::Sum(sum)) => RankedEntryValue::Sum(*sum), diff --git a/packages/rs-drive/src/query/drive_document_having_query/drive_dispatcher.rs b/packages/rs-drive/src/query/drive_document_having_query/drive_dispatcher.rs new file mode 100644 index 00000000000..8c5b35da0c6 --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_having_query/drive_dispatcher.rs @@ -0,0 +1,147 @@ +//! [`DocumentHavingRequest`] / [`DocumentHavingResponse`] and the +//! having-range dispatcher on `impl Drive` — the ABI drive-abci's +//! routing layer names. + +use super::super::drive_document_ranked_query::{RankedEntry, RankedPaginationInputs}; +use super::mode_detection::detect_having_mode; +use crate::drive::Drive; +use crate::error::Error; +use crate::query::having::HavingClause; +use crate::query::projection::SelectProjection; +use crate::query::{OrderClause, WhereClause}; +use dpp::data_contract::accessors::v0::DataContractV0Getters; +use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; +use dpp::data_contract::document_type::DocumentTypeRef; +use dpp::data_contract::DataContract; +use dpp::version::PlatformVersion; +use grovedb::TransactionArg; + +/// All inputs required by [`Drive::execute_document_having_request`]. +/// Built by the gRPC handler from a `GetDocumentsRequestV1` after +/// wire-decoding + contract lookup — the same construction pattern as +/// [`super::super::drive_document_ranked_query::DocumentRankedRequest`]. +/// +/// `where_clauses`, `offset` and `start_at` are carried even though a +/// having-range request must leave all of them empty: drive owns the +/// rejection, so the contract is enforced identically no matter which +/// upstream path built the request. See +/// [`super::mode_detection::detect_having_mode_v0`] for why each is +/// refused rather than ignored. +pub struct DocumentHavingRequest<'a> { + /// Live contract (already loaded by the handler). + pub contract: &'a DataContract, + /// Resolved document type within `contract`. + pub document_type: DocumentTypeRef<'a>, + /// The single `GROUP BY` property. Must be the ranked index's only + /// property. + pub group_by: &'a [String], + /// The projection whose aggregate the `having` clause bounds: + /// `COUNT(*)`, `SUM(field)` or `AVG(field)`. + pub select: SelectProjection, + /// The `HAVING` clauses. Exactly one, bounding the selected + /// aggregate. + pub having: &'a [HavingClause], + /// The `ORDER BY` clauses. Empty (ascending default) or exactly + /// one, naming the selected aggregate. + pub order_by: &'a [OrderClause], + /// Structured `where` clauses. Must be empty. + pub where_clauses: &'a [WhereClause], + /// Request `limit`. **Required**; `1 ..= MAX_HAVING_LIMIT`. + pub limit: Option, + /// Request `offset`. Must be `None` — the range walk has no skip. + pub offset: Option, + /// Whether the request carried a `start_at` / `start_after` cursor. + /// Must be `false`. + pub has_start_at: bool, + /// Whether to produce a proof instead of materializing entries. + pub prove: bool, +} + +/// Output shape of [`Drive::execute_document_having_request`]. +/// +/// - `Entries` — the matching groups **in axis order in the walk +/// direction**; the abci handler maps this straight onto the wire's +/// ranked-entries shape (with no rank base) without re-sorting. +/// - `Proof(Vec)` — grovedb indexed-axis range proof bytes the +/// client verifies with +/// [`DriveDocumentHavingQuery::verify_having_range_proof`](crate::query::DriveDocumentHavingQuery::verify_having_range_proof), +/// which recovers the same entry list. +#[derive(Debug, Clone)] +pub enum DocumentHavingResponse { + /// The groups whose aggregate falls inside the bound, cut at the + /// request's limit. + Entries(Vec), + /// Grovedb indexed-axis range proof bytes. + Proof(Vec), +} + +impl Drive { + /// Single entry point for a having-range document request. + /// + /// 1. [`detect_having_mode`] validates the request shape and + /// resolves the `(bounds, descending, limit, group property, + /// aggregate field)` tuple. + /// 2. The matching executor picks the covering ranked index and runs + /// the read or the proof. + /// 3. The result is wrapped in [`DocumentHavingResponse`]. + /// + /// Errors: + /// - Request-shape failures (wrong `group_by` arity, a clause on an + /// aggregate the select does not project, an untranslatable + /// operator, a missing or out-of-range `limit`, a `where`, an + /// `offset`) come back as `Error::Query(QuerySyntaxError::*)` — + /// see [`super::mode_detection::detect_having_mode_v0`] for the + /// full grammar. + /// - "No index declares this axis" comes back as + /// `Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty)` + /// naming the missing contract keyword. + /// - Everything else (grovedb, versioning) surfaces as its native + /// `Error` variant. + pub fn execute_document_having_request( + &self, + request: DocumentHavingRequest, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result { + let mode = detect_having_mode( + &request.select, + request.group_by, + request.having, + request.order_by, + request.where_clauses, + RankedPaginationInputs { + limit: request.limit, + offset: request.offset, + has_start_at: request.has_start_at, + }, + platform_version, + )?; + + let contract_id = request.contract.id_ref().to_buffer(); + let document_type_name = request.document_type.name().to_string(); + + if request.prove { + Ok(DocumentHavingResponse::Proof( + self.execute_document_having_range_proof( + contract_id, + request.document_type, + document_type_name, + &mode, + transaction, + platform_version, + )?, + )) + } else { + Ok(DocumentHavingResponse::Entries( + self.execute_document_having_range_no_proof( + contract_id, + request.document_type, + document_type_name, + &mode, + transaction, + platform_version, + )?, + )) + } + } +} diff --git a/packages/rs-drive/src/query/drive_document_having_query/execute_range.rs b/packages/rs-drive/src/query/drive_document_having_query/execute_range.rs new file mode 100644 index 00000000000..c9188d91f93 --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_having_query/execute_range.rs @@ -0,0 +1,176 @@ +//! The two having-range executors on [`DriveDocumentHavingQuery`]: a +//! direct value-bounded read of the axis secondary, and generation of +//! the equivalent proof. +//! +//! Both are thin — all of the work happens inside grovedb, which seeks +//! straight to the encoded bounds in the pre-sorted secondary Merk. No +//! value trees are opened, no documents are materialized, and the cost +//! is `O(log n + k)` in the number of *matching* groups returned, never +//! in the total group population. +//! +//! Whole module is gated `feature = "server"` via the parent's +//! `pub mod execute_range;` declaration. + +use super::super::drive_document_ranked_query::{RankedAxis, RankedEntry, RankedEntryValue}; +use super::{AxisRangeBounds, DriveDocumentHavingQuery}; +use crate::drive::Drive; +use crate::error::drive::DriveError; +use crate::error::Error; +use dpp::version::PlatformVersion; +use grovedb::TransactionArg; +use grovedb_costs::CostContext; + +impl DriveDocumentHavingQuery<'_> { + /// Read the matching groups directly from the axis secondary: every + /// group whose aggregate falls inside the bounds, up to `limit`, in + /// axis order in the walk direction. + /// + /// Fewer than `limit` entries is normal (fewer groups match) and is + /// not an error; exactly `limit` entries may mean the match set was + /// cut. A missing path *is* an error rather than an empty result, + /// for the same reason as on the ranked surface: the indexed + /// property-name tree is created at contract registration, so its + /// absence means the contract-level state is not what the request + /// claims. (An index with no documents has the tree, with an empty + /// secondary, and yields an empty entry list.) + pub fn execute_range_no_proof( + &self, + drive: &Drive, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result, Error> { + let grove_version = &platform_version.drive.grove_version; + let path = self.indexed_property_name_tree_path()?; + let path_refs: Vec<&[u8]> = path.iter().map(|segment| segment.as_slice()).collect(); + + // Costs are destructured away rather than `.unwrap()`-ed, same + // as the ranked executors: `CostContext::unwrap` is infallible + // but reads like a panicking unwrap at the call site. + let entries = match self.bounds { + AxisRangeBounds::Count { lo, hi } => { + let CostContext { value, cost: _ } = drive.grove.indexed_count_range( + path_refs.as_slice(), + lo, + hi, + self.descending, + self.limit, + transaction, + grove_version, + ); + value + .map_err(|e| Error::GroveDB(Box::new(e)))? + .into_iter() + .map(|(count, key)| RankedEntry { + key, + value: RankedEntryValue::Count(count), + }) + .collect::>() + } + AxisRangeBounds::Sum { lo, hi } => { + let CostContext { value, cost: _ } = drive.grove.indexed_sum_range( + path_refs.as_slice(), + lo, + hi, + self.descending, + self.limit, + transaction, + grove_version, + ); + value + .map_err(|e| Error::GroveDB(Box::new(e)))? + .into_iter() + .map(|(sum, key)| RankedEntry { + key, + value: RankedEntryValue::Sum(sum), + }) + .collect::>() + } + AxisRangeBounds::Avg { lo, hi } => { + let CostContext { value, cost: _ } = drive.grove.indexed_avg_range( + path_refs.as_slice(), + lo, + hi, + self.descending, + self.limit, + transaction, + grove_version, + ); + value + .map_err(|e| Error::GroveDB(Box::new(e)))? + .into_iter() + .map(|(avg, key)| RankedEntry { + key, + value: RankedEntryValue::AvgFixedPoint(avg), + }) + .collect::>() + } + }; + + // The limit is the contract with the caller, and on the prove + // path it is re-checked inside the proof envelope. Asserting it + // here keeps the no-proof and prove responses shape-identical. + if entries.len() > self.limit as usize { + return Err(Error::Drive(DriveError::CorruptedDriveState(format!( + "having {:?} range read returned {} entries for limit = {}", + self.bounds.axis(), + entries.len(), + self.limit + )))); + } + Ok(entries) + } + + /// Generate the grovedb indexed-axis range proof for this query. + /// + /// The envelope commits the in-range secondary entries, the + /// primary's root hash, the sibling axes' root hashes, and a + /// per-ancestor attestation chain up to the grovedb root — so the + /// client reconstructs the platform root hash from it. The Merk + /// query (the encoded bounds and walk direction) and the limit are + /// echoed and re-checked by grovedb's verifier against the client's + /// own reconstruction via [`AxisRangeBounds::merk_query`] — which is + /// why the bounds are validated rather than clamped upstream, and + /// why completeness needs no extra machinery: a Merk range proof + /// over a sorted keyspace commits its boundaries, so an in-range + /// group the server omitted fails reconstruction. + /// + /// Verified by + /// [`DriveDocumentHavingQuery::verify_having_range_proof`](crate::query::DriveDocumentHavingQuery::verify_having_range_proof). + pub fn execute_range_with_proof( + &self, + drive: &Drive, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result, Error> { + let grove_version = &platform_version.drive.grove_version; + let path = self.indexed_property_name_tree_path()?; + let path_refs: Vec<&[u8]> = path.iter().map(|segment| segment.as_slice()).collect(); + let secondary_query = self.bounds.merk_query(self.descending); + + // Same destructure-don't-unwrap rationale as the no-proof arm. + let CostContext { value, cost: _ } = match self.bounds.axis() { + RankedAxis::Count => drive.grove.prove_indexed_count_query( + path_refs.as_slice(), + secondary_query, + Some(self.limit), + transaction, + grove_version, + ), + RankedAxis::Sum => drive.grove.prove_indexed_sum_query( + path_refs.as_slice(), + secondary_query, + Some(self.limit), + transaction, + grove_version, + ), + RankedAxis::Avg => drive.grove.prove_indexed_avg_query( + path_refs.as_slice(), + secondary_query, + Some(self.limit), + transaction, + grove_version, + ), + }; + value.map_err(|e| Error::GroveDB(Box::new(e))) + } +} diff --git a/packages/rs-drive/src/query/drive_document_having_query/executors.rs b/packages/rs-drive/src/query/drive_document_having_query/executors.rs new file mode 100644 index 00000000000..97793ee923f --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_having_query/executors.rs @@ -0,0 +1,117 @@ +//! Per-mode having-range executors on `impl Drive`, plus the shared +//! mode-to-query resolution. The dispatcher +//! ([`super::drive_dispatcher`]) picks between the two executors on the +//! request's `prove` flag. +//! +//! Index resolution reuses the ranked surface's covering-index picker +//! ([`find_ranked_index_for_axis`]) — both surfaces read the same +//! indexed tree, and sharing the picker is what guarantees a proof and +//! an unproven read are about the same subtree. + +use super::super::drive_document_ranked_query::index_picker::find_ranked_index_for_axis; +use super::super::drive_document_ranked_query::RankedEntry; +use super::{DocumentHavingMode, DriveDocumentHavingQuery}; +use crate::drive::Drive; +use crate::error::query::QuerySyntaxError; +use crate::error::Error; +use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; +use dpp::data_contract::document_type::{DocumentTypeRef, Index}; +use dpp::version::PlatformVersion; +use grovedb::TransactionArg; +use std::collections::BTreeMap; + +/// Resolve a validated [`DocumentHavingMode`] against a document type's +/// indexes into the executable [`DriveDocumentHavingQuery`]. +/// +/// `indexes` is threaded in separately for the same lifetime reason as +/// the ranked resolver: the returned query's `&'a Index` must outlive +/// this frame. Callers pass `document_type.indexes()`. +pub(super) fn having_query_for_mode<'a>( + contract_id: [u8; 32], + document_type: DocumentTypeRef<'a>, + document_type_name: String, + indexes: &'a BTreeMap, + mode: &DocumentHavingMode, +) -> Result, Error> { + let axis = mode.bounds.axis(); + let index = find_ranked_index_for_axis( + indexes, + &mode.group_by_property, + axis, + &mode.aggregate_field, + ) + .ok_or_else(|| { + Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty(format!( + "no ranked index covers `group_by = [{}]` on the {:?} axis: a `having` bound \ + is served from that axis's pre-sorted secondary, so the document type needs \ + a single-property index on `{}` declaring `{}`{}", + mode.group_by_property, + axis, + mode.group_by_property, + axis.required_index_keyword(), + if mode.aggregate_field.is_empty() { + String::new() + } else { + format!(" with `summable: \"{}\"`", mode.aggregate_field) + } + ))) + })?; + Ok(DriveDocumentHavingQuery { + document_type, + contract_id, + document_type_name, + index, + bounds: mode.bounds, + descending: mode.descending, + limit: mode.limit, + }) +} + +impl Drive { + /// One page of groups matching a having bound, read without a proof. + pub fn execute_document_having_range_no_proof( + &self, + contract_id: [u8; 32], + document_type: DocumentTypeRef, + document_type_name: String, + mode: &DocumentHavingMode, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result, Error> { + let indexes = document_type.indexes(); + let having_query = having_query_for_mode( + contract_id, + document_type, + document_type_name, + indexes, + mode, + )?; + having_query.execute_range_no_proof(self, transaction, platform_version) + } + + /// Proof of one page of groups matching a having bound. + /// + /// The client verifies it with + /// [`DriveDocumentHavingQuery::verify_having_range_proof`](crate::query::DriveDocumentHavingQuery::verify_having_range_proof), + /// reconstructing the same query from the same contract — which is + /// why index resolution is shared with the no-proof executor. + pub fn execute_document_having_range_proof( + &self, + contract_id: [u8; 32], + document_type: DocumentTypeRef, + document_type_name: String, + mode: &DocumentHavingMode, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result, Error> { + let indexes = document_type.indexes(); + let having_query = having_query_for_mode( + contract_id, + document_type, + document_type_name, + indexes, + mode, + )?; + having_query.execute_range_with_proof(self, transaction, platform_version) + } +} diff --git a/packages/rs-drive/src/query/drive_document_having_query/mod.rs b/packages/rs-drive/src/query/drive_document_having_query/mod.rs new file mode 100644 index 00000000000..c2f5d68b1d4 --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_having_query/mod.rs @@ -0,0 +1,273 @@ +//! Types and module structure for the **boolean-`HAVING` range** document +//! query — `SELECT GROUP BY HAVING +//! [ORDER BY ASC|DESC] LIMIT n`. +//! +//! A having-range query answers "which groups' aggregate falls inside a +//! value bound?" ("hashtags with more than 100 posts") in `O(log n + k)` +//! with a proof, by range-reading the same per-axis *secondary* Merk the +//! ranked query walks (grovedb PR #657): the secondary is keyed by +//! `(sort_key ‖ group_key)` with an order-preserving sort-key encoding, +//! so an inclusive numeric bound on the aggregate is a contiguous byte +//! range in the secondary's keyspace. The same contract opt-in applies — +//! `rankedCountable` / `rankedSummable` / `rankedAverageable` (meta schema +//! v3 / PV14) — and a `HAVING` on an axis the index does not declare is +//! rejected, because serving it would mean walking every group. +//! +//! The implementation mirrors [`super::drive_document_ranked_query`] +//! sibling-for-sibling and reuses its axis / entry / pagination types +//! ([`RankedAxis`], [`RankedEntry`], [`RankedEntryValue`], +//! [`super::RankedPaginationInputs`]) and its covering-index picker — +//! both surfaces read the same tree, so sharing the resolution logic is +//! what keeps them provably about the same subtree: +//! - [`mode_detection`] — request-shape validation + the versioned +//! `(select, group_by, having, order_by, limit)` → +//! [`DocumentHavingMode`] resolution, including the operator → +//! inclusive-bounds translation. +//! - [`execute_range`] — the two executors on +//! [`DriveDocumentHavingQuery`] (no-proof read, proof generation). +//! - [`executors`] — the `impl Drive` wrappers the dispatcher calls. +//! - [`drive_dispatcher`] — [`DocumentHavingRequest`] / +//! [`DocumentHavingResponse`] and +//! [`crate::drive::Drive::execute_document_having_request`]. +//! - [`tests`] (cfg `server` + `test`) — unit + integration tests. +//! +//! ## What makes this query shape different from ranked +//! +//! Ranked addresses groups by **rank position** (`k` best, starting at +//! rank `offset`); having-range addresses them by **value bound** +//! (`aggregate ∈ [lo, hi]`). Three consequences: +//! +//! 1. **The bound is part of the proof contract.** The grovedb envelope +//! for a range read echoes the Merk query itself, and the verifier +//! re-builds that query from the request's bounds +//! ([`AxisRangeBounds::merk_query`]) — so prover and verifier must +//! share one bounds-to-query translation, exactly as they share the +//! grove path. Completeness comes from the Merk range proof: the +//! boundary commitments show no in-range group was omitted. +//! 2. **No `OFFSET`, no `start_at`.** The range primitives take a limit +//! but no skip; pagination of an over-long result set is a future +//! capability (a cursor on the `(sort_key ‖ group_key)` composite +//! keyspace), not an emulated one. A request carrying either is +//! rejected loudly. +//! 3. **Entry order is axis order in the walk direction.** Ascending by +//! default (`ORDER BY` is optional here — the bound, not the +//! ordering, is the point of the query); an explicit `ORDER BY` on +//! the selected aggregate flips the walk. Ties break by group key in +//! the direction of the walk, same as ranked. + +#[cfg(any(feature = "server", feature = "verify"))] +use dpp::data_contract::document_type::{DocumentTypeRef, Index}; + +#[cfg(any(feature = "server", feature = "verify"))] +use super::drive_document_ranked_query::{ + path::indexed_property_name_tree_path_for_index, RankedAxis, +}; +#[cfg(any(feature = "server", feature = "verify"))] +use crate::error::Error; +#[cfg(any(feature = "server", feature = "verify"))] +use grovedb::element::indexed::{encode_avg_sort_key, encode_count_sort_key, encode_sum_sort_key}; +#[cfg(any(feature = "server", feature = "verify"))] +use grovedb::Query; + +#[cfg(any(feature = "server", feature = "verify"))] +pub mod mode_detection; + +// Server-side execution paths. +#[cfg(feature = "server")] +pub mod drive_dispatcher; +#[cfg(feature = "server")] +pub mod execute_range; +#[cfg(feature = "server")] +pub mod executors; + +#[cfg(feature = "server")] +pub use drive_dispatcher::{DocumentHavingRequest, DocumentHavingResponse}; + +#[cfg(all(feature = "server", test))] +mod tests; + +/// Hard ceiling on a having-range request's `LIMIT`. Same value and same +/// rationale as [`super::drive_document_ranked_query::MAX_RANKED_LIMIT`]: +/// the proof commits one secondary entry per returned group, so proof +/// bytes grow linearly in the limit, and the ceiling is a hard rejection +/// rather than a clamp because the limit is echoed in the proof envelope +/// and re-checked by the verifier. +#[cfg(any(feature = "server", feature = "verify"))] +pub const MAX_HAVING_LIMIT: u16 = 100; + +/// Inclusive numeric bounds on one axis of an indexed tree — the resolved +/// form of a `HAVING ` clause. +/// +/// One variant per axis because the three axes have three value types +/// (`u64` count, `i64` sum, `i128` fixed-point average) and the bound +/// arithmetic (operator translation, successor/predecessor at exclusive +/// bounds) must be exact in the axis's own domain. Both bounds are +/// **inclusive**; the operator translation in +/// [`mode_detection`] normalizes every supported operator to this form, +/// rejecting translations that would overflow (`> MAX`) or invert +/// (`lo > hi`) instead of serving a silently-empty range. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg(any(feature = "server", feature = "verify"))] +pub enum AxisRangeBounds { + /// `COUNT(*) ∈ [lo, hi]`. + Count { + /// Inclusive lower bound. + lo: u64, + /// Inclusive upper bound. + hi: u64, + }, + /// `SUM(field) ∈ [lo, hi]`. + Sum { + /// Inclusive lower bound. + lo: i64, + /// Inclusive upper bound. + hi: i64, + }, + /// `AVG(field) ∈ [lo, hi]`, in the fixed-point domain described on + /// [`super::drive_document_ranked_query::RANKED_AVG_SCALE`]. + Avg { + /// Inclusive lower bound (fixed point). + lo: i128, + /// Inclusive upper bound (fixed point). + hi: i128, + }, +} + +#[cfg(any(feature = "server", feature = "verify"))] +impl AxisRangeBounds { + /// The axis these bounds constrain. + pub fn axis(&self) -> RankedAxis { + match self { + AxisRangeBounds::Count { .. } => RankedAxis::Count, + AxisRangeBounds::Sum { .. } => RankedAxis::Sum, + AxisRangeBounds::Avg { .. } => RankedAxis::Avg, + } + } + + /// The bounds as a byte range over the axis secondary's keyspace: + /// `(inclusive_lower, exclusive_upper)`, with `None` for an upper + /// bound at the axis's type maximum (no representable successor — + /// the range is unbounded above). + /// + /// Secondary keys are `(sort_key ‖ group_key)` with order-preserving + /// fixed-width sort keys, so the inclusive numeric range `[lo, hi]` + /// is exactly the byte range `[encode(lo), encode(hi + 1))`: the + /// exclusive upper at the *next* sort key admits every group-key + /// suffix under `hi` and nothing above it. This mirrors — and must + /// stay identical to — the bound construction inside grovedb's + /// `indexed_*_range` read primitives, so the no-proof read and the + /// proved read answer the same question. + /// + /// The `+ 1` cannot overflow: the `hi == MAX` case returns `None` + /// first. + pub fn secondary_key_bounds(&self) -> (Vec, Option>) { + match *self { + AxisRangeBounds::Count { lo, hi } => ( + encode_count_sort_key(lo).to_vec(), + (hi != u64::MAX).then(|| encode_count_sort_key(hi + 1).to_vec()), + ), + AxisRangeBounds::Sum { lo, hi } => ( + encode_sum_sort_key(lo).to_vec(), + (hi != i64::MAX).then(|| encode_sum_sort_key(hi + 1).to_vec()), + ), + AxisRangeBounds::Avg { lo, hi } => ( + encode_avg_sort_key(lo).to_vec(), + (hi != i128::MAX).then(|| encode_avg_sort_key(hi + 1).to_vec()), + ), + } + } + + /// The Merk query over the axis secondary that reads exactly these + /// bounds, walking in the requested direction. + /// + /// This is the **prover/verifier-agreement artifact** of the having + /// surface: grovedb's range-proof envelope is generated against this + /// query and verified against the verifier's own reconstruction of + /// it, so both sides must build it from the same bounds through this + /// one function — a divergence surfaces as a failed verification, + /// not a wrong answer. + pub fn merk_query(&self, descending: bool) -> Query { + let (lower, upper) = self.secondary_key_bounds(); + let mut query = Query::new_with_direction(!descending); + match upper { + Some(upper) => query.insert_range(lower..upper), + None => query.insert_range_from(lower..), + } + query + } +} + +/// The resolved shape of a having-range request: the bounds (which carry +/// the axis), the walk direction, the limit, and the `(group property, +/// aggregate field)` pair the index picker needs. +/// +/// Produced by [`mode_detection::detect_having_mode`]. Parallels +/// [`super::drive_document_ranked_query::DocumentRankedMode`]. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg(any(feature = "server", feature = "verify"))] +pub struct DocumentHavingMode { + /// Inclusive bounds on the aggregate, in the axis's own domain. + pub bounds: AxisRangeBounds, + /// Walk direction: `true` reads matching groups from the largest + /// aggregate down. Defaults to `false` (ascending) when the request + /// carries no `ORDER BY`. + pub descending: bool, + /// Maximum number of matching groups to return — + /// `1 ..= MAX_HAVING_LIMIT`, required. + pub limit: u16, + /// The single `GROUP BY` property; must be the covering ranked + /// index's only property. + pub group_by_property: String, + /// The field the aggregate applies to. Empty for `COUNT(*)`; the + /// index's `summable` property for `SUM` / `AVG`. + pub aggregate_field: String, +} + +/// A resolved having-range query. Shared by the prover and the verifier — +/// both build the grove path through +/// [`DriveDocumentHavingQuery::indexed_property_name_tree_path`] and the +/// secondary query through [`AxisRangeBounds::merk_query`], so the two +/// cannot drift on which subtree or which range the proof is about. +#[derive(Debug, Clone)] +#[cfg(any(feature = "server", feature = "verify"))] +pub struct DriveDocumentHavingQuery<'a> { + /// The document type being filtered. + pub document_type: DocumentTypeRef<'a>, + /// The contract id (32 bytes). Separate from `document_type` so the + /// verifier can build the query without the full contract. + pub contract_id: [u8; 32], + /// The document type name — a path segment. + pub document_type_name: String, + /// The covering ranked index. Single-property by construction; its + /// one property is both the `GROUP BY` property and the last path + /// segment. + pub index: &'a Index, + /// Inclusive bounds on the aggregate. Carry the axis; the index must + /// declare the matching `ranked_*` flag. + pub bounds: AxisRangeBounds, + /// `true` walks the secondary from the largest matching aggregate + /// down. Tie ordering is by group key in the direction of the walk, + /// exactly as on the ranked surface. + pub descending: bool, + /// Maximum number of matching groups to return. Fewer entries come + /// back when fewer groups fall inside the bounds; that is not an + /// error. **More matching groups than `limit` are silently cut at + /// `limit`** — the walk stops, and nothing marks the cut; a caller + /// that needs the full set must widen the limit or narrow the bound. + pub limit: u16, +} + +#[cfg(any(feature = "server", feature = "verify"))] +impl DriveDocumentHavingQuery<'_> { + /// Path of the terminal property-name tree the axis secondary hangs + /// off — identical to the ranked surface's path, because both read + /// the same indexed tree. See + /// [`DriveDocumentRankedQuery::indexed_property_name_tree_path`](super::drive_document_ranked_query::DriveDocumentRankedQuery::indexed_property_name_tree_path). + pub fn indexed_property_name_tree_path(&self) -> Result>, Error> { + indexed_property_name_tree_path_for_index( + &self.contract_id, + &self.document_type_name, + self.index, + ) + } +} diff --git a/packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs b/packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs new file mode 100644 index 00000000000..7231065fcd6 --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs @@ -0,0 +1,527 @@ +//! Request-shape validation for the having-range query, and the versioned +//! `(select, group_by, having, order_by, limit)` → [`DocumentHavingMode`] +//! resolution — including the operator → inclusive-bounds translation +//! that turns a `HAVING ` clause into an +//! [`AxisRangeBounds`]. +//! +//! Pure functions on the request shape — no Drive, no contract, no +//! indexes. Available under `server` and `verify` for the same reason as +//! [`super::super::drive_document_ranked_query::mode_detection`]: both +//! sides must agree on which requests are well-formed and on the exact +//! bounds a well-formed one resolves to, because the bounds are echoed +//! (as a Merk query) inside the proof envelope. +//! +//! Versioned through +//! `platform_version.drive.methods.document.query.detect_having_mode` — +//! the accepted grammar is a consensus-adjacent contract on the query +//! surface, so relaxing it later (multi-clause `HAVING`, `IN`, a +//! pagination cursor) lands behind a method-version bump. + +use super::super::drive_document_ranked_query::mode_detection::ranked_order_key; +use super::super::drive_document_ranked_query::{RankedAxis, RankedPaginationInputs}; +use super::{AxisRangeBounds, DocumentHavingMode, MAX_HAVING_LIMIT}; +use crate::error::query::QuerySyntaxError; +use crate::error::Error; +use crate::query::having::{ + HavingAggregateFunction, HavingClause, HavingOperator, HavingRightOperand, +}; +use crate::query::projection::{SelectFunction, SelectProjection}; +use crate::query::{OrderClause, WhereClause}; +use dpp::platform_value::Value; +use dpp::version::PlatformVersion; +use grovedb::element::indexed::AVG_FIXED_POINT_SCALE; + +/// Versioned entry point. Routes through +/// `platform_version.drive.methods.document.query.detect_having_mode`; +/// today only `0` is defined and maps to [`detect_having_mode_v0`] +/// verbatim. +#[allow(clippy::too_many_arguments)] +pub fn detect_having_mode( + select: &SelectProjection, + group_by: &[String], + having: &[HavingClause], + order_by: &[OrderClause], + where_clauses: &[WhereClause], + pagination: RankedPaginationInputs, + platform_version: &PlatformVersion, +) -> Result { + match platform_version + .drive + .methods + .document + .query + .detect_having_mode + { + 0 => detect_having_mode_v0( + select, + group_by, + having, + order_by, + where_clauses, + pagination, + ), + version => Err(Error::Query(QuerySyntaxError::Unsupported(format!( + "detect_having_mode: unknown method version {version}; only 0 is supported" + )))), + } +} + +/// v0 of the having-range request grammar. +/// +/// Accepts exactly: +/// +/// ```text +/// SELECT COUNT(*) GROUP BY p HAVING COUNT(*) [ORDER BY $count [ASC|DESC]] LIMIT n +/// SELECT SUM(f) GROUP BY p HAVING SUM(f) [ORDER BY f [ASC|DESC]] LIMIT n +/// SELECT AVG(f) GROUP BY p HAVING AVG(f) [ORDER BY f [ASC|DESC]] LIMIT n +/// ``` +/// +/// with no `WHERE`, no `OFFSET`, no `START AT` / `START AFTER`, exactly +/// one `GROUP BY` property, exactly one `HAVING` clause whose aggregate +/// **is the selected aggregate** (same function, same field), an operator +/// from the contiguous-range family (`=`, `>`, `>=`, `<`, `<=`, and the +/// four `BETWEEN*` variants — `!=` and `IN` describe non-contiguous +/// ranges and are rejected as not yet supported), at most one `ORDER BY` +/// clause naming the selected aggregate, and `1 ≤ n ≤` +/// [`MAX_HAVING_LIMIT`]. +/// +/// The single-clause / same-aggregate restriction is what makes the +/// query a *range read*: one clause on the selected aggregate is one +/// contiguous slice of one axis secondary. A second clause (implicit +/// AND) or a clause on a different aggregate would need a per-candidate +/// post-check against the primary — a future capability, rejected loudly +/// today. +/// +/// Worked examples: +/// +/// ```text +/// -- hashtags with more than 100 posts, biggest first +/// SELECT COUNT(*) GROUP BY hashtag HAVING $count > 100 ORDER BY $count DESC LIMIT 100 +/// +/// -- restaurants averaging a grade of at least 4 +/// SELECT AVG(grade) GROUP BY restaurantId HAVING grade >= 4 LIMIT 50 +/// +/// -- donors whose lifetime total sits between two bounds +/// SELECT SUM(amount) GROUP BY donorId HAVING amount BETWEEN 1000 AND 5000 LIMIT 100 +/// ``` +/// +/// Everything the grammar rejects is rejected *loudly* rather than +/// normalized away — including operator translations that produce an +/// empty range (`> u64::MAX`, `BETWEEN 10 AND 5`): a bound that cannot +/// match any group is a caller error, and silently proving an empty page +/// would hide it. +pub fn detect_having_mode_v0( + select: &SelectProjection, + group_by: &[String], + having: &[HavingClause], + order_by: &[OrderClause], + where_clauses: &[WhereClause], + pagination: RankedPaginationInputs, +) -> Result { + // ---- GROUP BY: exactly one property ---------------------------- + // + // Same contract as the ranked surface: ranked indexes are + // single-property, and the sole property is what the secondary's + // group keys are. + if group_by.len() != 1 { + return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + "having-range queries require exactly one `group_by` property (the ranked \ + index's only property); got {}. Ranked indexes are single-property — compound \ + ranked indexes are rejected at contract-parse time — so there is no compound \ + grouping to filter over.", + group_by.len() + )))); + } + let group_by_property = group_by[0].clone(); + if group_by_property.is_empty() { + return Err(Error::Query(QuerySyntaxError::InvalidParameter( + "having-range queries require a non-empty `group_by` property name".to_string(), + ))); + } + + // ---- SELECT: the axis, and the field it aggregates -------------- + // + // Same axis resolution as the ranked surface, because the same + // three secondaries serve both. + let (axis, aggregate_field) = match (select.function, select.field.as_str()) { + (SelectFunction::Count, "") => (RankedAxis::Count, String::new()), + (SelectFunction::Count, field) => { + return Err(Error::Query(QuerySyntaxError::Unsupported(format!( + "having-range queries support `COUNT(*)` but not `COUNT({field})`; the \ + count axis counts documents per group, which is what `COUNT(*)` means. \ + Drop the field to filter by group size." + )))); + } + (SelectFunction::Sum, "") | (SelectFunction::Avg, "") => { + return Err(Error::Query(QuerySyntaxError::InvalidParameter( + "`SUM` / `AVG` having-range queries require a non-empty select field naming \ + the index's `summable` property" + .to_string(), + ))); + } + (SelectFunction::Sum, field) => (RankedAxis::Sum, field.to_string()), + (SelectFunction::Avg, field) => (RankedAxis::Avg, field.to_string()), + (other, _) => { + return Err(Error::Query(QuerySyntaxError::Unsupported(format!( + "having-range queries support `COUNT(*)`, `SUM(field)` and `AVG(field)` \ + selects; got {other:?}. The bound is served from an indexed tree's \ + per-axis secondary, and grovedb maintains exactly those three axes." + )))); + } + }; + + // ---- HAVING: exactly one clause, on the selected aggregate ------ + let clause = match having { + [only] => only, + [] => { + return Err(Error::Query(QuerySyntaxError::InvalidParameter( + "having-range queries require exactly one `having` clause; got none. \ + Without a bound the request is a plain grouped aggregate — drop into \ + that surface instead." + .to_string(), + ))); + } + many => { + return Err(Error::Query(QuerySyntaxError::Unsupported(format!( + "multiple `having` clauses (implicit AND) are not yet supported: got {}. \ + One clause on the selected aggregate is one contiguous slice of one axis \ + secondary; a second clause would need a per-candidate post-check against \ + the primary. Narrow to a single clause.", + many.len() + )))); + } + }; + + let clause_axis = match clause.aggregate.function { + HavingAggregateFunction::Count => RankedAxis::Count, + HavingAggregateFunction::Sum => RankedAxis::Sum, + HavingAggregateFunction::Avg => RankedAxis::Avg, + }; + if clause_axis != axis || clause.aggregate.field != select.field { + return Err(Error::Query(QuerySyntaxError::Unsupported(format!( + "the `having` clause must bound the selected aggregate itself: the select is \ + `{:?}({})` but the clause bounds `{:?}({})`. Filtering by one aggregate while \ + projecting another would need a per-candidate post-check against the primary, \ + which is not yet supported.", + select.function, + if select.field.is_empty() { + "*" + } else { + select.field.as_str() + }, + clause.aggregate.function, + if clause.aggregate.field.is_empty() { + "*" + } else { + clause.aggregate.field.as_str() + } + )))); + } + + // ---- Operator + right operand → inclusive bounds ----------------- + let HavingRightOperand::Value(right) = &clause.right; + let bounds = match axis { + RankedAxis::Count => { + let (lo, hi) = bounds_for_operator( + clause.operator, + right, + count_operand, + u64::MIN, + u64::MAX, + |v| v.checked_add(1), + |v| v.checked_sub(1), + )?; + AxisRangeBounds::Count { lo, hi } + } + RankedAxis::Sum => { + let (lo, hi) = bounds_for_operator( + clause.operator, + right, + sum_operand, + i64::MIN, + i64::MAX, + |v| v.checked_add(1), + |v| v.checked_sub(1), + )?; + AxisRangeBounds::Sum { lo, hi } + } + RankedAxis::Avg => { + let (lo, hi) = bounds_for_operator( + clause.operator, + right, + avg_operand, + i128::MIN, + i128::MAX, + |v| v.checked_add(1), + |v| v.checked_sub(1), + )?; + AxisRangeBounds::Avg { lo, hi } + } + }; + + // ---- ORDER BY: absent (ascending default) or the aggregate ------ + // + // Optional here, unlike ranked, because the bound — not the + // ordering — is what the query is about. When present it must name + // the selected aggregate: matching groups come off a single-key + // secondary, so there is no other order the walk could serve. + let expected_order_key = ranked_order_key(select); + let descending = match order_by { + [] => false, + [only] if only.field == expected_order_key => !only.ascending, + [only] => { + return Err(Error::Query(QuerySyntaxError::Unsupported(format!( + "`HAVING … ORDER BY {}` is not supported: matching groups are read off the \ + axis secondary, so the only ordering available is the bounded aggregate \ + itself — write `ORDER BY {expected_order_key}` or omit `order_by` for \ + ascending.", + only.field + )))); + } + many => { + return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + "having-range queries accept at most one `order_by` clause (naming the \ + selected aggregate); got {}. The axis secondary is a single-key ordering, \ + so there is no second sort key to apply.", + many.len() + )))); + } + }; + + // ---- WHERE: must be absent -------------------------------------- + // + // Identical rationale to the ranked surface: single-property + // indexes have no equality prefix to narrow, and the secondary is + // ordered by aggregate, not by group key. + if !where_clauses.is_empty() { + return Err(Error::Query( + QuerySyntaxError::InvalidWhereClauseComponents( + "having-range queries do not accept `where` clauses: ranked indexes are \ + single-property, so there is no equality prefix to narrow, and the axis \ + secondary is ordered by aggregate rather than by group key — it cannot \ + bound a filtered subset. Bound the whole index, or add a narrower index.", + ), + )); + } + + // ---- LIMIT: required, 1 ..= MAX_HAVING_LIMIT --------------------- + // + // Required rather than defaulted for the same reason as ranked: the + // limit is echoed inside the proof envelope and re-checked by the + // verifier, so there is no server default a client could reproduce. + // Required *especially* here, because a threshold can match + // unboundedly many groups. + let limit = pagination.limit.ok_or_else(|| { + Error::Query(QuerySyntaxError::InvalidLimit(format!( + "having-range queries require an explicit `limit` (1 ..= {MAX_HAVING_LIMIT}): \ + a bound can match any number of groups, the walk stops at `limit`, and the \ + limit is echoed in the proof envelope and re-checked by the verifier, so \ + there is no server-side default a client could reproduce." + ))) + })?; + if limit == 0 { + return Err(Error::Query(QuerySyntaxError::InvalidLimit(format!( + "`LIMIT 0` selects nothing; having-range queries require 1 ≤ limit ≤ {MAX_HAVING_LIMIT}" + )))); + } + if limit > MAX_HAVING_LIMIT as u32 { + return Err(Error::Query(QuerySyntaxError::InvalidLimit(format!( + "`LIMIT {limit}` exceeds the having-range ceiling of {MAX_HAVING_LIMIT}; the \ + proof commits one secondary entry per returned group, so its size grows \ + linearly in the limit. The ceiling is a hard limit, not a clamp, because the \ + limit is echoed in the proof envelope and re-checked by the verifier. Narrow \ + the bound to shrink the result set." + )))); + } + // Bounded by MAX_HAVING_LIMIT (a u16) immediately above. + let limit = limit as u16; + + // ---- OFFSET / START AT: must be absent --------------------------- + // + // The range primitives take a limit but no skip, and unlike the + // ranked walk there is no counted-commitment shortcut for "skip m + // matching groups" — pagination of an over-long match set is a + // future cursor capability on the `(sort_key ‖ group_key)` + // keyspace, not an emulated offset. Rejected loudly, `OFFSET 0` + // included: a caller writing any offset asked for pagination + // semantics this surface does not have. + if pagination.offset.is_some() { + return Err(Error::Query(QuerySyntaxError::InvalidLimit( + "having-range queries do not accept `offset`: matching groups are read from \ + the bound's start and cut at `limit`. To reach deeper matches, tighten the \ + bound (e.g. move the threshold past the last aggregate value already seen)." + .to_string(), + ))); + } + if pagination.has_start_at { + return Err(Error::Query(QuerySyntaxError::InvalidLimit( + "having-range queries do not accept `start_at` / `start_after`: the cursor \ + names a document id, which does not appear in a keyspace sorted by \ + aggregate." + .to_string(), + ))); + } + + Ok(DocumentHavingMode { + bounds, + descending, + limit, + group_by_property, + aggregate_field, + }) +} + +/// Translate `(operator, right operand)` into inclusive `[lo, hi]` +/// bounds in one axis's value domain. +/// +/// `operand` extracts a single scalar from a [`Value`] in that domain; +/// `succ` / `pred` are the domain's checked successor / predecessor, +/// used to normalize the exclusive operators (`>`, `<`, the `BETWEEN` +/// exclusions) onto inclusive bounds. A `succ`/`pred` that overflows +/// means the operator excludes the entire domain past its own extreme +/// (`> MAX`, `< MIN`) — rejected, like every other empty translation, +/// rather than served as a proof of nothing. +fn bounds_for_operator( + operator: HavingOperator, + right: &Value, + operand: impl Fn(&Value) -> Result, + min: T, + max: T, + succ: impl Fn(T) -> Option, + pred: impl Fn(T) -> Option, +) -> Result<(T, T), Error> { + let scalar = || operand(right); + let pair = || -> Result<(T, T), Error> { + let Some(items) = right.as_array() else { + return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + "`{operator:?}` requires a 2-element list operand `[lower, upper]`; got a \ + non-list value" + )))); + }; + let [lower, upper] = items.as_slice() else { + return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + "`{operator:?}` requires a 2-element list operand `[lower, upper]`; got {} \ + element(s)", + items.len() + )))); + }; + Ok((operand(lower)?, operand(upper)?)) + }; + let strictly_above = |v: T| { + succ(v).ok_or_else(|| { + Error::Query(QuerySyntaxError::InvalidParameter(format!( + "`{operator:?} {v}` matches no possible aggregate value: {v} is the \ + largest value the aggregate can take" + ))) + }) + }; + let strictly_below = |v: T| { + pred(v).ok_or_else(|| { + Error::Query(QuerySyntaxError::InvalidParameter(format!( + "`{operator:?} {v}` matches no possible aggregate value: {v} is the \ + smallest value the aggregate can take" + ))) + }) + }; + + let (lo, hi) = match operator { + HavingOperator::Equal => { + let v = scalar()?; + (v, v) + } + HavingOperator::GreaterThan => (strictly_above(scalar()?)?, max), + HavingOperator::GreaterThanOrEquals => (scalar()?, max), + HavingOperator::LessThan => (min, strictly_below(scalar()?)?), + HavingOperator::LessThanOrEquals => (min, scalar()?), + HavingOperator::Between => pair()?, + HavingOperator::BetweenExcludeBounds => { + let (lower, upper) = pair()?; + (strictly_above(lower)?, strictly_below(upper)?) + } + HavingOperator::BetweenExcludeLeft => { + let (lower, upper) = pair()?; + (strictly_above(lower)?, upper) + } + HavingOperator::BetweenExcludeRight => { + let (lower, upper) = pair()?; + (lower, strictly_below(upper)?) + } + HavingOperator::NotEqual | HavingOperator::In => { + return Err(Error::Query(QuerySyntaxError::Unsupported(format!( + "`{operator:?}` is not yet supported in having-range queries: it describes \ + a non-contiguous set of aggregate values, and the axis secondary serves \ + one contiguous range per request. Use a range operator, or issue one \ + request per contiguous range." + )))); + } + }; + + if lo > hi { + return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + "the `having` bound resolves to the empty range [{lo}, {hi}] (lower above \ + upper), which matches no group; fix the operand" + )))); + } + Ok((lo, hi)) +} + +/// Extract a count operand: a non-negative integer. +fn count_operand(value: &Value) -> Result { + value.to_integer::().map_err(|_| { + Error::Query(QuerySyntaxError::InvalidParameter(format!( + "a `COUNT(*)` having bound must be a non-negative integer; got {value}" + ))) + }) +} + +/// Extract a sum operand: a signed integer in `i64` range. +fn sum_operand(value: &Value) -> Result { + value.to_integer::().map_err(|_| { + Error::Query(QuerySyntaxError::InvalidParameter(format!( + "a `SUM(field)` having bound must be an integer within i64 range; got {value}" + ))) + }) +} + +/// Extract an average operand and scale it into the axis's fixed-point +/// domain (see +/// [`super::super::drive_document_ranked_query::RANKED_AVG_SCALE`]). +/// +/// Integer operands scale exactly (`v × SCALE` — the product of any i64 +/// with the scale fits in `i128` by the compile-time bound next to the +/// scale constant). Float operands are scaled through `f64` +/// multiplication and truncated toward zero; that conversion is +/// deterministic (IEEE 754) but inexact above 2^53, which is fine for a +/// *threshold* — callers needing exact fixed-point bounds pass integers +/// or pre-scaled values. +fn avg_operand(value: &Value) -> Result { + if let Some(int) = value.as_integer::() { + return (int as i128) + .checked_mul(AVG_FIXED_POINT_SCALE) + .ok_or_else(|| { + Error::Query(QuerySyntaxError::InvalidParameter(format!( + "the `AVG(field)` having bound {int} does not fit the fixed-point domain" + ))) + }); + } + let float = value.to_float().map_err(|_| { + Error::Query(QuerySyntaxError::InvalidParameter(format!( + "an `AVG(field)` having bound must be an integer or a float; got {value}" + ))) + })?; + if !float.is_finite() { + return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + "an `AVG(field)` having bound must be finite; got {float}" + )))); + } + let scaled = float * AVG_FIXED_POINT_SCALE as f64; + // `f64 as i128` saturates rather than wrapping; the explicit range + // check keeps out-of-domain thresholds a loud caller error instead + // of a silent clamp to the domain edge. + if scaled <= i128::MIN as f64 || scaled >= i128::MAX as f64 { + return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + "the `AVG(field)` having bound {float} does not fit the fixed-point domain" + )))); + } + Ok(scaled as i128) +} diff --git a/packages/rs-drive/src/query/drive_document_having_query/tests.rs b/packages/rs-drive/src/query/drive_document_having_query/tests.rs new file mode 100644 index 00000000000..8b56398769b --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_having_query/tests.rs @@ -0,0 +1,1112 @@ +//! Unit + integration tests for the having-range query surface. +//! +//! Mirrors the structure of +//! [`super::super::drive_document_ranked_query::tests`]: grammar and +//! bounds tests are pure (no Drive), execution tests run against a real +//! Drive with the shared `restaurants` fixture (see that module's docs +//! for the doctype → axis table) and documents inserted through the +//! real write path, with every proof round-tripped through +//! [`DriveDocumentHavingQuery::verify_having_range_proof`] and checked +//! against the live grovedb root hash. + +use super::mode_detection::detect_having_mode_v0; +use super::{AxisRangeBounds, MAX_HAVING_LIMIT}; +use crate::query::drive_document_ranked_query::RankedPaginationInputs; +use crate::query::having::{ + HavingAggregate, HavingAggregateFunction, HavingClause, HavingOperator, HavingRightOperand, +}; +use crate::query::projection::SelectProjection; +use crate::query::OrderClause; +use dpp::platform_value::Value; + +fn clause( + function: HavingAggregateFunction, + field: &str, + operator: HavingOperator, + right: Value, +) -> HavingClause { + HavingClause { + aggregate: HavingAggregate { + function, + field: field.to_string(), + }, + operator, + right: HavingRightOperand::Value(right), + } +} + +fn pagination(limit: u32) -> RankedPaginationInputs { + RankedPaginationInputs { + limit: Some(limit), + offset: None, + has_start_at: false, + } +} + +mod grammar { + use super::*; + + #[test] + fn count_greater_than_resolves_to_exclusive_lower_bound() { + let mode = detect_having_mode_v0( + &SelectProjection::count_star(), + &["hashtag".to_string()], + &[clause( + HavingAggregateFunction::Count, + "", + HavingOperator::GreaterThan, + Value::U64(100), + )], + &[], + &[], + pagination(10), + ) + .expect("should resolve"); + assert_eq!( + mode.bounds, + AxisRangeBounds::Count { + lo: 101, + hi: u64::MAX + } + ); + assert!(!mode.descending); + assert_eq!(mode.limit, 10); + assert_eq!(mode.group_by_property, "hashtag"); + assert_eq!(mode.aggregate_field, ""); + } + + #[test] + fn sum_between_is_inclusive_on_both_ends() { + let mode = detect_having_mode_v0( + &SelectProjection::sum("amount"), + &["donorId".to_string()], + &[clause( + HavingAggregateFunction::Sum, + "amount", + HavingOperator::Between, + Value::Array(vec![Value::I64(1000), Value::I64(5000)]), + )], + &[], + &[], + pagination(100), + ) + .expect("should resolve"); + assert_eq!(mode.bounds, AxisRangeBounds::Sum { lo: 1000, hi: 5000 }); + } + + #[test] + fn between_exclude_bounds_moves_both_ends_inward() { + let mode = detect_having_mode_v0( + &SelectProjection::count_star(), + &["hashtag".to_string()], + &[clause( + HavingAggregateFunction::Count, + "", + HavingOperator::BetweenExcludeBounds, + Value::Array(vec![Value::U64(5), Value::U64(10)]), + )], + &[], + &[], + pagination(10), + ) + .expect("should resolve"); + assert_eq!(mode.bounds, AxisRangeBounds::Count { lo: 6, hi: 9 }); + } + + #[test] + fn avg_integer_threshold_scales_exactly_into_fixed_point() { + use crate::query::drive_document_ranked_query::RANKED_AVG_SCALE; + let mode = detect_having_mode_v0( + &SelectProjection::avg("grade"), + &["restaurantId".to_string()], + &[clause( + HavingAggregateFunction::Avg, + "grade", + HavingOperator::GreaterThanOrEquals, + Value::U64(4), + )], + &[], + &[], + pagination(50), + ) + .expect("should resolve"); + assert_eq!( + mode.bounds, + AxisRangeBounds::Avg { + lo: 4 * RANKED_AVG_SCALE, + hi: i128::MAX + } + ); + } + + #[test] + fn order_by_the_selected_aggregate_sets_direction() { + let mode = detect_having_mode_v0( + &SelectProjection::count_star(), + &["hashtag".to_string()], + &[clause( + HavingAggregateFunction::Count, + "", + HavingOperator::GreaterThan, + Value::U64(100), + )], + &[OrderClause { + field: "$count".to_string(), + ascending: false, + }], + &[], + pagination(10), + ) + .expect("should resolve"); + assert!(mode.descending); + } + + #[test] + fn ordering_by_anything_else_is_rejected() { + let result = detect_having_mode_v0( + &SelectProjection::count_star(), + &["hashtag".to_string()], + &[clause( + HavingAggregateFunction::Count, + "", + HavingOperator::GreaterThan, + Value::U64(100), + )], + &[OrderClause { + field: "hashtag".to_string(), + ascending: true, + }], + &[], + pagination(10), + ); + assert!(result.is_err(), "ordering by a schema property must fail"); + } + + #[test] + fn clause_on_a_different_aggregate_than_the_select_is_rejected() { + let result = detect_having_mode_v0( + &SelectProjection::count_star(), + &["hashtag".to_string()], + &[clause( + HavingAggregateFunction::Sum, + "amount", + HavingOperator::GreaterThan, + Value::I64(100), + )], + &[], + &[], + pagination(10), + ); + assert!(result.is_err(), "cross-aggregate having must fail"); + } + + #[test] + fn multiple_clauses_are_rejected() { + let single = clause( + HavingAggregateFunction::Count, + "", + HavingOperator::GreaterThan, + Value::U64(100), + ); + let result = detect_having_mode_v0( + &SelectProjection::count_star(), + &["hashtag".to_string()], + &[single.clone(), single], + &[], + &[], + pagination(10), + ); + assert!(result.is_err(), "multi-clause having must fail"); + } + + #[test] + fn not_equal_and_in_are_rejected_as_non_contiguous() { + for operator in [HavingOperator::NotEqual, HavingOperator::In] { + let result = detect_having_mode_v0( + &SelectProjection::count_star(), + &["hashtag".to_string()], + &[clause( + HavingAggregateFunction::Count, + "", + operator, + Value::U64(100), + )], + &[], + &[], + pagination(10), + ); + assert!(result.is_err(), "{operator:?} must fail"); + } + } + + #[test] + fn greater_than_the_type_maximum_is_rejected_not_served_empty() { + let result = detect_having_mode_v0( + &SelectProjection::count_star(), + &["hashtag".to_string()], + &[clause( + HavingAggregateFunction::Count, + "", + HavingOperator::GreaterThan, + Value::U64(u64::MAX), + )], + &[], + &[], + pagination(10), + ); + assert!(result.is_err(), "> u64::MAX must fail loudly"); + } + + #[test] + fn inverted_between_is_rejected() { + let result = detect_having_mode_v0( + &SelectProjection::count_star(), + &["hashtag".to_string()], + &[clause( + HavingAggregateFunction::Count, + "", + HavingOperator::Between, + Value::Array(vec![Value::U64(10), Value::U64(5)]), + )], + &[], + &[], + pagination(10), + ); + assert!(result.is_err(), "inverted bounds must fail loudly"); + } + + #[test] + fn negative_count_bound_is_rejected() { + let result = detect_having_mode_v0( + &SelectProjection::count_star(), + &["hashtag".to_string()], + &[clause( + HavingAggregateFunction::Count, + "", + HavingOperator::GreaterThan, + Value::I64(-1), + )], + &[], + &[], + pagination(10), + ); + assert!(result.is_err(), "a negative COUNT bound must fail"); + } + + #[test] + fn limit_is_required_and_capped() { + let having = [clause( + HavingAggregateFunction::Count, + "", + HavingOperator::GreaterThan, + Value::U64(100), + )]; + let select = SelectProjection::count_star(); + let group_by = ["hashtag".to_string()]; + + let missing = detect_having_mode_v0( + &select, + &group_by, + &having, + &[], + &[], + RankedPaginationInputs::default(), + ); + assert!(missing.is_err(), "a missing limit must fail"); + + let over = detect_having_mode_v0( + &select, + &group_by, + &having, + &[], + &[], + pagination(MAX_HAVING_LIMIT as u32 + 1), + ); + assert!(over.is_err(), "an over-ceiling limit must fail"); + } + + #[test] + fn offset_and_start_at_are_rejected() { + let having = [clause( + HavingAggregateFunction::Count, + "", + HavingOperator::GreaterThan, + Value::U64(100), + )]; + let select = SelectProjection::count_star(); + let group_by = ["hashtag".to_string()]; + + let with_offset = detect_having_mode_v0( + &select, + &group_by, + &having, + &[], + &[], + RankedPaginationInputs { + limit: Some(10), + offset: Some(0), + has_start_at: false, + }, + ); + assert!(with_offset.is_err(), "any offset (even 0) must fail"); + + let with_start = detect_having_mode_v0( + &select, + &group_by, + &having, + &[], + &[], + RankedPaginationInputs { + limit: Some(10), + offset: None, + has_start_at: true, + }, + ); + assert!(with_start.is_err(), "start_at must fail"); + } +} + +mod bounds { + use super::*; + use grovedb::element::indexed::{ + encode_avg_sort_key, encode_count_sort_key, encode_sum_sort_key, + }; + + #[test] + fn count_byte_bounds_bracket_the_inclusive_range() { + let bounds = AxisRangeBounds::Count { lo: 101, hi: 200 }; + let (lower, upper) = bounds.secondary_key_bounds(); + assert_eq!(lower, encode_count_sort_key(101).to_vec()); + assert_eq!(upper, Some(encode_count_sort_key(201).to_vec())); + } + + #[test] + fn unbounded_above_uses_range_from() { + let bounds = AxisRangeBounds::Count { + lo: 101, + hi: u64::MAX, + }; + let (_, upper) = bounds.secondary_key_bounds(); + assert_eq!(upper, None, "hi == MAX has no representable successor"); + + let sum_bounds = AxisRangeBounds::Sum { + lo: 0, + hi: i64::MAX, + }; + assert_eq!(sum_bounds.secondary_key_bounds().1, None); + + let avg_bounds = AxisRangeBounds::Avg { + lo: 0, + hi: i128::MAX, + }; + assert_eq!(avg_bounds.secondary_key_bounds().1, None); + } + + #[test] + fn sum_and_avg_bounds_use_the_sign_flipped_encodings() { + let sum_bounds = AxisRangeBounds::Sum { lo: -5, hi: 5 }; + let (lower, upper) = sum_bounds.secondary_key_bounds(); + assert_eq!(lower, encode_sum_sort_key(-5).to_vec()); + assert_eq!(upper, Some(encode_sum_sort_key(6).to_vec())); + + let avg_bounds = AxisRangeBounds::Avg { lo: -5, hi: 5 }; + let (lower, upper) = avg_bounds.secondary_key_bounds(); + assert_eq!(lower, encode_avg_sort_key(-5).to_vec()); + assert_eq!(upper, Some(encode_avg_sort_key(6).to_vec())); + } + + #[test] + fn merk_query_direction_follows_descending() { + let bounds = AxisRangeBounds::Count { lo: 101, hi: 200 }; + assert!(bounds.merk_query(false).left_to_right); + assert!(!bounds.merk_query(true).left_to_right); + } +} + +mod execution { + //! End-to-end behaviour against the `restaurants` fixture: the + //! dispatcher run through its public entry point (the same call + //! drive-abci makes), no-proof and proved, on all three axes. + + use super::super::drive_dispatcher::{DocumentHavingRequest, DocumentHavingResponse}; + use super::super::mode_detection::detect_having_mode; + use super::super::{AxisRangeBounds, DriveDocumentHavingQuery}; + use super::clause; + use crate::drive::Drive; + use crate::error::Error; + use crate::query::drive_document_ranked_query::index_picker::find_ranked_index_for_axis; + use crate::query::drive_document_ranked_query::{ + RankedEntry, RankedEntryValue, RankedPaginationInputs, RANKED_COUNT_ORDER_KEY, + }; + use crate::query::having::{HavingAggregateFunction, HavingClause, HavingOperator}; + use crate::query::projection::SelectProjection; + use crate::query::OrderClause; + use crate::util::object_size_info::DocumentInfo::DocumentRefInfo; + use crate::util::object_size_info::{DocumentAndContractInfo, OwnedDocumentInfo}; + use crate::util::storage_flags::StorageFlags; + use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; + use dpp::block::block_info::BlockInfo; + use dpp::data_contract::accessors::v0::DataContractV0Getters; + use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; + use dpp::data_contract::document_type::random_document::CreateRandomDocument; + use dpp::document::{Document, DocumentV0Setters}; + use dpp::platform_value::Value; + use dpp::prelude::DataContract; + use dpp::tests::json_document::json_document_to_contract; + use dpp::version::PlatformVersion; + use grovedb::element::indexed::compute_avg_fixed_point; + use std::collections::BTreeMap; + + const GROUP_PROPERTY: &str = "restaurantId"; + + fn platform_version() -> &'static PlatformVersion { + PlatformVersion::latest() + } + + fn setup_restaurants() -> (Drive, DataContract) { + let drive = setup_drive_with_initial_state_structure(None); + let pv = platform_version(); + let contract = json_document_to_contract( + "tests/supporting_files/contract/restaurants/restaurants-contract.json", + false, + pv, + ) + .expect("expected to parse the restaurants contract"); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + pv, + ) + .expect("expected to apply the restaurants contract"); + (drive, contract) + } + + /// Same real-write-path insertion as the ranked suite; see its docs + /// for the disjoint-seed requirement. + fn insert_docs( + drive: &Drive, + contract: &DataContract, + document_type_name: &str, + aggregated_property: &str, + first_seed: u64, + rows: &[(&str, i64)], + ) { + let pv = platform_version(); + let document_type = contract + .document_type_for_name(document_type_name) + .unwrap_or_else(|_| panic!("{document_type_name} doctype exists")); + for (i, (restaurant, value)) in rows.iter().enumerate() { + let mut doc: Document = document_type + .random_document(Some(first_seed + i as u64), pv) + .expect("random document"); + let mut props = BTreeMap::new(); + props.insert( + GROUP_PROPERTY.to_string(), + Value::Text(restaurant.to_string()), + ); + props.insert(aggregated_property.to_string(), Value::I64(*value)); + doc.set_properties(props); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo((&doc, None)), + owner_id: None, + }, + contract, + document_type, + }, + false, + BlockInfo::default(), + true, + None, + pv, + None, + ) + .unwrap_or_else(|e| { + panic!("expected to insert a {document_type_name} document: {e}") + }); + } + } + + /// One having request, minus the `prove` flag. + #[derive(Clone)] + struct HavingCase { + document_type_name: &'static str, + select: SelectProjection, + having: HavingClause, + /// `None` means no `ORDER BY` (ascending default); + /// `Some(ascending)` orders by the selected aggregate. + order_ascending: Option, + limit: Option, + } + + impl HavingCase { + fn count(operator: HavingOperator, right: Value, limit: u32) -> Self { + Self { + document_type_name: "visit", + select: SelectProjection::count_star(), + having: clause(HavingAggregateFunction::Count, "", operator, right), + order_ascending: None, + limit: Some(limit), + } + } + + fn sum(operator: HavingOperator, right: Value, limit: u32) -> Self { + Self { + document_type_name: "tip", + select: SelectProjection::sum("amount"), + having: clause(HavingAggregateFunction::Sum, "amount", operator, right), + order_ascending: None, + limit: Some(limit), + } + } + + fn avg(operator: HavingOperator, right: Value, limit: u32) -> Self { + Self { + document_type_name: "review", + select: SelectProjection::avg("grade"), + having: clause(HavingAggregateFunction::Avg, "grade", operator, right), + order_ascending: None, + limit: Some(limit), + } + } + + fn ordered(mut self, ascending: bool) -> Self { + self.order_ascending = Some(ascending); + self + } + + fn order_by(&self) -> Vec { + match self.order_ascending { + None => Vec::new(), + Some(ascending) => { + let field = match self.select.field.as_str() { + "" => RANKED_COUNT_ORDER_KEY.to_string(), + field => field.to_string(), + }; + vec![OrderClause { field, ascending }] + } + } + } + } + + /// Run a case through the public dispatcher entry point — the same + /// call drive-abci's routing layer makes. + fn run( + drive: &Drive, + contract: &DataContract, + case: &HavingCase, + prove: bool, + ) -> Result { + let group_by = vec![GROUP_PROPERTY.to_string()]; + let having = vec![case.having.clone()]; + let order_by = case.order_by(); + let document_type = contract + .document_type_for_name(case.document_type_name) + .expect("doctype exists"); + drive.execute_document_having_request( + DocumentHavingRequest { + contract, + document_type, + group_by: &group_by, + select: case.select.clone(), + having: &having, + order_by: &order_by, + where_clauses: &[], + limit: case.limit, + offset: None, + has_start_at: false, + prove, + }, + None, + platform_version(), + ) + } + + fn entries_of(response: DocumentHavingResponse) -> Vec { + match response { + DocumentHavingResponse::Entries(entries) => entries, + DocumentHavingResponse::Proof(_) => panic!("expected entries, got a proof"), + } + } + + fn proof_of(response: DocumentHavingResponse) -> Vec { + match response { + DocumentHavingResponse::Proof(proof) => proof, + DocumentHavingResponse::Entries(_) => panic!("expected a proof, got entries"), + } + } + + fn keys_of(entries: &[RankedEntry]) -> Vec { + entries + .iter() + .map(|entry| { + String::from_utf8(entry.key.clone()).expect("fixture group keys are utf-8") + }) + .collect() + } + + /// Rebuild the query the way a client would: re-run the same + /// versioned validation (which resolves the bounds), then resolve + /// the index off the contract — the shape the SDK's proof helper + /// takes. + fn client_side_query<'a>( + contract: &'a DataContract, + case: &HavingCase, + ) -> DriveDocumentHavingQuery<'a> { + let group_by = vec![GROUP_PROPERTY.to_string()]; + let having = vec![case.having.clone()]; + let order_by = case.order_by(); + let mode = detect_having_mode( + &case.select, + &group_by, + &having, + &order_by, + &[], + RankedPaginationInputs { + limit: case.limit, + offset: None, + has_start_at: false, + }, + platform_version(), + ) + .expect("the case is well-formed"); + let indexes = contract + .document_types() + .get(case.document_type_name) + .expect("doctype exists") + .indexes(); + let index = find_ranked_index_for_axis( + indexes, + &mode.group_by_property, + mode.bounds.axis(), + &mode.aggregate_field, + ) + .expect("the fixture declares the axis"); + DriveDocumentHavingQuery { + document_type: contract + .document_type_for_name(case.document_type_name) + .expect("doctype exists"), + contract_id: contract.id_ref().to_buffer(), + document_type_name: case.document_type_name.to_string(), + index, + bounds: mode.bounds, + descending: mode.descending, + limit: mode.limit, + } + } + + fn grovedb_root_hash(drive: &Drive) -> [u8; 32] { + drive + .grove + .root_hash(None, &platform_version().drive.grove_version) + .unwrap() + .expect("root hash must be readable") + } + + /// Prove the case, verify the proof, and assert the verified + /// entries and root hash match the live database. + fn assert_proof_round_trips( + drive: &Drive, + contract: &DataContract, + case: &HavingCase, + expected: &[RankedEntry], + ) { + let proof = proof_of(run(drive, contract, case, true).expect("prove must succeed")); + let query = client_side_query(contract, case); + let (root_hash, verified) = query + .verify_having_range_proof(&proof, platform_version()) + .expect("the proof must verify"); + assert_eq!( + verified, expected, + "verified entries must equal what the unproven read returned" + ); + assert_eq!( + root_hash, + grovedb_root_hash(drive), + "the proof must reconstruct the live grovedb root hash" + ); + } + + /// Visits per restaurant: alpha 1, beta 3, gamma 2, delta 4. + /// `HAVING COUNT(*) > 2` must return exactly beta and delta, in + /// ascending count order (no ORDER BY), and the proof must commit + /// the same page. + #[test] + fn count_threshold_reads_and_proves_consistently() { + let (drive, contract) = setup_restaurants(); + insert_docs( + &drive, + &contract, + "visit", + "guests", + 100, + &[ + ("alpha", 1), + ("beta", 1), + ("beta", 2), + ("beta", 3), + ("gamma", 1), + ("gamma", 2), + ("delta", 1), + ("delta", 2), + ("delta", 3), + ("delta", 4), + ], + ); + + let case = HavingCase::count(HavingOperator::GreaterThan, Value::U64(2), 10); + let entries = entries_of(run(&drive, &contract, &case, false).expect("read succeeds")); + assert_eq!( + keys_of(&entries), + vec!["beta", "delta"], + "ascending count order: beta (3) before delta (4)" + ); + assert_eq!( + entries.iter().map(|e| e.value).collect::>(), + vec![RankedEntryValue::Count(3), RankedEntryValue::Count(4)] + ); + + assert_proof_round_trips(&drive, &contract, &case, &entries); + } + + /// Same state, descending: `HAVING $count >= 2 ORDER BY $count + /// DESC` walks from the largest matching count down. + #[test] + fn descending_walk_returns_biggest_matches_first() { + let (drive, contract) = setup_restaurants(); + insert_docs( + &drive, + &contract, + "visit", + "guests", + 200, + &[ + ("alpha", 1), + ("beta", 1), + ("beta", 2), + ("gamma", 1), + ("gamma", 2), + ("gamma", 3), + ], + ); + + let case = HavingCase::count(HavingOperator::GreaterThanOrEquals, Value::U64(2), 10) + .ordered(false); + let entries = entries_of(run(&drive, &contract, &case, false).expect("read succeeds")); + assert_eq!(keys_of(&entries), vec!["gamma", "beta"]); + + assert_proof_round_trips(&drive, &contract, &case, &entries); + } + + /// Tips per restaurant: alpha 150, beta 900, gamma 400. + /// `HAVING SUM(amount) BETWEEN 100 AND 500` returns alpha and gamma + /// — bounds inclusive on both ends. + #[test] + fn sum_between_reads_and_proves_consistently() { + let (drive, contract) = setup_restaurants(); + insert_docs( + &drive, + &contract, + "tip", + "amount", + 300, + &[("alpha", 100), ("alpha", 50), ("beta", 900), ("gamma", 400)], + ); + + let case = HavingCase::sum( + HavingOperator::Between, + Value::Array(vec![Value::I64(100), Value::I64(500)]), + 10, + ); + let entries = entries_of(run(&drive, &contract, &case, false).expect("read succeeds")); + assert_eq!(keys_of(&entries), vec!["alpha", "gamma"]); + assert_eq!( + entries.iter().map(|e| e.value).collect::>(), + vec![RankedEntryValue::Sum(150), RankedEntryValue::Sum(400)] + ); + + assert_proof_round_trips(&drive, &contract, &case, &entries); + } + + /// Reviews: alpha (90+80)/2 = 85, beta (60+70+50)/3 = 60, gamma 95. + /// `HAVING AVG(grade) >= 85` returns alpha and gamma; the entries + /// carry the exact fixed points. + #[test] + fn avg_threshold_reads_and_proves_consistently() { + let (drive, contract) = setup_restaurants(); + insert_docs( + &drive, + &contract, + "review", + "grade", + 400, + &[ + ("alpha", 90), + ("alpha", 80), + ("beta", 60), + ("beta", 70), + ("beta", 50), + ("gamma", 95), + ], + ); + + let case = HavingCase::avg(HavingOperator::GreaterThanOrEquals, Value::U64(85), 10); + let entries = entries_of(run(&drive, &contract, &case, false).expect("read succeeds")); + assert_eq!(keys_of(&entries), vec!["alpha", "gamma"]); + assert_eq!( + entries.iter().map(|e| e.value).collect::>(), + vec![ + RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(170, 2)), + RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(95, 1)), + ] + ); + + assert_proof_round_trips(&drive, &contract, &case, &entries); + } + + /// The limit cuts an over-long match set — and the cut page still + /// proves. With ascending order and `LIMIT 2`, the two *smallest* + /// matching counts come back. + #[test] + fn limit_cuts_the_match_set_and_the_cut_page_proves() { + let (drive, contract) = setup_restaurants(); + insert_docs( + &drive, + &contract, + "visit", + "guests", + 500, + &[ + ("alpha", 1), + ("alpha", 2), + ("beta", 1), + ("beta", 2), + ("beta", 3), + ("gamma", 1), + ("gamma", 2), + ("gamma", 3), + ("gamma", 4), + ], + ); + + let case = HavingCase::count(HavingOperator::GreaterThanOrEquals, Value::U64(2), 2); + let entries = entries_of(run(&drive, &contract, &case, false).expect("read succeeds")); + assert_eq!( + keys_of(&entries), + vec!["alpha", "beta"], + "three groups match but the limit keeps the two smallest" + ); + + assert_proof_round_trips(&drive, &contract, &case, &entries); + } + + /// A bound matching nothing is a legitimate, provable answer — + /// both against populated state and against a freshly registered + /// contract whose secondary is empty. + #[test] + fn an_empty_match_set_reads_empty_and_proves_empty() { + let (drive, contract) = setup_restaurants(); + + // Empty secondary (no documents at all): the unproven read + // returns the empty list, but grovedb's range prover — unlike + // the ranked surface's paginated prover — has no absence-proof + // shape for a completely empty tree and refuses. drive-abci + // maps this exact failure class onto an `InvalidArgument` + // telling the caller to retry unproved + // (`empty_ranking_proof_rejection`); at the drive level it + // surfaces as the grovedb error asserted here. If a future + // grovedb pin makes empty range proofs work, this arm should + // flip to a round-trip assertion. + let case = HavingCase::count(HavingOperator::GreaterThan, Value::U64(100), 10); + let entries = entries_of(run(&drive, &contract, &case, false).expect("read succeeds")); + assert!(entries.is_empty()); + let error = run(&drive, &contract, &case, true) + .expect_err("proving against an empty secondary is refused by grovedb"); + assert!( + format!("{error}").contains("Cannot create proof for empty tree"), + "the failure must be the recognized empty-tree class, got: {error}" + ); + + // Populated secondary, bound above every count: a genuine + // absence proof, which works — the tree has content to anchor + // the boundary commitments to. + insert_docs( + &drive, + &contract, + "visit", + "guests", + 600, + &[("alpha", 1), ("beta", 1), ("beta", 2)], + ); + let entries = entries_of(run(&drive, &contract, &case, false).expect("read succeeds")); + assert!(entries.is_empty()); + assert_proof_round_trips(&drive, &contract, &case, &entries); + } + + /// A proof generated for one bound must not verify as a different + /// bound **whose answer differs**: verification re-runs the Merk + /// query against the proof, so a wider bound demands proof of a + /// group the narrower proof never committed (gamma, count 2, below) + /// and fails. + /// + /// The state is chosen so the two bounds genuinely disagree. With + /// no group between the two thresholds the same proof *does* + /// verify under both — correctly, because the range boundaries + /// prove both claims — so the distinguishing group is the point of + /// the fixture. + #[test] + fn a_proof_does_not_verify_under_different_bounds() { + let (drive, contract) = setup_restaurants(); + insert_docs( + &drive, + &contract, + "visit", + "guests", + 700, + &[ + ("alpha", 1), + ("gamma", 1), + ("gamma", 2), + ("beta", 1), + ("beta", 2), + ("beta", 3), + ], + ); + + let over_two = HavingCase::count(HavingOperator::GreaterThan, Value::U64(2), 10); + let proof = proof_of(run(&drive, &contract, &over_two, true).expect("prove succeeds")); + + // Honest verification succeeds… + assert!(client_side_query(&contract, &over_two) + .verify_having_range_proof(&proof, platform_version()) + .is_ok()); + + // …but the same bytes under a different threshold must not. + let over_one = HavingCase::count(HavingOperator::GreaterThan, Value::U64(1), 10); + let mut tampered_query = client_side_query(&contract, &over_one); + assert_eq!( + tampered_query.bounds, + AxisRangeBounds::Count { + lo: 2, + hi: u64::MAX + } + ); + assert!( + tampered_query + .verify_having_range_proof(&proof, platform_version()) + .is_err(), + "a proof of `> 2` must not verify as `> 1`" + ); + + // Nor under a different direction or limit. + tampered_query = client_side_query(&contract, &over_two); + tampered_query.descending = true; + assert!(tampered_query + .verify_having_range_proof(&proof, platform_version()) + .is_err()); + + tampered_query = client_side_query(&contract, &over_two); + tampered_query.limit = 5; + assert!(tampered_query + .verify_having_range_proof(&proof, platform_version()) + .is_err()); + } + + /// A `having` on an axis no index declares is refused with the + /// contract keyword the author needs to add. The `review` doctype's + /// index is `rankedAverageable` only — a COUNT bound has no + /// covering secondary. + #[test] + fn a_bound_on_an_undeclared_axis_names_the_missing_keyword() { + let (drive, contract) = setup_restaurants(); + let case = HavingCase { + document_type_name: "review", + select: SelectProjection::count_star(), + having: clause( + HavingAggregateFunction::Count, + "", + HavingOperator::GreaterThan, + Value::U64(2), + ), + order_ascending: None, + limit: Some(10), + }; + let error = run(&drive, &contract, &case, false).expect_err("no covering axis"); + assert!( + format!("{error}").contains("rankedCountable"), + "the rejection must name the missing keyword, got: {error}" + ); + } + + /// Equal bounds are a point lookup on the axis: `HAVING SUM(amount) + /// = 400` returns exactly the group whose running sum is 400. + #[test] + fn equality_is_a_point_bound() { + let (drive, contract) = setup_restaurants(); + insert_docs( + &drive, + &contract, + "tip", + "amount", + 800, + &[("alpha", 150), ("beta", 400), ("gamma", 400)], + ); + + let case = HavingCase::sum(HavingOperator::Equal, Value::I64(400), 10); + let entries = entries_of(run(&drive, &contract, &case, false).expect("read succeeds")); + assert_eq!( + keys_of(&entries), + vec!["beta", "gamma"], + "equal sums tie-break by group key in walk direction" + ); + + assert_proof_round_trips(&drive, &contract, &case, &entries); + } + + /// Pagination-by-bound: after a page cut at the limit, the caller + /// tightens the bound past the last seen value and continues — + /// the documented substitute for `OFFSET` on this surface. + #[test] + fn tightening_the_bound_continues_past_a_cut_page() { + let (drive, contract) = setup_restaurants(); + insert_docs( + &drive, + &contract, + "visit", + "guests", + 900, + &[ + ("alpha", 1), + ("alpha", 2), + ("beta", 1), + ("beta", 2), + ("beta", 3), + ("gamma", 1), + ("gamma", 2), + ("gamma", 3), + ("gamma", 4), + ], + ); + + // Page 1: counts >= 2, limit 1 → alpha (count 2). + let page_one = HavingCase::count(HavingOperator::GreaterThanOrEquals, Value::U64(2), 1); + let first = entries_of(run(&drive, &contract, &page_one, false).expect("read succeeds")); + assert_eq!(keys_of(&first), vec!["alpha"]); + let RankedEntryValue::Count(last_seen) = first[0].value else { + panic!("count axis returns count values"); + }; + + // Page 2: counts > last seen → beta, gamma. + let page_two = HavingCase::count(HavingOperator::GreaterThan, Value::U64(last_seen), 10); + let rest = entries_of(run(&drive, &contract, &page_two, false).expect("read succeeds")); + assert_eq!(keys_of(&rest), vec!["beta", "gamma"]); + + assert_proof_round_trips(&drive, &contract, &page_two, &rest); + } +} diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/path.rs b/packages/rs-drive/src/query/drive_document_ranked_query/path.rs index be504f18ea2..2475f080a8c 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/path.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/path.rs @@ -17,6 +17,35 @@ use super::DriveDocumentRankedQuery; use crate::drive::RootTree; use crate::error::drive::DriveError; use crate::error::Error; +use dpp::data_contract::document_type::Index; + +/// Path of a single-property index's terminal property-name tree — +/// shared by the ranked and having-range query surfaces, which read +/// the same indexed tree. See +/// [`DriveDocumentRankedQuery::indexed_property_name_tree_path`] for +/// the segment layout and the single-property requirement. +pub(crate) fn indexed_property_name_tree_path_for_index( + contract_id: &[u8; 32], + document_type_name: &str, + index: &Index, +) -> Result>, Error> { + let [property] = index.properties.as_slice() else { + return Err(Error::Drive(DriveError::NotSupported( + "ranked queries require a single-property index: the ranked secondary \ + lives on the index's terminal property-name tree, and for a compound \ + index that tree sits under a prefix value tree whose value only a \ + `where` clause could name — but ranked queries accept no `where` \ + clauses", + ))); + }; + Ok(vec![ + vec![RootTree::DataContractDocuments as u8], + contract_id.to_vec(), + vec![1u8], + document_type_name.as_bytes().to_vec(), + property.name.as_bytes().to_vec(), + ]) +} impl DriveDocumentRankedQuery<'_> { /// Path of the **terminal property-name tree** — the indexed tree @@ -44,21 +73,10 @@ impl DriveDocumentRankedQuery<'_> { /// typed error than a path pointing at a prefix level whose element /// is not an indexed tree at all. pub fn indexed_property_name_tree_path(&self) -> Result>, Error> { - let [property] = self.index.properties.as_slice() else { - return Err(Error::Drive(DriveError::NotSupported( - "ranked queries require a single-property index: the ranked secondary \ - lives on the index's terminal property-name tree, and for a compound \ - index that tree sits under a prefix value tree whose value only a \ - `where` clause could name — but ranked queries accept no `where` \ - clauses", - ))); - }; - Ok(vec![ - vec![RootTree::DataContractDocuments as u8], - self.contract_id.to_vec(), - vec![1u8], - self.document_type_name.as_bytes().to_vec(), - property.name.as_bytes().to_vec(), - ]) + indexed_property_name_tree_path_for_index( + &self.contract_id, + &self.document_type_name, + self.index, + ) } } diff --git a/packages/rs-drive/src/query/mod.rs b/packages/rs-drive/src/query/mod.rs index 72be0fbf86e..4a8db199505 100644 --- a/packages/rs-drive/src/query/mod.rs +++ b/packages/rs-drive/src/query/mod.rs @@ -17,6 +17,14 @@ pub use { drive_document_count_query::{ CountMode, DocumentCountMode, DriveDocumentCountQuery, SplitCountEntry, }, + // Having-range verifier-shareable types — same split as ranked: + // `DocumentHavingMode` + `AxisRangeBounds` to re-run the same + // versioned request validation (and bounds translation) the prover + // ran, `DriveDocumentHavingQuery` to rebuild the proved grove path + // and secondary query. Entries reuse the ranked `RankedEntry` shape. + drive_document_having_query::{ + AxisRangeBounds, DocumentHavingMode, DriveDocumentHavingQuery, MAX_HAVING_LIMIT, + }, // Ranked-query verifier-shareable types. The verifier needs the // whole set: `DocumentRankedMode` + `RankedPaginationInputs` to // re-run the same versioned request validation the prover ran, @@ -71,6 +79,13 @@ pub use drive_document_average_query::{DocumentAverageRequest, DocumentAverageRe // as the count / sum / average request types above. #[cfg(feature = "server")] pub use drive_document_ranked_query::{DocumentRankedRequest, DocumentRankedResponse}; + +// `DocumentHavingRequest` / `DocumentHavingResponse` are the +// server-side dispatcher ABI for the having-range surface — the types +// drive-abci's routing layer names. Server-only for the same reason as +// the ranked request types above. +#[cfg(feature = "server")] +pub use drive_document_having_query::{DocumentHavingRequest, DocumentHavingResponse}; // Imports available when either "server" or "verify" features are enabled #[cfg(any(feature = "server", feature = "verify"))] use { @@ -240,6 +255,14 @@ pub mod drive_document_sum_query; #[cfg(any(feature = "server", feature = "verify"))] pub mod drive_document_average_query; +/// A query to filter an index's groups by a per-group aggregate bound — +/// "hashtags with more than 100 posts" — served as a value-bounded +/// range read of the same per-axis secondary Merk the ranked surface +/// walks (PR #657, PV14). Like ranked, it never opens the value trees, +/// so a having-range read is `O(log n + k)` with a proof. +#[cfg(any(feature = "server", feature = "verify"))] +pub mod drive_document_having_query; + /// A query to rank an index's groups by a per-group aggregate — "top /// 5 restaurants by average grade" — reading grovedb's per-axis /// secondary Merk of an indexed tree (PR #657, PV14). Unlike the diff --git a/packages/rs-drive/src/verify/document_having/mod.rs b/packages/rs-drive/src/verify/document_having/mod.rs new file mode 100644 index 00000000000..a3ddcec97e1 --- /dev/null +++ b/packages/rs-drive/src/verify/document_having/mod.rs @@ -0,0 +1,19 @@ +//! Verifies grovedb proofs produced by the having-range +//! (`GROUP BY … HAVING LIMIT n`) query surface. +//! +//! Mirrors the layering of [`super::document_ranked`]: a pure +//! grovedb-level verifier as a method on +//! [`DriveDocumentHavingQuery`](crate::query::DriveDocumentHavingQuery) +//! taking raw `proof: &[u8]` and returning `(RootHash, T)`. The +//! tenderdash signature composition that wraps this call lives in +//! `rs-drive-proof-verifier`. +//! +//! Only one verifier exists here, for the same reason as on the ranked +//! surface: every having-range request — any of the three axes, either +//! direction, any contiguous bound — resolves to one +//! `prove_indexed_axis_query` envelope that differs only in the Merk +//! query (the encoded bounds + direction) and limit it echoes. + +/// Indexed-axis range proof verification — returns the groups the proof +/// commits to as falling inside the bound, in axis order. +pub mod verify_having_range_proof; diff --git a/packages/rs-drive/src/verify/document_having/verify_having_range_proof/mod.rs b/packages/rs-drive/src/verify/document_having/verify_having_range_proof/mod.rs new file mode 100644 index 00000000000..b4ba63ead28 --- /dev/null +++ b/packages/rs-drive/src/verify/document_having/verify_having_range_proof/mod.rs @@ -0,0 +1,52 @@ +mod v0; + +use crate::error::drive::DriveError; +use crate::error::Error; +use crate::query::DriveDocumentHavingQuery; +use crate::verify::RootHash; +use dpp::version::PlatformVersion; + +impl DriveDocumentHavingQuery<'_> { + /// Verifies a grovedb indexed-axis range proof and returns + /// `(root_hash, entries)`. + /// + /// Counterpart to the prover-side + /// [`execute_range_with_proof`](Self::execute_range_with_proof). + /// Both sides derive the proved subtree from + /// [`indexed_property_name_tree_path`](Self::indexed_property_name_tree_path) + /// and the secondary query from + /// [`AxisRangeBounds::merk_query`](crate::query::drive_document_having_query::AxisRangeBounds::merk_query), + /// so the verifier cannot drift from the prover on *which* bound over + /// *which* tree it is checking. + /// + /// The returned entries are in axis order in the walk direction, + /// exactly as the unproven + /// [`execute_range_no_proof`](Self::execute_range_no_proof) would + /// return them. The caller combines `root_hash` with the surrounding + /// tenderdash signature — see `rs-drive-proof-verifier` for the + /// canonical composition. + /// + /// # Arguments + /// * `proof` — raw grovedb proof bytes. + /// * `platform_version` — selects the method version. + pub fn verify_having_range_proof( + &self, + proof: &[u8], + platform_version: &PlatformVersion, + ) -> Result<(RootHash, Vec), Error> { + match platform_version + .drive + .methods + .verify + .document_ranked + .verify_having_range_proof + { + 0 => self.verify_having_range_proof_v0(proof), + version => Err(Error::Drive(DriveError::UnknownVersionMismatch { + method: "DriveDocumentHavingQuery::verify_having_range_proof".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive/src/verify/document_having/verify_having_range_proof/v0/mod.rs b/packages/rs-drive/src/verify/document_having/verify_having_range_proof/v0/mod.rs new file mode 100644 index 00000000000..fc9a0ab1727 --- /dev/null +++ b/packages/rs-drive/src/verify/document_having/verify_having_range_proof/v0/mod.rs @@ -0,0 +1,112 @@ +use crate::error::drive::DriveError; +use crate::error::Error; +use crate::query::{DriveDocumentHavingQuery, RankedAxis, RankedEntry, RankedEntryValue}; +use crate::verify::RootHash; +use grovedb::operations::proof::indexed_axis::AxisEntries; +use grovedb::GroveDb; + +impl DriveDocumentHavingQuery<'_> { + /// v0 of [`Self::verify_having_range_proof`]. + /// + /// Rebuilds the proved subtree path with + /// [`Self::indexed_property_name_tree_path`] and the secondary query + /// with + /// [`AxisRangeBounds::merk_query`](crate::query::drive_document_having_query::AxisRangeBounds::merk_query), + /// then hands the proof to the matching + /// `GroveDb::verify_indexed_*_query` — an associated function, no + /// database handle, so this compiles and runs in a verifier-only + /// build. + /// + /// Three things are checked before the entries are returned: + /// + /// 1. **The envelope matches this query.** grovedb re-checks the + /// proof against the reconstructed Merk query (the encoded bounds + /// and walk direction) and the expected limit, so a proof + /// generated for a different bound — or a different direction, or + /// a different limit — is rejected rather than silently + /// reinterpreted. Completeness rides on the same check: a Merk + /// range proof commits its boundaries, so an in-range group the + /// prover omitted fails reconstruction. + /// 2. **The result's axis shape matches the requested axis** — the + /// same belt-and-braces check the ranked verifier does. + /// 3. **At most `limit` entries.** Fewer is normal — fewer groups + /// may match the bound — but more would mean the proof committed + /// a longer walk than the request authorized. + /// + /// No `platform_version` argument: the parent dispatcher already + /// consumed it to select this version, and verification derives + /// everything else from the proof bytes plus the query. + #[inline(always)] + pub(super) fn verify_having_range_proof_v0( + &self, + proof: &[u8], + ) -> Result<(RootHash, Vec), Error> { + let path = self.indexed_property_name_tree_path()?; + let path_refs: Vec<&[u8]> = path.iter().map(|segment| segment.as_slice()).collect(); + let secondary_query = self.bounds.merk_query(self.descending); + + let result = match self.bounds.axis() { + RankedAxis::Count => GroveDb::verify_indexed_count_query( + proof, + path_refs.as_slice(), + secondary_query, + Some(self.limit), + ), + RankedAxis::Sum => GroveDb::verify_indexed_sum_query( + proof, + path_refs.as_slice(), + secondary_query, + Some(self.limit), + ), + RankedAxis::Avg => GroveDb::verify_indexed_avg_query( + proof, + path_refs.as_slice(), + secondary_query, + Some(self.limit), + ), + } + .map_err(|e| Error::GroveDB(Box::new(e)))?; + + let entries = match (self.bounds.axis(), result.entries) { + (RankedAxis::Count, AxisEntries::Count(entries)) => entries + .into_iter() + .map(|(count, key)| RankedEntry { + key, + value: RankedEntryValue::Count(count), + }) + .collect::>(), + (RankedAxis::Sum, AxisEntries::Sum(entries)) => entries + .into_iter() + .map(|(sum, key)| RankedEntry { + key, + value: RankedEntryValue::Sum(sum), + }) + .collect::>(), + (RankedAxis::Avg, AxisEntries::Avg(entries)) => entries + .into_iter() + .map(|(avg, key)| RankedEntry { + key, + value: RankedEntryValue::AvgFixedPoint(avg), + }) + .collect::>(), + (axis, other) => { + return Err(Error::Drive(DriveError::CorruptedDriveState(format!( + "having range proof for the {axis:?} axis verified to {} entries of a \ + different axis shape", + other.len() + )))); + } + }; + + if entries.len() > self.limit as usize { + return Err(Error::Drive(DriveError::CorruptedDriveState(format!( + "having range proof for the {:?} axis verified to {} entries for limit = {}", + self.bounds.axis(), + entries.len(), + self.limit + )))); + } + + Ok((result.root_hash, entries)) + } +} diff --git a/packages/rs-drive/src/verify/mod.rs b/packages/rs-drive/src/verify/mod.rs index 8777f4b8397..3875f2b8507 100644 --- a/packages/rs-drive/src/verify/mod.rs +++ b/packages/rs-drive/src/verify/mod.rs @@ -7,6 +7,10 @@ pub mod document; /// Document-count verification methods on proofs (the /// `GetDocumentsCount` endpoint's prove-path verifiers). pub mod document_count; +/// Having-range verification methods on proofs (the +/// `GROUP BY … HAVING LIMIT n` surface's +/// prove-path verifier). +pub mod document_having; /// Document-ranked verification methods on proofs (the /// `GROUP BY … ORDER BY LIMIT n` surface's prove-path /// verifier). diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/mod.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/mod.rs index 7c3e3e6dab1..b0594decefc 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/mod.rs @@ -1,6 +1,7 @@ pub mod v0; pub mod v1; pub mod v2; +pub mod v3; use versioned_feature_core::{FeatureVersion, FeatureVersionBounds}; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/v3.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/v3.rs new file mode 100644 index 00000000000..74c2e14564a --- /dev/null +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/v3.rs @@ -0,0 +1,31 @@ +use crate::version::drive_abci_versions::drive_abci_query_versions::v2::DRIVE_ABCI_QUERY_VERSIONS_V2; +use crate::version::drive_abci_versions::drive_abci_query_versions::{ + DriveAbciDocumentQueryHelperVersions, DriveAbciQueryVersions, +}; + +/// Version 3 of the Drive ABCI query versions. +/// +/// Differs from v2 in exactly one slot: +/// `document_query_helpers.compute_aggregate_mode_and_check_limit` is 2 +/// rather than 1. That is the boolean-`HAVING` routing gate. The v1 +/// helper rejects every non-empty `having` ("HAVING clause is not yet +/// implemented"); the v2 helper routes a grouped aggregate carrying +/// exactly one `having` clause (`GROUP BY p HAVING +/// LIMIT n`) to the having-range executor, which serves it as a +/// value-bounded range read of the covering ranked index's axis +/// secondary. Everything else — including multi-clause `having` and +/// `having` on a select with no ranked axis — keeps the v1 behavior. +/// +/// Same mixed-network rationale as the v1 → v2 flip: earlier protocol +/// versions keep the v2 table and keep rejecting the shape, so nodes +/// agree until the upgrade carries. The wire surface is unchanged — +/// `GetDocumentsRequestV1.having` has been wire-stable since the v1 +/// document query, and the response reuses the additive +/// `ResultData.ranked` entries shape (with `skipped` unset, since a +/// range page has no rank base). +pub const DRIVE_ABCI_QUERY_VERSIONS_V3: DriveAbciQueryVersions = DriveAbciQueryVersions { + document_query_helpers: DriveAbciDocumentQueryHelperVersions { + compute_aggregate_mode_and_check_limit: 2, + }, + ..DRIVE_ABCI_QUERY_VERSIONS_V2 +}; diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/mod.rs b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/mod.rs index 7b152be0f71..712f8a0c6f6 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/mod.rs @@ -41,6 +41,14 @@ pub struct DriveDocumentQueryMethodVersions { /// versions; the routing itself is unreachable before the ranked /// contract grammar activates. pub detect_ranked_mode: FeatureVersion, + /// Mode-detection routing table for boolean `HAVING` range queries + /// (`GROUP BY p HAVING LIMIT n`) served from an + /// indexed tree's axis secondary. Same versioning rationale and + /// same dormancy pattern as `detect_ranked_mode`: the slot exists + /// in every table, and the routing is unreachable before both the + /// ranked contract grammar and the v2 aggregate-routing helper + /// activate (protocol v14). + pub detect_having_mode: FeatureVersion, } #[derive(Clone, Debug, Default)] diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v1.rs b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v1.rs index dc32d71f710..7a7b2227786 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v1.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v1.rs @@ -18,6 +18,7 @@ pub const DRIVE_DOCUMENT_METHOD_VERSIONS_V1: DriveDocumentMethodVersions = detect_count_mode: 0, detect_sum_mode: 0, detect_ranked_mode: 0, + detect_having_mode: 0, }, delete: DriveDocumentDeleteMethodVersions { add_estimation_costs_for_remove_document_to_primary_storage: 0, diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v2.rs b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v2.rs index 6fcfc1b5dca..79889c33b9e 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v2.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v2.rs @@ -20,6 +20,7 @@ pub const DRIVE_DOCUMENT_METHOD_VERSIONS_V2: DriveDocumentMethodVersions = detect_count_mode: 0, detect_sum_mode: 0, detect_ranked_mode: 0, + detect_having_mode: 0, }, delete: DriveDocumentDeleteMethodVersions { add_estimation_costs_for_remove_document_to_primary_storage: 0, diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v3.rs b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v3.rs index 823842c8c01..cee8321e2f7 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v3.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v3.rs @@ -30,6 +30,7 @@ pub const DRIVE_DOCUMENT_METHOD_VERSIONS_V3: DriveDocumentMethodVersions = detect_count_mode: 0, detect_sum_mode: 0, detect_ranked_mode: 0, + detect_having_mode: 0, }, delete: DriveDocumentDeleteMethodVersions { add_estimation_costs_for_remove_document_to_primary_storage: 0, diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs index 5993ebac2a9..fbeec83b6e6 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs @@ -71,6 +71,7 @@ pub const DRIVE_DOCUMENT_METHOD_VERSIONS_V4: DriveDocumentMethodVersions = detect_count_mode: 0, detect_sum_mode: 0, detect_ranked_mode: 0, + detect_having_mode: 0, }, delete: DriveDocumentDeleteMethodVersions { add_estimation_costs_for_remove_document_to_primary_storage: 0, diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_verify_method_versions/mod.rs b/packages/rs-platform-version/src/version/drive_versions/drive_verify_method_versions/mod.rs index 83e293a9c7e..84af8082e48 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_verify_method_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_verify_method_versions/mod.rs @@ -73,13 +73,15 @@ pub struct DriveVerifyDocumentSumMethodVersions { pub verify_point_lookup_count_and_sum_proof: FeatureVersion, } -/// Versions for the ranked-aggregate (`HAVING ... TOP/BOTTOM/MIN/MAX`) -/// prove-path verifier. The single method is implemented on -/// `DriveDocumentRankedQuery` and returns `(RootHash, Vec)`, -/// delegating to grovedb's `verify_indexed_axis_top_k`. +/// Versions for the indexed-axis prove-path verifiers: the ranked +/// (top-k) verifier and the boolean-`HAVING` range verifier. Both are +/// implemented on the respective drive query types and delegate to +/// grovedb's indexed-axis proof verification +/// (`verify_indexed_axis_top_k_paginated` / `verify_indexed_axis_query`). #[derive(Clone, Debug, Default)] pub struct DriveVerifyDocumentRankedMethodVersions { pub verify_ranked_top_k_proof: FeatureVersion, + pub verify_having_range_proof: FeatureVersion, } #[derive(Clone, Debug, Default)] diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_verify_method_versions/v1.rs b/packages/rs-platform-version/src/version/drive_versions/drive_verify_method_versions/v1.rs index 30e0adb14f8..b9412b58615 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_verify_method_versions/v1.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_verify_method_versions/v1.rs @@ -41,6 +41,7 @@ pub const DRIVE_VERIFY_METHOD_VERSIONS_V1: DriveVerifyMethodVersions = DriveVeri }, document_ranked: DriveVerifyDocumentRankedMethodVersions { verify_ranked_top_k_proof: 0, + verify_having_range_proof: 0, }, identity: DriveVerifyIdentityMethodVersions { verify_full_identities_by_public_key_hashes: 0, diff --git a/packages/rs-platform-version/src/version/v14.rs b/packages/rs-platform-version/src/version/v14.rs index 984a7e46d64..f9c0d730100 100644 --- a/packages/rs-platform-version/src/version/v14.rs +++ b/packages/rs-platform-version/src/version/v14.rs @@ -16,7 +16,7 @@ use crate::version::dpp_versions::dpp_voting_versions::v2::VOTING_VERSION_V2; use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v9::DRIVE_ABCI_METHOD_VERSIONS_V9; -use crate::version::drive_abci_versions::drive_abci_query_versions::v2::DRIVE_ABCI_QUERY_VERSIONS_V2; +use crate::version::drive_abci_versions::drive_abci_query_versions::v3::DRIVE_ABCI_QUERY_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v10::DRIVE_ABCI_VALIDATION_VERSIONS_V10; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -95,14 +95,17 @@ pub const PROTOCOL_VERSION_14: ProtocolVersion = 14; /// `verify_ranked_top_k_proof`. All are 0 today. The same table bumps the /// four index walkers to v2 and the document update walker to v1 for the /// shared-prefix fix. -/// * `DRIVE_ABCI_QUERY_VERSIONS_V2` bumps -/// `document_query_helpers.compute_aggregate_mode_and_check_limit` 0 → 1, -/// opening the ranked path on the v1 document-query handler: a grouped -/// aggregate whose single `order_by` names the selected aggregate -/// (`ORDER BY [ASC|DESC] LIMIT n [OFFSET m]`) routes to the ranked -/// executor. v13 and earlier keep the v1 table and therefore keep -/// rejecting that shape, so mixed-version networks agree across the -/// upgrade. +/// * `DRIVE_ABCI_QUERY_VERSIONS_V3` bumps +/// `document_query_helpers.compute_aggregate_mode_and_check_limit` 0 → 2, +/// opening two routes on the v1 document-query handler: the ranked path +/// (a grouped aggregate whose single `order_by` names the selected +/// aggregate — `ORDER BY [ASC|DESC] LIMIT n [OFFSET m]`) and the +/// boolean-`HAVING` range path (a grouped aggregate carrying exactly one +/// `having` clause on the selected aggregate — `GROUP BY p HAVING +/// LIMIT n`), the latter served as a value-bounded range +/// read of the covering ranked index's axis secondary. v13 and earlier +/// keep the v1 table and therefore keep rejecting both shapes, so +/// mixed-version networks agree across the upgrade. /// * `DRIVE_ABCI_VALIDATION_VERSIONS_V10` bumps /// `document_create_transition_structure_validation` 0 → 1, requiring a /// contested create transition's prefunded voting balance to name the @@ -123,7 +126,7 @@ pub const PLATFORM_V14: PlatformVersion = PlatformVersion { methods: DRIVE_ABCI_METHOD_VERSIONS_V9, validation_and_processing: DRIVE_ABCI_VALIDATION_VERSIONS_V10, // changed: contested create transitions must name the contested index they resolve to withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, - query: DRIVE_ABCI_QUERY_VERSIONS_V2, // changed: ranked HAVING routing gate + query: DRIVE_ABCI_QUERY_VERSIONS_V3, // changed: ranked + boolean-HAVING routing gate checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, }, dpp: DPPVersion { @@ -155,16 +158,18 @@ mod tests { use super::*; use crate::version::v13::PLATFORM_V13; - /// The ranked-HAVING routing gate lives in v14's own query table, so - /// flipping it to feature version 1 touches only v14: a v13 node keeps - /// running the v0 helper, which rejects every non-empty HAVING, so a + /// The ranked / boolean-HAVING routing gate lives in v14's own query + /// table, so flipping it touches only v14: a v13 node keeps running + /// the v0 helper, which rejects every non-empty HAVING, so a /// mixed-version network agrees until the upgrade vote carries. /// - /// The flip is real as of the ranked routing landing — v14 selects the - /// v1 helper, which routes a single ranking-operand HAVING clause to - /// `dispatch_ranked_v1`. A change that made v13 non-zero here would be - /// consensus-breaking for already-deployed nodes, which is exactly what - /// the v13 half of this assertion guards. + /// v14 selects the v2 helper, which routes the ranked shape + /// (`ORDER BY LIMIT n`) to `dispatch_ranked_v1` and the + /// boolean-HAVING range shape (exactly one `having` clause on the + /// selected aggregate) to `dispatch_having_v1`. A change that made + /// v13 non-zero here would be consensus-breaking for + /// already-deployed nodes, which is exactly what the v13 half of + /// this assertion guards. #[test] fn ranked_having_routing_gate_is_v14_only() { assert_eq!( @@ -181,7 +186,7 @@ mod tests { .query .document_query_helpers .compute_aggregate_mode_and_check_limit, - 1 + 2 ); } @@ -263,6 +268,10 @@ mod tests { PLATFORM_V14.drive.methods.document.query.detect_ranked_mode, 0 ); + assert_eq!( + PLATFORM_V14.drive.methods.document.query.detect_having_mode, + 0 + ); assert_eq!( PLATFORM_V14 .drive @@ -272,6 +281,15 @@ mod tests { .verify_ranked_top_k_proof, 0 ); + assert_eq!( + PLATFORM_V14 + .drive + .methods + .verify + .document_ranked + .verify_having_range_proof, + 0 + ); let grove = &PLATFORM_V14.drive.grove_methods.batch; assert_eq!(grove.batch_insert_empty_provable_count_indexed_tree, 0); assert_eq!(grove.batch_insert_empty_provable_sum_indexed_tree, 0); diff --git a/packages/rs-sdk/src/mock/requests.rs b/packages/rs-sdk/src/mock/requests.rs index 7cb0696ac20..9015c19a583 100644 --- a/packages/rs-sdk/src/mock/requests.rs +++ b/packages/rs-sdk/src/mock/requests.rs @@ -789,3 +789,27 @@ impl MockResponse for drive_proof_verifier::DocumentRankedEntries { } } } + +impl MockResponse for drive_proof_verifier::DocumentHavingEntries { + /// Rides the ranked page encoding with a starting rank of `0`: a + /// having page is the same ordered `(group key, axis tag, value)` + /// list, just addressed by value bound instead of by rank, and it + /// has no rank base to preserve. + fn mock_serialize(&self, sdk: &MockDashPlatformSdk) -> Vec { + drive_proof_verifier::DocumentRankedEntries { + starting_rank: 0, + entries: self.entries.clone(), + } + .mock_serialize(sdk) + } + + fn mock_deserialize(sdk: &MockDashPlatformSdk, buf: &[u8]) -> Self + where + Self: Sized, + { + let page = drive_proof_verifier::DocumentRankedEntries::mock_deserialize(sdk, buf); + drive_proof_verifier::DocumentHavingEntries { + entries: page.entries, + } + } +} diff --git a/packages/rs-sdk/src/platform/documents/document_having_entries.rs b/packages/rs-sdk/src/platform/documents/document_having_entries.rs new file mode 100644 index 00000000000..876a034c041 --- /dev/null +++ b/packages/rs-sdk/src/platform/documents/document_having_entries.rs @@ -0,0 +1,319 @@ +//! `FromProof` + `Fetch` for [`DocumentHavingEntries`] — the +//! **having-range** (`GROUP BY … HAVING +//! LIMIT n`) view of the unified `getDocuments` endpoint. +//! +//! A having-range query answers "which groups' aggregate falls inside a +//! value bound?" — *hashtags with more than 100 posts* — in +//! `O(log n + k)`, with a proof whose Merk range boundaries also attest +//! **completeness**: a node cannot silently omit a matching group. It +//! reads the same pre-sorted per-axis *secondary* Merk the ranked +//! surface walks (grovedb PR #657), addressed by value bound instead of +//! by rank. +//! +//! Per-request resolution (which axis, which bounds the operator +//! translates to, which index covers them) lives in +//! [`super::having_proof_helpers`]; this module is the thin +//! `Fetch`-side wrapper. +//! +//! ## Request shape +//! +//! Exactly one aggregate `select`, exactly one `group_by` property, +//! exactly one `having` clause **bounding the selected aggregate** with +//! a contiguous-range operator (`=`, `>`, `>=`, `<`, `<=`, `BETWEEN*` — +//! `!=` and `IN` are rejected), and a `LIMIT`. `ORDER BY` is optional: +//! omitted means ascending by the aggregate; naming the selected +//! aggregate sets the direction. No `where`, no `offset`, no +//! `start_at`. +//! +//! ## Contract prerequisites +//! +//! Same as the ranked surface: the index must opt in with +//! `rankedCountable` / `rankedSummable` / `rankedAverageable` +//! (meta-schema v3, **protocol version 14+**), and ranked indexes are +//! single-property. Against a pre-v14 node the request is refused with +//! "HAVING clause is not yet implemented" — the intended activation +//! gate. +//! +//! ## Reading the result +//! +//! Entries come back in axis order in the walk direction; **do not +//! re-sort**. Fewer than `n` entries means fewer groups matched. +//! **Exactly `n` may mean the match set was cut at the limit** — to +//! continue, tighten the bound past the last aggregate value seen and +//! ask again. Averages are fixed-point integers, exact on this (proved) +//! path; see the ranked module's notes, which apply verbatim. +//! +//! ## Example: hashtags with more than 100 posts +//! +//! `SELECT COUNT(*) GROUP BY hashtag HAVING $count > 100 ORDER BY $count DESC LIMIT 100` +//! +//! ```rust,no_run +//! use dash_sdk::{Sdk, platform::{DataContract, DocumentQuery, Fetch, Identifier}}; +//! use dash_sdk::drive::query::{ +//! HavingAggregate, HavingAggregateFunction, HavingClause, HavingOperator, +//! HavingRightOperand, SelectProjection, +//! }; +//! use dash_sdk::platform::documents::document_query::RankingDirection; +//! use dpp::platform_value::Value; +//! use drive_proof_verifier::DocumentHavingEntries; +//! use futures::executor::block_on; +//! +//! # const POSTS_CONTRACT_ID: [u8; 32] = [0; 32]; +//! let sdk = Sdk::new_mock(); +//! let contract = block_on(DataContract::fetch(&sdk, Identifier::new(POSTS_CONTRACT_ID))) +//! .expect("fetch contract") +//! .expect("contract exists"); +//! +//! let query = DocumentQuery::new(contract, "post") +//! .expect("document type exists") +//! .with_select(SelectProjection::count_star()) +//! .with_group_by("hashtag") +//! .with_having(vec![HavingClause { +//! aggregate: HavingAggregate { +//! function: HavingAggregateFunction::Count, +//! field: String::new(), +//! }, +//! operator: HavingOperator::GreaterThan, +//! right: HavingRightOperand::Value(Value::U64(100)), +//! }]) +//! .order_by_selected_aggregate(RankingDirection::Descending) +//! .with_limit(100); +//! +//! let matching = block_on(DocumentHavingEntries::fetch(&sdk, query)) +//! .expect("fetch succeeds") +//! .expect("a well-formed having query always answers"); +//! +//! for entry in &matching.entries { +//! let hashtag = String::from_utf8_lossy(&entry.key); +//! println!("#{hashtag}: {} posts", entry.value.as_f64()); +//! } +//! ``` + +use crate::platform::documents::document_query::DocumentQuery; +use crate::platform::documents::having_proof_helpers::verify_having_query; +use crate::platform::Fetch; +use dapi_grpc::platform::v0::{GetDocumentsResponse, Proof, ResponseMetadata}; +use dash_context_provider::ContextProvider; +use dpp::dashcore::Network; +use dpp::version::PlatformVersion; +use drive_proof_verifier::{DocumentHavingEntries, FromProof}; + +impl FromProof for DocumentHavingEntries { + type Request = DocumentQuery; + type Response = GetDocumentsResponse; + + fn maybe_from_proof_with_metadata<'a, I: Into, O: Into>( + request: I, + response: O, + _network: Network, + platform_version: &PlatformVersion, + provider: &'a dyn ContextProvider, + ) -> Result<(Option, ResponseMetadata, Proof), drive_proof_verifier::Error> + where + Self: 'a, + { + let request: Self::Request = request.into(); + let response: Self::Response = response.into(); + // Same single-pass design as the ranked impl: the grammar check + // is the first step of resolution, inside the helper. + let (entries, mtd, proof) = + verify_having_query(request, response, platform_version, provider)?; + Ok(( + entries.map(DocumentHavingEntries::from_verified), + mtd, + proof, + )) + } +} + +impl Fetch for DocumentHavingEntries { + type Query = DocumentQuery; + type Request = dapi_grpc::platform::v0::GetDocumentsRequest; +} + +#[cfg(test)] +mod tests { + //! Offline tests for the having client surface: the request→wire + //! encoding and the client-side grammar mirror. Proof verification + //! is exercised end-to-end in rs-drive's + //! `drive_document_having_query::tests` and rs-drive-abci's + //! `having_range_tests`, where a populated Drive exists. + + use super::*; + use crate::platform::documents::document_query::RankingDirection; + use crate::platform::documents::having_proof_helpers::assert_having_shape; + use dapi_grpc::platform::v0::get_documents_request::get_documents_request_v1::select as proto_select; + use dapi_grpc::platform::v0::get_documents_request::{ + having_aggregate, having_clause, GetDocumentsRequestV1, Version as RequestVersion, + }; + use dapi_grpc::platform::v0::GetDocumentsRequest; + use dpp::data_contract::DataContract; + use dpp::platform_value::Value; + use dpp::tests::fixtures::get_data_contract_fixture; + use dpp::version::TryFromPlatformVersioned; + use drive::query::{ + AxisRangeBounds, HavingAggregate, HavingAggregateFunction, HavingClause, HavingOperator, + HavingRightOperand, SelectProjection, + }; + use std::sync::Arc; + + fn platform_version() -> &'static PlatformVersion { + PlatformVersion::latest() + } + + fn contract() -> Arc { + Arc::new( + get_data_contract_fixture(None, 0, platform_version().protocol_version) + .data_contract_owned(), + ) + } + + fn count_over_100() -> HavingClause { + HavingClause { + aggregate: HavingAggregate { + function: HavingAggregateFunction::Count, + field: String::new(), + }, + operator: HavingOperator::GreaterThan, + right: HavingRightOperand::Value(Value::U64(100)), + } + } + + /// `SELECT COUNT(*) GROUP BY hashtag HAVING $count > 100 LIMIT 100`. + fn hashtags_over_100() -> DocumentQuery { + DocumentQuery::new(contract(), "niceDocument") + .expect("the fixture has this document type") + .with_select(SelectProjection::count_star()) + .with_group_by("hashtag") + .with_having(vec![count_over_100()]) + .with_limit(100) + } + + fn v1_of(query: DocumentQuery) -> GetDocumentsRequestV1 { + let request = GetDocumentsRequest::try_from_platform_versioned(query, platform_version()) + .expect("a having query encodes onto the V1 wire"); + match request.version.expect("the encoder always sets a version") { + RequestVersion::V1(v1) => v1, + RequestVersion::V0(_) => { + panic!("a having query must encode onto the V1 wire; V0 has no `having` field") + } + } + } + + /// The headline round-trip: the wire shape must be exactly what the + /// server's routing accepts — one select, one group_by, one having + /// clause, a limit, nothing else. + #[test] + fn having_query_encodes_the_expected_wire_shape() { + let v1 = v1_of(hashtags_over_100()); + + assert_eq!(v1.selects.len(), 1); + assert_eq!(v1.selects[0].function, proto_select::Function::Count as i32); + assert_eq!(v1.selects[0].field, ""); + assert_eq!(v1.group_by, vec!["hashtag".to_string()]); + + assert_eq!(v1.having.len(), 1, "exactly one having clause"); + let clause = &v1.having[0]; + let aggregate = clause.aggregate.as_ref().expect("aggregate is set"); + assert_eq!(aggregate.function, having_aggregate::Function::Count as i32); + assert_eq!(aggregate.field, ""); + assert_eq!(clause.operator, having_clause::Operator::GreaterThan as i32); + assert!(clause.right.is_some(), "the right operand rides the oneof"); + + assert_eq!(v1.limit, Some(100)); + assert!(v1.where_clauses.is_empty()); + assert!( + v1.order_by.is_empty(), + "order_by is optional and unset here" + ); + assert_eq!(v1.offset, None); + assert!(v1.start.is_none()); + assert!(v1.prove, "the Fetch path always requests a proof"); + } + + /// The client-side grammar must resolve the same bounds the server + /// (and therefore the prover) resolves — the bounds are rebuilt + /// into the proof's Merk query at verification time, so a client + /// that translated `> 100` differently could not verify an honest + /// proof. + #[test] + fn assert_having_shape_resolves_the_bounds() { + let mode = assert_having_shape(&hashtags_over_100(), platform_version()) + .expect("the headline query is well-formed"); + assert_eq!( + mode.bounds, + AxisRangeBounds::Count { + lo: 101, + hi: u64::MAX + } + ); + assert!(!mode.descending, "no order_by means ascending"); + assert_eq!(mode.limit, 100); + assert_eq!(mode.group_by_property, "hashtag"); + } + + /// An explicit descending ordering on the selected aggregate flips + /// the walk; biggest matching groups come first. + #[test] + fn ordering_by_the_aggregate_sets_the_direction() { + let query = hashtags_over_100().order_by_selected_aggregate(RankingDirection::Descending); + let mode = assert_having_shape(&query, platform_version()) + .expect("having + ORDER BY the aggregate is well-formed"); + assert!(mode.descending); + } + + /// Every knob the range walk cannot honour is rejected client side, + /// before a round trip — mirroring the server's rejections. + #[test] + fn assert_having_shape_rejects_what_the_range_cannot_honour() { + let base = hashtags_over_100(); + + // No having at all: a plain grouped aggregate. + let mut no_having = base.clone(); + no_having.having = Vec::new(); + assert!(assert_having_shape(&no_having, platform_version()).is_err()); + + // Two clauses: implicit AND is a future capability. + let two = base + .clone() + .with_having(vec![count_over_100(), count_over_100()]); + assert!(assert_having_shape(&two, platform_version()).is_err()); + + // A clause on a different aggregate than the select. + let cross = base.clone().with_having(vec![HavingClause { + aggregate: HavingAggregate { + function: HavingAggregateFunction::Sum, + field: "amount".to_string(), + }, + operator: HavingOperator::GreaterThan, + right: HavingRightOperand::Value(Value::I64(100)), + }]); + assert!(assert_having_shape(&cross, platform_version()).is_err()); + + // An offset: the range walk has no skip. + let with_offset = base.clone().with_offset(4); + assert!(assert_having_shape(&with_offset, platform_version()).is_err()); + + // Non-contiguous operators. + for operator in [HavingOperator::NotEqual, HavingOperator::In] { + let mut clause = count_over_100(); + clause.operator = operator; + let query = base.clone().with_having(vec![clause]); + assert!(assert_having_shape(&query, platform_version()).is_err()); + } + } + + /// The generic FromProof guard in drive-proof-verifier must not be + /// reachable from the SDK path: this impl (on `DocumentQuery`) is + /// the one `fetch` resolves, and it runs the real verification. + #[test] + fn limit_is_required_and_capped_client_side() { + for limit in [0u32, 101] { + let query = hashtags_over_100().with_limit(limit); + assert!( + assert_having_shape(&query, platform_version()).is_err(), + "LIMIT {limit} is outside 1..=100 and must be rejected, not clamped" + ); + } + } +} diff --git a/packages/rs-sdk/src/platform/documents/document_query.rs b/packages/rs-sdk/src/platform/documents/document_query.rs index 57648cd1988..38d6a5c4dc1 100644 --- a/packages/rs-sdk/src/platform/documents/document_query.rs +++ b/packages/rs-sdk/src/platform/documents/document_query.rs @@ -89,16 +89,26 @@ pub struct DocumentQuery { /// [`drive::query::HavingOperator`] for the catalogs. Multiple /// entries combine with implicit `AND`. /// - /// **Every non-empty value is rejected by the server** with - /// `QuerySyntaxError::Unsupported("HAVING clause is not yet - /// implemented")`, at every protocol version. The typed builder - /// exists so callers can encode `HAVING` ahead of server support - /// landing without a wire-format change. + /// **Served from protocol version 14, for exactly one clause + /// bounding the selected aggregate** with a contiguous-range + /// operator (`=`, `>`, `>=`, `<`, `<=`, `BETWEEN*`) — the + /// having-range surface, fetched as + /// [`DocumentHavingEntries`](drive_proof_verifier::DocumentHavingEntries) + /// and served as a value-bounded range read of the covering ranked + /// index's axis secondary (the index must declare the matching + /// `rankedCountable` / `rankedSummable` / `rankedAverageable` + /// keyword). Everything else — multiple clauses (implicit AND), a + /// clause on an aggregate the select does not project, `!=` / `IN` + /// — is still rejected with `QuerySyntaxError::Unsupported`, as is + /// any non-empty value at protocol version 13 and earlier. /// /// **`having` does not express ranking.** "The n highest-scoring /// groups" is [`Self::order_by_selected_aggregate`] + /// [`Self::with_limit`] — SQL's own `ORDER BY DESC LIMIT n` - /// — which *is* served, from protocol version 14. + /// — which is also served from protocol version 14. The two + /// compose only in the one shape the having grammar allows: an + /// `ORDER BY` naming the selected aggregate sets the having + /// range's walk direction. #[cfg_attr(feature = "mocks", serde(default))] pub having: Vec, /// `order_by` clauses for the query. diff --git a/packages/rs-sdk/src/platform/documents/having_proof_helpers.rs b/packages/rs-sdk/src/platform/documents/having_proof_helpers.rs new file mode 100644 index 00000000000..2958f7ab0e7 --- /dev/null +++ b/packages/rs-sdk/src/platform/documents/having_proof_helpers.rs @@ -0,0 +1,162 @@ +//! Having-range proof dispatch used by [`DocumentHavingEntries`]. +//! +//! Having-side analog of [`super::ranked_proof_helpers`]: it turns a +//! caller-built [`DocumentQuery`] plus the node's response into a +//! verified entry list. The routing decisions — which axis, which +//! inclusive bounds the operator translates to, which direction, which +//! index covers them — are **not** re-derived here. They come from +//! rs-drive's own [`detect_having_mode`] and +//! [`find_ranked_index_for_axis`], the same two functions the server +//! calls, so client and server land on the same grove path and the same +//! bounds by construction rather than by two copies of a grammar +//! agreeing. The bounds matter doubly here: the verifier rebuilds the +//! proof's Merk query from them, so a divergence is a failed +//! verification, not a subtly different answer. +//! +//! [`DocumentHavingEntries`]: drive_proof_verifier::DocumentHavingEntries + +use crate::platform::documents::document_query::DocumentQuery; +use dapi_grpc::platform::v0::{GetDocumentsResponse, Proof, ResponseMetadata}; +use dapi_grpc::platform::VersionedGrpcResponse; +use dash_context_provider::ContextProvider; +use dpp::version::PlatformVersion; +use dpp::{ + data_contract::accessors::v0::DataContractV0Getters, + data_contract::document_type::accessors::DocumentTypeV0Getters, +}; +use drive::query::drive_document_having_query::mode_detection::detect_having_mode; +use drive::query::drive_document_ranked_query::index_picker::find_ranked_index_for_axis; +use drive::query::{ + DocumentHavingMode, DriveDocumentHavingQuery, RankedEntry, RankedPaginationInputs, +}; +use drive_proof_verifier::verify_having_range_proof; + +/// Validate that the caller-built [`DocumentQuery`] really describes a +/// having-range query, and resolve it into the `(bounds, descending, +/// limit, group property, aggregate field)` tuple the index picker and +/// the prover both work from. +/// +/// Same return-the-mode design as +/// [`assert_ranked_shape`](super::ranked_proof_helpers::assert_ranked_shape), +/// for the same reason: the grammar check *is* the first step of +/// resolution. The grammar lives in rs-drive ([`detect_having_mode`]) +/// and is versioned through +/// `platform_version.drive.methods.document.query.detect_having_mode`, +/// so the SDK cannot resolve a clause to different bounds than the +/// prover used. +pub(super) fn assert_having_shape( + request: &DocumentQuery, + platform_version: &PlatformVersion, +) -> Result { + // Same sentinel handling as the ranked helper: `limit == 0` is + // `DocumentQuery`'s "unset", reported to rs-drive as `None`. + let pagination = RankedPaginationInputs { + limit: (request.limit != 0).then_some(request.limit), + offset: request.offset, + has_start_at: request.start.is_some(), + }; + + detect_having_mode( + &request.select, + &request.group_by, + &request.having, + &request.order_by_clauses, + &request.where_clauses, + pagination, + platform_version, + ) + .map_err(|e| drive_proof_verifier::Error::RequestError { + error: format!( + "this DocumentQuery is not a well-formed having-range query: {e}. A having-range \ + query is `.with_select()`, `.with_group_by()`, \ + `.with_having()` and `.with_limit(n)`, optionally \ + `.order_by_selected_aggregate()`, with no where clauses, no offset \ + and no start_at." + ), + }) +} + +/// Verify a having-range proof and return the verified entries — the +/// matching groups **in axis order in the walk direction**. +/// +/// Single source of truth for the having proof path, mirroring +/// [`verify_ranked_query`](super::ranked_proof_helpers::verify_ranked_query) +/// step for step: re-run rs-drive's versioned request validation +/// (which resolves the bounds), resolve the covering index off the +/// contract, rebuild the query, verify. The root-hash binding to the +/// quorum-signed app hash happens inside [`verify_having_range_proof`] +/// and cannot be skipped through this helper. +pub(super) fn verify_having_query( + request: DocumentQuery, + response: GetDocumentsResponse, + platform_version: &PlatformVersion, + provider: &dyn ContextProvider, +) -> Result<(Option>, ResponseMetadata, Proof), drive_proof_verifier::Error> { + let document_type = request + .data_contract + .document_type_for_name(&request.document_type_name) + .map_err(|e| drive_proof_verifier::Error::RequestError { + error: format!( + "document type {} not found in contract: {}", + request.document_type_name, e + ), + })?; + let proof = response + .proof() + .or(Err(drive_proof_verifier::Error::NoProofInResult))?; + let mtd = response + .metadata() + .or(Err(drive_proof_verifier::Error::EmptyResponseMetadata))?; + + let mode = assert_having_shape(&request, platform_version)?; + let axis = mode.bounds.axis(); + + // Pick the index the prover picked — rs-drive's own picker, shared + // with the ranked surface because both read the same indexed tree. + let index = find_ranked_index_for_axis( + document_type.indexes(), + &mode.group_by_property, + axis, + &mode.aggregate_field, + ) + .ok_or_else(|| drive_proof_verifier::Error::RequestError { + error: format!( + "no index on document type `{}` can serve a `{:?}` having bound grouped on \ + `{}`: a having-range query needs a single-property index over `{}` declaring \ + `{}` (and, for SUM / AVG, `summable: \"{}\"`). Ranked indexes are opt-in \ + contract grammar (meta-schema v3, protocol version 14+).", + request.document_type_name, + axis, + mode.group_by_property, + mode.group_by_property, + axis.required_index_keyword(), + mode.aggregate_field, + ), + })?; + + let having_query = DriveDocumentHavingQuery { + document_type, + contract_id: request.data_contract.id().to_buffer(), + document_type_name: request.document_type_name.clone(), + index, + bounds: mode.bounds, + descending: mode.descending, + limit: mode.limit, + }; + + // Binds the reconstructed grovedb root hash to the quorum-signed + // app hash before returning — see the module docs. + let (root_hash, entries) = + verify_having_range_proof(&having_query, proof, mtd, platform_version, provider)?; + + tracing::trace!( + target: "dash_sdk::having_query", + root_hash = hex::encode(root_hash), + height = mtd.height, + entries = entries.len(), + "verified having range proof" + ); + + Ok((Some(entries), mtd.clone(), proof.clone())) +} diff --git a/packages/rs-sdk/src/platform/documents/mod.rs b/packages/rs-sdk/src/platform/documents/mod.rs index dbb6c5ae5ba..5f7531c7d51 100644 --- a/packages/rs-sdk/src/platform/documents/mod.rs +++ b/packages/rs-sdk/src/platform/documents/mod.rs @@ -4,6 +4,12 @@ pub(super) mod count_proof_helpers; /// `(count, sum)`; client divides. pub mod document_average; pub mod document_count; +/// `Fetch` impl for the having-range (`GROUP BY … HAVING +/// LIMIT n`) result — one entry per matching group, in +/// axis order, with proof-attested completeness. Requires an index +/// declaring `rankedCountable` / `rankedSummable` / `rankedAverageable` +/// (protocol version 14+). +pub mod document_having_entries; pub mod document_history_query; pub mod document_query; /// `Fetch` impl for the ranked (`GROUP BY … ORDER BY LIMIT n @@ -22,6 +28,7 @@ pub mod document_split_sums; /// `Fetch` impl for the sum-side aggregate result. Mirrors /// `document_count`. Lights up alongside grovedb PR 670. pub mod document_sum; +pub(super) mod having_proof_helpers; pub(super) mod ranked_proof_helpers; pub(super) mod sum_proof_helpers; pub mod transitions; From 25cce542fe25a6e263dd9a3dd39fbafc6f39577d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 04:54:47 +0700 Subject: [PATCH 02/12] test(drive): having-range proof round-trip with identifier group keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins the worked example — SELECT AVG(grade) GROUP BY identityId HAVING AVG(grade) > 80 — against a contract whose group key is a 32-byte identifier rather than a string: strict-bound exclusion of an exactly-at-threshold average, inclusion of a fractional average just above it, byte-exact identifier keys in both walk directions, and proof verification against the live root hash. Co-Authored-By: Claude Fable 5 --- .../drive_document_having_query/tests.rs | 286 ++++++++++++++++++ .../grades/grades-ranked-contract.json | 45 +++ 2 files changed, 331 insertions(+) create mode 100644 packages/rs-drive/tests/supporting_files/contract/grades/grades-ranked-contract.json diff --git a/packages/rs-drive/src/query/drive_document_having_query/tests.rs b/packages/rs-drive/src/query/drive_document_having_query/tests.rs index 8b56398769b..4ed267aca9a 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/tests.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/tests.rs @@ -1110,3 +1110,289 @@ mod execution { assert_proof_round_trips(&drive, &contract, &page_two, &rest); } } + +mod identifier_group_keys { + //! `SELECT AVG(grade) FROM grades GROUP BY identityId HAVING + //! AVG(grade) > 80` — the same surface as the `execution` suite + //! above, but with a **32-byte identifier** as the group key + //! instead of a string. Identifier and string properties encode + //! differently into the axis secondary's `sort_key‖group_key` + //! keyspace, so this pins that identifier group keys round-trip + //! byte-exact through the read, the proof, and the verifier. + + use super::super::drive_dispatcher::{DocumentHavingRequest, DocumentHavingResponse}; + use super::super::mode_detection::detect_having_mode; + use super::super::DriveDocumentHavingQuery; + use super::clause; + use crate::drive::Drive; + use crate::query::drive_document_ranked_query::index_picker::find_ranked_index_for_axis; + use crate::query::drive_document_ranked_query::{ + RankedEntry, RankedEntryValue, RankedPaginationInputs, + }; + use crate::query::having::{HavingAggregateFunction, HavingOperator}; + use crate::query::projection::SelectProjection; + use crate::query::OrderClause; + use crate::util::object_size_info::DocumentInfo::DocumentRefInfo; + use crate::util::object_size_info::{DocumentAndContractInfo, OwnedDocumentInfo}; + use crate::util::storage_flags::StorageFlags; + use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; + use dpp::block::block_info::BlockInfo; + use dpp::data_contract::accessors::v0::DataContractV0Getters; + use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; + use dpp::data_contract::document_type::random_document::CreateRandomDocument; + use dpp::document::{Document, DocumentV0Setters}; + use dpp::platform_value::Value; + use dpp::prelude::DataContract; + use dpp::tests::json_document::json_document_to_contract; + use dpp::version::PlatformVersion; + use grovedb::element::indexed::compute_avg_fixed_point; + use std::collections::BTreeMap; + + const GROUP_PROPERTY: &str = "identityId"; + const DOCUMENT_TYPE: &str = "grade"; + + fn platform_version() -> &'static PlatformVersion { + PlatformVersion::latest() + } + + fn setup_grades_ranked() -> (Drive, DataContract) { + let drive = setup_drive_with_initial_state_structure(None); + let pv = platform_version(); + let contract = json_document_to_contract( + "tests/supporting_files/contract/grades/grades-ranked-contract.json", + false, + pv, + ) + .expect("expected to parse the ranked grades contract"); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + pv, + ) + .expect("expected to apply the ranked grades contract"); + (drive, contract) + } + + fn insert_grades( + drive: &Drive, + contract: &DataContract, + first_seed: u64, + rows: &[([u8; 32], i64)], + ) { + let pv = platform_version(); + let document_type = contract + .document_type_for_name(DOCUMENT_TYPE) + .expect("grade doctype exists"); + for (i, (identity, grade)) in rows.iter().enumerate() { + let mut doc: Document = document_type + .random_document(Some(first_seed + i as u64), pv) + .expect("random document"); + let mut props = BTreeMap::new(); + props.insert(GROUP_PROPERTY.to_string(), Value::Identifier(*identity)); + props.insert("grade".to_string(), Value::I64(*grade)); + doc.set_properties(props); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo((&doc, None)), + owner_id: None, + }, + contract, + document_type, + }, + false, + BlockInfo::default(), + true, + None, + pv, + None, + ) + .expect("expected to insert a grade document"); + } + } + + fn run( + drive: &Drive, + contract: &DataContract, + order_by: &[OrderClause], + prove: bool, + ) -> DocumentHavingResponse { + let group_by = vec![GROUP_PROPERTY.to_string()]; + let having = vec![clause( + HavingAggregateFunction::Avg, + "grade", + HavingOperator::GreaterThan, + Value::U64(80), + )]; + drive + .execute_document_having_request( + DocumentHavingRequest { + contract, + document_type: contract + .document_type_for_name(DOCUMENT_TYPE) + .expect("grade doctype exists"), + group_by: &group_by, + select: SelectProjection::avg("grade"), + having: &having, + order_by, + where_clauses: &[], + limit: Some(10), + offset: None, + has_start_at: false, + prove, + }, + None, + platform_version(), + ) + .expect("the having request must execute") + } + + fn client_side_query<'a>( + contract: &'a DataContract, + order_by: &[OrderClause], + ) -> DriveDocumentHavingQuery<'a> { + let group_by = vec![GROUP_PROPERTY.to_string()]; + let having = vec![clause( + HavingAggregateFunction::Avg, + "grade", + HavingOperator::GreaterThan, + Value::U64(80), + )]; + let mode = detect_having_mode( + &SelectProjection::avg("grade"), + &group_by, + &having, + order_by, + &[], + RankedPaginationInputs { + limit: Some(10), + offset: None, + has_start_at: false, + }, + platform_version(), + ) + .expect("the case is well-formed"); + let indexes = contract + .document_types() + .get(DOCUMENT_TYPE) + .expect("grade doctype exists") + .indexes(); + let index = find_ranked_index_for_axis( + indexes, + &mode.group_by_property, + mode.bounds.axis(), + &mode.aggregate_field, + ) + .expect("the fixture declares the avg axis"); + DriveDocumentHavingQuery { + document_type: contract + .document_type_for_name(DOCUMENT_TYPE) + .expect("grade doctype exists"), + contract_id: contract.id_ref().to_buffer(), + document_type_name: DOCUMENT_TYPE.to_string(), + index, + bounds: mode.bounds, + descending: mode.descending, + limit: mode.limit, + } + } + + fn assert_proof_round_trips( + drive: &Drive, + contract: &DataContract, + order_by: &[OrderClause], + expected: &[RankedEntry], + ) { + let proof = match run(drive, contract, order_by, true) { + DocumentHavingResponse::Proof(proof) => proof, + DocumentHavingResponse::Entries(_) => panic!("expected a proof, got entries"), + }; + let (root_hash, verified) = client_side_query(contract, order_by) + .verify_having_range_proof(&proof, platform_version()) + .expect("the proof must verify"); + assert_eq!( + verified, expected, + "verified entries must equal what the unproven read returned" + ); + assert_eq!( + root_hash, + drive + .grove + .root_hash(None, &platform_version().drive.grove_version) + .unwrap() + .expect("root hash must be readable"), + "the proof must reconstruct the live grovedb root hash" + ); + } + + /// Averages exactly *at* the threshold stay out (`>` is strict), + /// fractional averages just above it come in (80.5 > 80 even + /// though both grades round-trip as integers), and the entry keys + /// are the raw 32-byte identifiers. + #[test] + fn avg_threshold_over_identifier_groups_reads_and_proves() { + let (drive, contract) = setup_grades_ranked(); + let at_threshold = [1u8; 32]; // 80, 80 → avg 80: excluded + let just_above = [2u8; 32]; // 80, 81 → avg 80.5: included + let well_above = [3u8; 32]; // 85, 95 → avg 90: included + let below = [4u8; 32]; // 60, 80 → avg 70: excluded + insert_grades( + &drive, + &contract, + 1000, + &[ + (at_threshold, 80), + (at_threshold, 80), + (just_above, 80), + (just_above, 81), + (well_above, 85), + (well_above, 95), + (below, 60), + (below, 80), + ], + ); + + let entries = match run(&drive, &contract, &[], false) { + DocumentHavingResponse::Entries(entries) => entries, + DocumentHavingResponse::Proof(_) => panic!("expected entries, got a proof"), + }; + assert_eq!( + entries, + vec![ + RankedEntry { + key: just_above.to_vec(), + value: RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(161, 2)), + }, + RankedEntry { + key: well_above.to_vec(), + value: RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(180, 2)), + }, + ], + "ascending walk: 80.5 then 90, keyed by raw identifier bytes" + ); + assert_proof_round_trips(&drive, &contract, &[], &entries); + + // `ORDER BY AVG(grade) DESC` walks the same match set from the + // top; the identifier keys must survive the flipped direction + // and its proof too. + let descending = vec![OrderClause { + field: "grade".to_string(), + ascending: false, + }]; + let flipped = match run(&drive, &contract, &descending, false) { + DocumentHavingResponse::Entries(entries) => entries, + DocumentHavingResponse::Proof(_) => panic!("expected entries, got a proof"), + }; + assert_eq!( + flipped.iter().map(|e| &e.key).collect::>(), + vec![&well_above.to_vec(), &just_above.to_vec()], + "descending walk: 90 then 80.5" + ); + assert_proof_round_trips(&drive, &contract, &descending, &flipped); + } +} diff --git a/packages/rs-drive/tests/supporting_files/contract/grades/grades-ranked-contract.json b/packages/rs-drive/tests/supporting_files/contract/grades/grades-ranked-contract.json new file mode 100644 index 00000000000..0a51abcedf7 --- /dev/null +++ b/packages/rs-drive/tests/supporting_files/contract/grades/grades-ranked-contract.json @@ -0,0 +1,45 @@ +{ + "$formatVersion": "0", + "id": "9gradesS5w7Y9R4nDqJk2vHpL3uM6tF1xE8cA2bN7zXq", + "ownerId": "7m6mTfWqkrCnvLLPK3eqxQM2x2RDpYV6dsAyhVKsAEAQ", + "version": 1, + "documentSchemas": { + "grade": { + "type": "object", + "documentsMutable": false, + "canBeDeleted": false, + "indices": [ + { + "name": "byIdentity", + "properties": [ + { "identityId": "asc" } + ], + "averageable": "grade", + "rangeAverageable": true, + "rankedAverageable": true + } + ], + "properties": { + "identityId": { + "type": "array", + "byteArray": true, + "minItems": 32, + "maxItems": 32, + "position": 0, + "contentMediaType": "application/x.dash.dpp.identifier" + }, + "grade": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "position": 1 + } + }, + "required": [ + "identityId", + "grade" + ], + "additionalProperties": false + } + } +} From 11340379242d7cf7d9125aa905cb105266d83658 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 05:00:23 +0700 Subject: [PATCH 03/12] test(drive): pin single-property group_by contract on the having path GROUP BY identityId, class HAVING AVG(grade) > 80 must be rejected, not misserved: ranked axes live on single-property indexes (a ranked flag on a compound index is already rejected at contract-parse time, covered by dpp's test_index_try_from_ranked_on_compound_index_rejected). Pins the drive grammar rejection and that it surfaces through the abci wire path as InvalidArgument naming the single-property rule. Co-Authored-By: Claude Fable 5 --- .../src/query/document_query/v1/tests.rs | 37 +++++++++++++++++++ .../drive_document_having_query/tests.rs | 27 ++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs index 1d5e4850e4b..de57aae029b 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs @@ -3242,6 +3242,43 @@ mod having_range_tests { } } + /// `GROUP BY restaurantId, guests HAVING …` — the routing layer + /// sends any grouped single-clause having down the having path + /// (it owns *where* the request goes, not the grammar), and + /// drive's mode detection rejects the compound grouping: ranked + /// axes live on single-property indexes. + #[test] + fn compound_group_by_is_rejected_on_the_having_path() { + let (platform, state, version) = setup_platform(None, Network::Testnet, None); + let contract = register_restaurants(&platform, version); + + let mut request = having_request( + &contract, + "visit", + select(v1_select::Function::Count, ""), + hc( + having_aggregate::Function::Count, + "", + having_clause::Operator::GreaterThan, + Value::U64(2), + ), + Vec::new(), + Some(10), + false, + ); + request.group_by = vec![GROUP_PROPERTY.to_string(), "guests".to_string()]; + + match ranked_error(&platform, &state, request, version) { + QueryError::Query(QuerySyntaxError::InvalidParameter(message)) => { + assert!( + message.contains("exactly one `group_by` property"), + "the rejection must say the surface is single-property, got: {message}" + ); + } + other => panic!("expected InvalidParameter, got {other:?}"), + } + } + /// `OFFSET` stays ranked-only: the having-range walk has no skip, /// so the post-routing offset gate fires with its long-standing /// message. diff --git a/packages/rs-drive/src/query/drive_document_having_query/tests.rs b/packages/rs-drive/src/query/drive_document_having_query/tests.rs index 4ed267aca9a..159dd725f36 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/tests.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/tests.rs @@ -200,6 +200,33 @@ mod grammar { assert!(result.is_err(), "cross-aggregate having must fail"); } + /// `GROUP BY identityId, class HAVING AVG(grade) > 80` — compound + /// grouping — is rejected: ranked axes live on single-property + /// indexes (a contract declaring a ranked flag on a compound index + /// is already rejected at contract-parse time), so there is no + /// compound grouping for a bound to filter over. + #[test] + fn compound_group_by_is_rejected() { + let result = detect_having_mode_v0( + &SelectProjection::avg("grade"), + &["identityId".to_string(), "class".to_string()], + &[clause( + HavingAggregateFunction::Avg, + "grade", + HavingOperator::GreaterThan, + Value::U64(80), + )], + &[], + &[], + pagination(10), + ); + let error = result.expect_err("compound group_by must fail"); + assert!( + format!("{error}").contains("exactly one `group_by` property"), + "the rejection must say the surface is single-property, got: {error}" + ); + } + #[test] fn multiple_clauses_are_rejected() { let single = clause( From 0f8068e2b5216a47a2e1e5975f2ccda33f25fe89 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 07:38:56 +0700 Subject: [PATCH 04/12] feat(drive): per-prefix ranked aggregates on compound indexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lift the single-property restriction on ranked aggregate index flags: a compound index like [identityId, class] may now declare rankedCountable / rankedSummable / rankedAverageable, with per-prefix semantics — the ranked flags land on the terminal property-name level, one ordered secondary per prefix value, ranking only that prefix's trailing-property groups. No global cross-prefix ordering exists. rs-dpp keeps the one structurally impossible shape rejected, now as a cross-index check where the doctype's full index set is visible (validate_no_ranked_prefix_overlap): a countable/summable index terminating at exactly the compound's leading prefix would demand the NonCounted/NotSummed shell grovedb rejects around indexed trees. The check runs on validating and structural parses alike; drive's INDEXED_INNER_UNWRAPPABLE guard remains the fail-closed backstop. The write path needed no walker changes: the v2 walkers are arity-generic, and the pinned grovedb rev supports creating an indexed primary and populating it in the same batch, so the lazily-created per-prefix terminal trees maintain their secondaries through the ordinary document write path (verified end to end by the new integration suites; stale comments claiming otherwise corrected). Both query surfaces gain equality-prefix routing, v1 equality-only: every leading index property must be pinned by an == where clause (IN and range operators on a prefix are rejected loudly; multi-IN branching can layer on later), group_by names the trailing property. Resolution — covering-index pick plus pin encoding via serialize_value_for_key into prefix path segments — is one shared function per surface (resolve_ranked_query_for_mode / resolve_having_query_for_mode), called by the server executors and the SDK proof helpers, and the shared path builder gained the prefix-value segments, so prover and verifier cannot drift on which subtree a proof is about. abci needs no routing changes: the v2 aggregate-mode helper already routes grouped having / aggregate-ordered shapes regardless of where clauses, and both dispatchers forward where clauses to drive untouched. Protocol version 13 keeps rejecting every new shape before any contract fetch (pinned by wire-level tests). Everything sits in PV14-only modules (meta-schema v3 grammar, DRIVE_ABCI_QUERY_VERSIONS_V3), and PV14 is unreleased, so no version slots were added or renumbered. Co-Authored-By: Claude Fable 5 --- book/src/drive/document-ranked-trees.md | 12 +- .../try_from_schema/common/mod.rs | 85 +++ .../class_methods/try_from_schema/v3/mod.rs | 184 +++++++ .../data_contract/document_type/index/mod.rs | 74 ++- .../src/query/document_query/v1/tests.rs | 195 ++++++- .../v0/tests/ranked_index_e2e_tests.rs | 73 +-- packages/rs-drive/src/fees/op.rs | 18 +- .../drive_dispatcher.rs | 15 +- .../drive_document_having_query/executors.rs | 72 +-- .../query/drive_document_having_query/mod.rs | 103 +++- .../mode_detection.rs | 49 +- .../drive_document_having_query/tests.rs | 504 ++++++++++++++++-- .../drive_dispatcher.rs | 19 +- .../executors/mod.rs | 59 +- .../executors/top_k_no_proof.rs | 1 + .../executors/top_k_proof.rs | 1 + .../index_picker.rs | 227 +++++++- .../query/drive_document_ranked_query/mod.rs | 59 +- .../mode_detection.rs | 133 +++-- .../query/drive_document_ranked_query/path.rs | 87 ++- .../drive_document_ranked_query/tests.rs | 439 +++++++++++++-- .../grades-compound-ranked-contract.json | 52 ++ .../documents/document_having_entries.rs | 15 +- .../documents/document_ranked_entries.rs | 11 +- .../documents/having_proof_helpers.rs | 60 +-- .../documents/ranked_proof_helpers.rs | 68 +-- 26 files changed, 2117 insertions(+), 498 deletions(-) create mode 100644 packages/rs-drive/tests/supporting_files/contract/grades/grades-compound-ranked-contract.json diff --git a/book/src/drive/document-ranked-trees.md b/book/src/drive/document-ranked-trees.md index 07db94bad79..6197c622352 100644 --- a/book/src/drive/document-ranked-trees.md +++ b/book/src/drive/document-ranked-trees.md @@ -91,7 +91,7 @@ One asymmetry is worth knowing when authoring: **the meta-schema demands the lit Two structural rules, both enforced at contract-parse time in rs-dpp: -- **Single-property indexes only.** `ranked aggregates are only supported on single-property indexes in this protocol version`. Two reasons, both relaxable at a future protocol version. First, a compound index whose *prefix* level also terminates an aggregating index would need its ranked terminal tree wrapped in a `NonCounted` / `NotSummed` shell so it contributes zero to the parent's aggregate — and the storage layer structurally rejects any wrapper around an indexed tree, because the wrapper would neutralise the very aggregates the secondaries order by. (Drive's fail-closed guard for this is `INDEXED_INNER_UNWRAPPABLE`.) Second, the ranked query surface has no equality-prefix routing: with more than one property there would be a prefix to fix before ranking, and nothing to express it with. +- **No aggregating index on a compound ranked index's full prefix.** Ranked flags are allowed on compound indexes, with **per-prefix** semantics: a ranked `[identityId, class]` puts the indexed tree at each prefix value's terminal `class` property-name level — one ordered secondary per `identityId`, each ranking only that identity's `class` groups. There is deliberately no global cross-prefix ordering; the query surfaces require every leading property to be pinned by an equality `where` clause. The one shape that stays impossible — and is rejected per document type, where all indexes are visible (`validate_no_ranked_prefix_overlap`) — is a countable/summable index terminating at exactly the compound's leading prefix: its aggregating value trees would demand the `NonCounted` / `NotSummed` shell around the ranked terminal tree, and the storage layer structurally rejects any wrapper around an indexed tree, because the wrapper would neutralise the very aggregates the secondaries order by. (Drive's fail-closed guard behind the parse-time check is `INDEXED_INNER_UNWRAPPABLE`.) Only the exact `n-1` prefix conflicts: an aggregating index at a shorter prefix wraps a plain intermediate tree, and one extending past the ranked terminal lives inside its value trees — both supported. - **Non-unique indexes only.** `ranked aggregates are not supported on unique indexes: each group of a unique index contains at most one document, so there is nothing meaningful to rank`. Contested indexes are covered transitively — a contested index is unique by construction, so it hits the same check rather than needing its own. ### Version Gate @@ -231,9 +231,9 @@ Every ranked read — and, on the prove path, every ranked proof — is issued a / // e.g. b"restaurantId" ``` -The children of that tree are the *groups*: one value tree per distinct value of the last index property, keyed by the raw index-key bytes of that value (for a `string` property, its UTF-8 bytes — e.g. `b"alpha"`). The secondary entries a top-k read returns are keyed by those same group keys. A compound index `[a, b]` inserts ` / ` between the doctype and the terminal `` level — which is exactly the shape ranked indexes don't support yet. +The children of that tree are the *groups*: one value tree per distinct value of the last index property, keyed by the raw index-key bytes of that value (for a `string` property, its UTF-8 bytes — e.g. `b"alpha"`). The secondary entries a top-k read returns are keyed by those same group keys. A compound index `[a, b]` inserts ` / ` between the doctype and the terminal `` level — the value segment comes from the request's equality `where` pin on `a`, encoded with the same `serialize_value_for_key` the write path used to key that prefix's value tree, so the walk lands on **that prefix's own** indexed tree and secondary. -Prover and verifier build this path through the same function, `DriveDocumentRankedQuery::indexed_property_name_tree_path`, which is why they agree on the root hash by construction. +Prover and verifier build this path through the same function, `DriveDocumentRankedQuery::indexed_property_name_tree_path` (with the pinned prefix values encoded by the shared resolver, `resolve_ranked_query_for_mode`), which is why they agree on the root hash by construction. ## Write-Path Cost: The Grove v4 Cleanup Gates @@ -265,7 +265,7 @@ A demoted `CountSumTree` value tree contributes its `(count, sum)` to a ranked i - each group's value tree demotes from `ProvableCountProvableSumTree` to `CountSumTree`; - the `chefId` continuation inside it goes in `Element::NonCounted`, contributing zero to the group's count and sum. -The one place the two changes genuinely collide is the case the single-property rule already forbids: a ranked *terminal* level sitting inside an aggregating value tree would need a wrapper, and an indexed tree can never be wrapped. That is the `INDEXED_INNER_UNWRAPPABLE` guard, and it fails closed. +The one place the two changes genuinely collide is the case the prefix-overlap rule already forbids at contract-parse time: a ranked *terminal* level sitting inside an aggregating value tree would need a wrapper, and an indexed tree can never be wrapped. That is the `INDEXED_INNER_UNWRAPPABLE` guard, and it fails closed. ## Storage-Layout Invariants @@ -337,11 +337,11 @@ Note that the fixture puts each shape on its **own document type**. That's not a | You want | Set | |---|---| -| Top / bottom K groups by document count | `rankedCountable: true` on a single-property, non-unique index that already has `countable` + `rangeCountable: true` | +| Top / bottom K groups by document count | `rankedCountable: true` on a non-unique index that already has `countable` + `rangeCountable: true` | | Top / bottom K groups by sum of a property | `rankedSummable: true` on an index with `summable: ""` + `rangeSummable: true` | | Top / bottom K groups by average of a property | `rankedAverageable: true` on an index with `averageable: ""` + `rangeAverageable: true` (or the count+sum longhand) | | Two rankings on one index (e.g. by count *and* by average) | Both keywords. The tree is a PCPSIT carrying both axes in its TLV; you pay one secondary Merk per axis on every write. | -| A ranking filtered by another property (`top 5 restaurants in London`) | Not available. Ranked indexes are single-property and ranked queries take no `where` clause — the secondary is sorted by aggregate, not by group key, so it cannot express a filtered subset. Model the filter as part of the grouping property, or rank client-side over a range query. | +| A ranking filtered by another property (`top 5 restaurants in London`) | A **compound ranked index** with the filter property leading: `[city, restaurantId]` with the ranked flags. Each city gets its own secondary; the query pins the prefix with an equality `where` (`WHERE city == "London" GROUP BY restaurantId ORDER BY DESC LIMIT 5`). Only equality pins — a range or `IN` on the prefix is rejected, and there is no cross-prefix (global) ordering on a compound ranked index. | | A ranking on a unique or contested index | Not available, and not meaningful: every group holds at most one document. | | Range aggregates without ranking (the 4.0 surface) | Just the `range*` flags. Ranking is strictly additive — adding it never changes what a range query returns. | | Nothing ranking-aware (default) | Don't set any `ranked*` flag. The terminal property-name tree keeps the type its range flags give it. | diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs index 29f77e22a4a..ecf68eec54e 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs @@ -945,12 +945,97 @@ fn parse_indices( .transpose()? .unwrap_or_default(); + // Cross-index structural check for the ranked grammar. Gated on the + // generation constant only to skip the scan where it cannot fire: + // without `admit_ranked` the `ranked*` keywords do not parse at all, + // so no index below can carry a ranking axis. + if ctx.generation.admit_ranked { + validate_no_ranked_prefix_overlap(&indices)?; + } + let index_structure = IndexLevel::try_from_indices(indices.values(), ctx.name, ctx.platform_version)?; Ok((indices, index_structure)) } +/// Rejects the one compound-ranked shape the storage layer cannot lay +/// out: a compound ranked index whose **full leading prefix** also +/// terminates a separate countable and/or summable index. +/// +/// A ranked flag on a compound index `[p1, …, pn]` puts an indexed tree +/// at each prefix's terminal `pn` property-name level — inside the value +/// trees of the `[p1, …, pn-1]` level. When another countable/summable +/// index terminates at exactly that prefix, those value trees are +/// aggregating (`CountTree` / `SumTree` / …), and every continuation +/// subtree inside them must be wrapped in a `NonCounted` / `NotSummed` +/// shell so its contents don't pollute the prefix index's aggregates. +/// grovedb structurally rejects that shell around an indexed tree — the +/// wrapper would neutralize the very aggregates the ranking indexes — +/// so the write path fails closed at document insert. Rejecting the +/// contract here surfaces the conflict at registration instead. +/// +/// Only the **exact** `n-1` prefix conflicts. An aggregating index +/// terminating at a shorter prefix wraps a plain intermediate +/// property-name tree (fine), and one extending *past* the ranked +/// terminal lives inside the indexed tree's value trees, which the +/// storage layer supports (see rs-drive's +/// `ranked_terminator_with_a_compound_continuation_gets_both_treatments`). +/// +/// Property comparison is by name, positionally: the merged index-level +/// tree keys sub-levels by property name in declaration order, so +/// `[a, b]` and `[b, a]` never share a level and cannot conflict. +/// +/// Unconditional (not gated on `full_validation`): the same structural +/// impossibility must reject the contract on every parse path — a +/// contract admitted through a non-validating parse would brick the +/// first document insert under the ranked index. +fn validate_no_ranked_prefix_overlap( + indices: &BTreeMap, +) -> Result<(), ProtocolError> { + for ranked in indices.values() { + let is_ranked = + ranked.ranked_countable || ranked.ranked_summable || ranked.ranked_averageable; + if !is_ranked || ranked.properties.len() < 2 { + continue; + } + let prefix = &ranked.properties[..ranked.properties.len() - 1]; + for other in indices.values() { + if other.name == ranked.name { + continue; + } + let terminates_at_prefix = other.properties.len() == prefix.len() + && other + .properties + .iter() + .zip(prefix.iter()) + .all(|(a, b)| a.name == b.name); + let aggregates = other.countable.is_countable() || other.summable.is_some(); + if terminates_at_prefix && aggregates { + return Err(consensus_or_protocol_data_contract_error( + DataContractError::InvalidContractStructure(format!( + "compound ranked index `{}` conflicts with index `{}`: the ranked \ + index's leading prefix [{}] also terminates a countable/summable \ + index, so the ranked terminal tree would sit inside aggregating \ + value trees and need a NonCounted/NotSummed shell — which the \ + storage layer rejects for indexed trees because the wrapper would \ + neutralize the aggregates the ranking indexes. Drop the ranked \ + flags, or drop the aggregate flags from the prefix index", + ranked.name, + other.name, + prefix + .iter() + .map(|p| p.name.as_str()) + .collect::>() + .join(", "), + )), + )); + } + } + } + Ok(()) +} + /// The per-property half of index validation: an already-indexed system /// property may not be indexed again, a user property must be defined, and an /// indexed property's type must be one the index encoding supports within its diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs index 30ae2b94db9..e30d3e52bb7 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs @@ -982,4 +982,188 @@ mod tests { parse_bound(ranked_bound_schema(string_property(63), range_only)) .expect("a range-averageable index without a ranking axis keeps the generic limit"); } + + // ------------------------------------------------------------------- + // Compound ranked indexes (per-prefix semantics) and the + // prefix-overlap conflict + // ------------------------------------------------------------------- + + /// A doctype with a compound ranked index `[region, restaurantId]` + /// (avg axis on `grade`), plus optional extra single-property + /// indexes to provoke — or fail to provoke — the prefix-overlap + /// conflict. `extra_index` entries are `(name, property, keys)`. + fn compound_ranked_schema(extra_indexes: Vec<(&str, &str, Vec<(&str, Value)>)>) -> Value { + let compound_entry: Vec<(Value, Value)> = vec![ + ( + Value::Text("name".to_string()), + Value::Text("byRegionRestaurant".to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![ + Value::Map(vec![( + Value::Text("region".to_string()), + Value::Text("asc".to_string()), + )]), + Value::Map(vec![( + Value::Text("restaurantId".to_string()), + Value::Text("asc".to_string()), + )]), + ]), + ), + ( + Value::Text("averageable".to_string()), + Value::Text("grade".to_string()), + ), + ( + Value::Text("rangeAverageable".to_string()), + Value::Bool(true), + ), + ( + Value::Text("rankedAverageable".to_string()), + Value::Bool(true), + ), + ]; + let mut indices = vec![Value::Map(compound_entry)]; + for (name, property, keys) in extra_indexes { + let mut entry: Vec<(Value, Value)> = vec![ + ( + Value::Text("name".to_string()), + Value::Text(name.to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![Value::Map(vec![( + Value::Text(property.to_string()), + Value::Text("asc".to_string()), + )])]), + ), + ]; + entry.extend( + keys.into_iter() + .map(|(key, value)| (Value::Text(key.to_string()), value)), + ); + indices.push(Value::Map(entry)); + } + + Value::Map(vec![ + ( + Value::Text("type".to_string()), + Value::Text("object".to_string()), + ), + ( + Value::Text("properties".to_string()), + platform_value!({ + "region": { + "type": "string", + "maxLength": 32, + "position": 0, + }, + "restaurantId": { + "type": "string", + "maxLength": 32, + "position": 1, + }, + "grade": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "position": 2, + }, + }), + ), + ( + Value::Text("required".to_string()), + Value::Array(vec![ + Value::Text("region".to_string()), + Value::Text("restaurantId".to_string()), + Value::Text("grade".to_string()), + ]), + ), + ( + Value::Text("additionalProperties".to_string()), + Value::Bool(false), + ), + (Value::Text("indices".to_string()), Value::Array(indices)), + ]) + } + + /// A compound ranked index is accepted at PV14 with per-prefix + /// semantics — on both the validating and the structural parse + /// paths — and the ranked flags land on the index alongside the + /// range axes they require. + #[test] + fn compound_ranked_index_accepted_at_pv14() { + for full_validation in [true, false] { + let v2 = parse_with(compound_ranked_schema(vec![]), pv14(), full_validation) + .unwrap_or_else(|e| { + panic!( + "a compound ranked index must parse \ + (full_validation: {full_validation}): {e}" + ) + }); + let index = v2 + .indices + .get("byRegionRestaurant") + .expect("index parsed under its name"); + assert!(index.ranked_averageable); + assert!(index.range_countable && index.range_summable); + assert_eq!(index.properties.len(), 2); + assert_eq!(index.properties[1].name, "restaurantId"); + } + } + + /// The one structurally impossible shape: a countable/summable + /// index terminating at the compound ranked index's full leading + /// prefix. The ranked terminal tree would sit inside aggregating + /// value trees and need the NonCounted/NotSummed shell the storage + /// layer rejects for indexed trees — so the contract is refused at + /// parse time, on the validating AND the structural path (a + /// contract smuggled through check_tx would brick document inserts). + #[test] + fn compound_ranked_with_aggregating_prefix_index_rejected() { + let schema = compound_ranked_schema(vec![( + "byRegion", + "region", + vec![("countable", Value::Text("countable".to_string()))], + )]); + for full_validation in [true, false] { + let error = parse_with(schema.clone(), pv14(), full_validation).expect_err( + "an aggregating index on the ranked compound's full prefix must be rejected", + ); + let message = format!("{error:?}"); + assert!( + message.contains("byRegionRestaurant") + && message.contains("byRegion") + && message.contains("NonCounted"), + "the rejection must name both indexes and the structural conflict \ + (full_validation: {full_validation}); got {message}" + ); + } + } + + /// Only the **exact** leading prefix conflicts: an aggregating + /// index over a different property — same arity as the prefix, but + /// not the prefix — coexists with the compound ranked index, as + /// does a plain (non-aggregating) index on the prefix property. + #[test] + fn compound_ranked_with_non_conflicting_indexes_accepted() { + // Countable over the *trailing* property's own single-property + // index: terminates at [restaurantId], not at the ranked + // index's [region] prefix. + let aggregating_elsewhere = compound_ranked_schema(vec![( + "byRestaurant", + "restaurantId", + vec![("countable", Value::Text("countable".to_string()))], + )]); + parse_with(aggregating_elsewhere, pv14(), true) + .expect("an aggregating index off the prefix must not conflict"); + + // A plain index on the prefix property: terminates at [region] + // but carries no aggregates, so its value trees stay normal and + // no wrapper shell is ever needed. + let plain_prefix = compound_ranked_schema(vec![("byRegion", "region", vec![])]); + parse_with(plain_prefix, pv14(), true) + .expect("a non-aggregating index on the prefix must not conflict"); + } } diff --git a/packages/rs-dpp/src/data_contract/document_type/index/mod.rs b/packages/rs-dpp/src/data_contract/document_type/index/mod.rs index edc39f7c934..2f099c93014 100644 --- a/packages/rs-dpp/src/data_contract/document_type/index/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/index/mod.rs @@ -1181,22 +1181,21 @@ impl Index { )); } - // Ranked aggregates are restricted to single-property indexes in - // this protocol version. Two reasons: a compound index whose prefix - // level also terminates an aggregating index would need its ranked - // terminal tree wrapped in a NonCounted/NotSummed shell, which the - // storage layer structurally rejects for indexed trees (the wrapper - // would neutralize the very aggregates the ranking indexes); and the - // ranked query surface deliberately has no equality-prefix routing - // yet. Both are relaxable at a future protocol version. - if (ranked_countable || ranked_summable || ranked_averageable) && index_properties.len() > 1 - { - return Err(DataContractError::InvalidContractStructure( - "ranked aggregates are only supported on single-property \ - indexes in this protocol version" - .to_string(), - )); - } + // Ranked aggregates are allowed on compound indexes, with + // **per-prefix** semantics: the ranked flags land on the index's + // TERMINAL property-name level (see `IndexLevelTypeInfo`), so a + // ranked `[identityId, class]` maintains one ordered secondary per + // `identityId` value — each ordering that identity's `class` groups + // by the group aggregate. There is deliberately no global + // cross-prefix ordering; the query surfaces require every leading + // property to be pinned by an equality `where` clause. + // + // One compound shape stays structurally impossible and is rejected + // per document type (all indexes are needed to see it): a compound + // ranked index whose full leading prefix also terminates a separate + // countable/summable index. That check lives with the document + // type's index collection — see `validate_no_ranked_prefix_overlap` + // in `try_from_schema::common`. // `nullSearchable: false` suppresses the terminal reference for a // document that leaves the indexed property out — but the document @@ -2263,12 +2262,17 @@ mod tests { assert!(!index.ranked_averageable); } - /// Ranked flags on compound indexes are a v1 scope restriction (wrapper - /// conflict under aggregating prefixes + no equality-prefix query - /// routing), rejected at parse time so the limitation is visible at - /// registration rather than at document-insert or query time. + /// Ranked flags on compound indexes are accepted with per-prefix + /// semantics: the ranking axes attach to the terminal property-name + /// level, one ordered secondary per prefix value. The flag-dependency + /// rules (`ranked*` requires the matching `range*`) apply exactly as + /// on single-property indexes; the one structurally impossible shape + /// (a countable/summable index terminating at the compound's full + /// leading prefix) is a cross-index condition rejected where all the + /// document type's indexes are known — see + /// `validate_no_ranked_prefix_overlap` in `try_from_schema::common`. #[test] - fn test_index_try_from_ranked_on_compound_index_rejected() { + fn test_index_try_from_ranked_on_compound_index_accepted() { let mut index_map = ranked_index_map(vec![ ("averageable", Value::Text("score".to_string())), ("rangeAverageable", Value::Bool(true)), @@ -2285,15 +2289,37 @@ mod tests { Value::Text("asc".to_string()), )]), ]); + let index = Index::try_from_value_map(index_map.as_slice(), true) + .expect("ranked flags on a compound index must be accepted"); + assert!(index.ranked_averageable); + assert_eq!(index.properties.len(), 2); + assert_eq!(index.properties[1].name, "score"); + } + + /// The `ranked* requires range*` dependency is enforced on compound + /// indexes exactly as on single-property ones. + #[test] + fn test_index_try_from_compound_ranked_without_range_flags_rejected() { + let mut index_map = ranked_index_map(vec![("rankedCountable", Value::Bool(true))]); + index_map[0].1 = Value::Array(vec![ + Value::Map(vec![( + Value::Text("region".to_string()), + Value::Text("asc".to_string()), + )]), + Value::Map(vec![( + Value::Text("score".to_string()), + Value::Text("asc".to_string()), + )]), + ]); let result = Index::try_from_value_map(index_map.as_slice(), true); assert!( result.is_err(), - "ranked flags on a compound index must be rejected" + "rankedCountable without rangeCountable must be rejected on a compound index too" ); let msg = format!("{:?}", result.unwrap_err()); assert!( - msg.contains("single-property"), - "error must explain the single-property restriction; got {msg}" + msg.contains("rankedCountable") && msg.contains("rangeCountable"), + "error must name both flags; got {msg}" ); } diff --git a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs index de57aae029b..e9cd1f1e343 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs @@ -3070,6 +3070,11 @@ mod having_range_tests { GROUP_PROPERTY, PROTOCOL_VERSION_V13, }; use super::*; + use crate::rpc::core::MockCoreRPCLike; + use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; + use dpp::document::{Document, DocumentV0Setters}; + use dpp::tests::json_document::json_document_to_contract; + use std::collections::BTreeMap; /// The canonical having-range request: one aggregate select, one /// `group_by`, one `having` clause on the selected aggregate, a @@ -3242,11 +3247,14 @@ mod having_range_tests { } } - /// `GROUP BY restaurantId, guests HAVING …` — the routing layer - /// sends any grouped single-clause having down the having path - /// (it owns *where* the request goes, not the grammar), and - /// drive's mode detection rejects the compound grouping: ranked - /// axes live on single-property indexes. + /// `GROUP BY restaurantId, guests HAVING …` — an **unpinned** + /// two-field grouping — is still rejected: the routing layer sends + /// any grouped single-clause having down the having path (it owns + /// *where* the request goes, not the grammar), and drive's mode + /// detection rejects the compound grouping, steering the caller to + /// the served pinned form (`WHERE = X GROUP BY + /// ` — exercised end to end in + /// [`pinned_prefix_having_is_served_end_to_end`]). #[test] fn compound_group_by_is_rejected_on_the_having_path() { let (platform, state, version) = setup_platform(None, Network::Testnet, None); @@ -3271,14 +3279,187 @@ mod having_range_tests { match ranked_error(&platform, &state, request, version) { QueryError::Query(QuerySyntaxError::InvalidParameter(message)) => { assert!( - message.contains("exactly one `group_by` property"), - "the rejection must say the surface is single-property, got: {message}" + message.contains("exactly one `group_by` property") + && message.contains("equality `where` clause"), + "the rejection must steer to the pinned-prefix form, got: {message}" ); } other => panic!("expected InvalidParameter, got {other:?}"), } } + /// Shared with rs-drive's `pinned_prefix` suites — the compound + /// ranked index `[identityId, class]` with the Avg axis on `grade`. + const GRADES_COMPOUND_CONTRACT_PATH: &str = + "../rs-drive/tests/supporting_files/contract/grades/grades-compound-ranked-contract.json"; + + fn register_grades_compound( + platform: &Platform, + platform_version: &PlatformVersion, + ) -> dpp::prelude::DataContract { + let contract = + json_document_to_contract(GRADES_COMPOUND_CONTRACT_PATH, false, platform_version) + .expect("expected to parse the compound ranked grades contract"); + store_data_contract(platform, &contract, platform_version); + contract + } + + fn insert_grade_docs( + platform: &Platform, + contract: &dpp::prelude::DataContract, + first_seed: u64, + rows: &[([u8; 32], &str, i64)], + platform_version: &PlatformVersion, + ) { + let document_type = contract + .document_type_for_name("grade") + .expect("grade doctype exists"); + for (i, (identity, class, grade)) in rows.iter().enumerate() { + let mut document: Document = document_type + .random_document(Some(first_seed + i as u64), platform_version) + .expect("random document"); + let mut properties = BTreeMap::new(); + properties.insert("identityId".to_string(), Value::Identifier(*identity)); + properties.insert("class".to_string(), Value::Text(class.to_string())); + properties.insert("grade".to_string(), Value::I64(*grade)); + document.set_properties(properties); + store_document( + platform, + contract, + document_type, + &document, + platform_version, + ); + } + } + + /// The pinned-prefix form end to end on the wire: `WHERE identityId + /// = X GROUP BY class HAVING AVG(grade) > 80 LIMIT 10` routes to + /// the same having executor, descends to X's terminal `class` tree, + /// and answers with `ResultData.ranked` (`skipped` unset) — with + /// per-prefix isolation visible in the entries. Value-level + /// behaviour (bounds, proofs, tamper) is pinned in rs-drive's + /// `pinned_prefix` suite; this pins the wire encoding and routing. + #[test] + fn pinned_prefix_having_is_served_end_to_end() { + let (platform, state, version) = setup_platform(None, Network::Testnet, None); + let contract = register_grades_compound(&platform, version); + let identity_x = [1u8; 32]; + let identity_y = [2u8; 32]; + insert_grade_docs( + &platform, + &contract, + 20_000, + &[ + (identity_x, "math", 80), + (identity_x, "math", 80), + (identity_x, "english", 80), + (identity_x, "english", 81), + (identity_x, "art", 85), + (identity_x, "art", 95), + // Y's science (avg 95) would qualify for X's bound too + // if the prefixes shared a secondary. + (identity_y, "science", 95), + (identity_y, "science", 95), + ], + version, + ); + + let mut request = having_request( + &contract, + "grade", + select(v1_select::Function::Avg, "grade"), + hc( + having_aggregate::Function::Avg, + "grade", + having_clause::Operator::GreaterThan, + Value::U64(80), + ), + Vec::new(), + Some(10), + false, + ); + request.group_by = vec!["class".to_string()]; + request.where_clauses = vec![wc( + "identityId", + ProtoWhereOperator::Equal, + Value::Bytes(identity_x.to_vec()), + )]; + + let page = ranked_page(&platform, &state, request.clone(), version); + assert_eq!( + page.skipped, None, + "a having-range page must leave the rank-based `skipped` field unset" + ); + assert_eq!( + group_keys(&page.entries), + vec!["english", "art"], + "ascending average order over X's own classes: english (80.5) before art (90); \ + math sits exactly at the threshold and Y's science must not leak in" + ); + + // The proved variant answers with a Proof payload. + request.prove = true; + let result = platform + .query_documents_v1(request, &state, version) + .expect("query call should not error at the transport layer"); + assert!( + result.errors.is_empty(), + "expected no validation errors, got {:?}", + result.errors + ); + match result.data { + Some(GetDocumentsResponseV1 { + result: Some(get_documents_response_v1::Result::Proof(_)), + metadata: Some(_), + }) => {} + other => panic!("expected a Proof result, got {:?}", other), + } + } + + /// The pinned-prefix shape is still a HAVING request, and protocol + /// version 13's query table (v0 helper) has no having path: the + /// same request a v14 node serves is refused with the blanket + /// rejection before any contract fetch. + #[test] + fn pinned_prefix_having_still_rejected_at_protocol_version_13() { + let (platform, state, version) = + setup_platform(None, Network::Testnet, Some(PROTOCOL_VERSION_V13)); + + let request = GetDocumentsRequestV1 { + data_contract_id: vec![0u8; 32], + document_type: "grade".to_string(), + where_clauses: vec![wc( + "identityId", + ProtoWhereOperator::Equal, + Value::Bytes(vec![1u8; 32]), + )], + order_by: Vec::new(), + limit: Some(10), + start: None, + prove: false, + selects: select(v1_select::Function::Avg, "grade"), + group_by: vec!["class".to_string()], + having: vec![hc( + having_aggregate::Function::Avg, + "grade", + having_clause::Operator::GreaterThan, + Value::U64(80), + )], + offset: None, + }; + + match ranked_error(&platform, &state, request, version) { + QueryError::Query(QuerySyntaxError::Unsupported(message)) => { + assert!( + message.contains("HAVING clause") && message.contains("not yet implemented"), + "expected v13's blanket rejection, got: {message}" + ); + } + other => panic!("expected Unsupported, got {other:?}"), + } + } + /// `OFFSET` stays ranked-only: the having-range walk has no skip, /// so the post-routing offset gate fires with its long-standing /// message. diff --git a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/ranked_index_e2e_tests.rs b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/ranked_index_e2e_tests.rs index b043fd6fe4e..1ccbf4ae45b 100644 --- a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/ranked_index_e2e_tests.rs +++ b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/ranked_index_e2e_tests.rs @@ -751,13 +751,11 @@ fn try_build_dish_contract( /// The write path's terminal-level resolver has to pick the indexed variant /// for a **compound** ranked index too — that level lives one step below the -/// doctype tree and is materialized lazily by the document index walker, not -/// at contract registration. -/// -/// Compound ranked indexes are rejected at contract-parse time (rs-dpp, -/// single-property restriction), so the index is built as a struct literal -/// here: this pins that the drive dispatch is already correct for the day a -/// future protocol version relaxes the grammar. +/// doctype tree and is materialized lazily, per prefix value, by the +/// document index walker rather than at contract registration. This pins +/// the resolver's half of that contract; the end-to-end half (documents +/// inserted, per-prefix secondaries read and proved) lives in the query +/// suites' `pinned_prefix` modules. #[test] fn compound_ranked_index_resolves_its_terminal_level_to_an_indexed_tree() { use crate::drive::document::ranked_index_tree_type::property_name_tree_type_and_ranked_axes; @@ -819,23 +817,24 @@ fn compound_ranked_index_resolves_its_terminal_level_to_an_indexed_tree() { assert_eq!(terminal_axes, vec![IndexAxis::Avg]); } -/// Compound ranked indexes are rejected at contract-parse time (rs-dpp's -/// single-property restriction) — a v1 scope choice: a ranked terminal -/// level under an aggregating prefix would need a NonCounted/NotSummed -/// wrapper, which grovedb structurally rejects for indexed trees, and the -/// ranked query surface has no equality-prefix routing yet. (GroveDB's -/// merged PR #657 does support creating and populating an indexed tree in -/// one batch, so lazy terminal creation itself is no longer a blocker — -/// the restriction is relaxable at a future protocol version.) +/// A compound ranked index parses on its own (per-prefix semantics — +/// the terminal level's indexed tree is materialized lazily per prefix +/// by the document walker; grovedb's PR #657 supports creating and +/// populating an indexed tree in one batch). What stays rejected, at +/// contract-parse time, is the one structurally impossible pairing: a +/// countable/summable index terminating at the compound's full leading +/// prefix, whose aggregating value trees would demand the +/// NonCounted/NotSummed wrapper grovedb structurally rejects for +/// indexed trees. /// -/// Storage-level backstop behind this gate: both wrapper dispatchers in -/// `fees/op.rs` fail closed (`DriveError::NotSupported`) for a ranked -/// terminal level inside an aggregating value tree — the frozen v0 -/// diagonal and the v14 zero-contribution matrix alike. The latter -/// matters since the v14 shared-prefix fix: its unwrapped fallback for -/// non-sum children of sum-only parents would otherwise have accepted an -/// indexed continuation, quietly creating a ranked tree that neither the -/// grammar nor the ranked query picker supports. +/// Storage-level backstop behind that parse-time gate: both wrapper +/// dispatchers in `fees/op.rs` fail closed (`DriveError::NotSupported`) +/// for a ranked terminal level inside an aggregating value tree — the +/// frozen v0 diagonal and the v14 zero-contribution matrix alike. The +/// latter matters since the v14 shared-prefix fix: its unwrapped +/// fallback for non-sum children of sum-only parents would otherwise +/// have accepted an indexed continuation, quietly creating a ranked +/// tree the picker never resolves. /// /// Note that a ranked index *sharing* its property with a compound index /// — `[a]` ranked next to `[a, b]` — is a different (and, since v14, @@ -843,16 +842,21 @@ fn compound_ranked_index_resolves_its_terminal_level_to_an_indexed_tree() { /// one, and the continuation hangs *below* it. See /// `ranked_index_ranks_correctly_next_to_a_compound_index_sharing_its_property`. #[test] -fn compound_ranked_index_contract_is_rejected_at_parse_time() { - for standalone_prefix_index in [false, true] { - let error = try_build_dish_contract(standalone_prefix_index) - .expect_err("a compound ranked index must be rejected at contract-parse time"); - let message = error.to_string(); - assert!( - message.contains("single-property"), - "expected the single-property restriction, got: {message}" - ); - } +fn compound_ranked_index_contract_parses_unless_its_prefix_aggregates() { + try_build_dish_contract(false) + .expect("a compound ranked index with no aggregating prefix index must parse"); + + let error = try_build_dish_contract(true).expect_err( + "a countable index terminating at the ranked compound's prefix must be rejected", + ); + let message = error.to_string(); + assert!( + message.contains("byRestaurantCourse") + && message.contains("byRestaurant") + && message.contains("NonCounted"), + "expected the prefix-overlap rejection naming both indexes and the structural \ + conflict, got: {message}" + ); } /// Build a `visit` contract whose single-property `rankedCountable` index @@ -1427,8 +1431,9 @@ fn verified_ranked_avg_page( document_type, contract_id: contract.id().to_buffer(), document_type_name: "review".to_string(), - index: find_ranked_index_for_axis(indexes, GROUP_PROPERTY, RankedAxis::Avg, "grade") + index: find_ranked_index_for_axis(indexes, GROUP_PROPERTY, &[], RankedAxis::Avg, "grade") .expect("the fixture declares rankedAverageable on grade"), + equality_prefix_values: vec![], axis: RankedAxis::Avg, descending: true, k: limit as u16, diff --git a/packages/rs-drive/src/fees/op.rs b/packages/rs-drive/src/fees/op.rs index 736441c13ca..6dcc2fc1cba 100644 --- a/packages/rs-drive/src/fees/op.rs +++ b/packages/rs-drive/src/fees/op.rs @@ -224,18 +224,22 @@ pub enum LowLevelDriveOperation { /// secondary root keys derived from it — is committed. A wrapped indexed tree /// would have nowhere to hang its secondaries. /// -/// Reachable shape: a ranked index whose terminal property-name tree sits -/// inside a value tree that itself aggregates, i.e. a compound ranked index -/// `[a, b]` on a doctype that ALSO declares an aggregating index terminating -/// at `[a]`. Failing closed here is deliberate — the alternative is silently -/// writing a non-indexed tree and having ranked queries return nothing. +/// The shape that would reach this — a ranked index whose terminal +/// property-name tree sits inside a value tree that itself aggregates, i.e. a +/// compound ranked index `[a, b]` on a doctype that ALSO declares an +/// aggregating index terminating at `[a]` — is rejected at contract-parse +/// time (`validate_no_ranked_prefix_overlap` in rs-dpp), so this is the +/// fail-closed backstop behind that check. Failing closed here is deliberate +/// — the alternative is silently writing a non-indexed tree and having +/// ranked queries return nothing. const INDEXED_INNER_UNWRAPPABLE: &str = "an indexed tree cannot be wrapped in NonCounted / NotSummed / NotCountedOrSummed: the \ wrapper suppresses the subtree's contribution to its parent's aggregate, but an indexed \ primary commits its aggregate (and the derived secondary root keys) through that very \ parent element. A ranked index's terminal property-name tree therefore cannot live inside \ - an aggregating value tree — i.e. a ranked compound index [a, b] is unsupported when the \ - same doctype also declares a countable/summable index terminating at [a]."; + an aggregating value tree — i.e. a ranked compound index [a, b] cannot coexist with a \ + countable/summable index terminating at [a]; contracts declaring that pair are rejected \ + at parse time."; impl LowLevelDriveOperation { /// Returns a list of the costs of the Drive operations. diff --git a/packages/rs-drive/src/query/drive_document_having_query/drive_dispatcher.rs b/packages/rs-drive/src/query/drive_document_having_query/drive_dispatcher.rs index 8c5b35da0c6..b484460081b 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/drive_dispatcher.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/drive_dispatcher.rs @@ -21,12 +21,11 @@ use grovedb::TransactionArg; /// wire-decoding + contract lookup — the same construction pattern as /// [`super::super::drive_document_ranked_query::DocumentRankedRequest`]. /// -/// `where_clauses`, `offset` and `start_at` are carried even though a -/// having-range request must leave all of them empty: drive owns the -/// rejection, so the contract is enforced identically no matter which -/// upstream path built the request. See -/// [`super::mode_detection::detect_having_mode_v0`] for why each is -/// refused rather than ignored. +/// `offset` and `start_at` are carried even though a having-range +/// request must leave both empty: drive owns the rejection, so the +/// contract is enforced identically no matter which upstream path built +/// the request. See [`super::mode_detection::detect_having_mode_v0`] +/// for why each is refused rather than ignored. pub struct DocumentHavingRequest<'a> { /// Live contract (already loaded by the handler). pub contract: &'a DataContract, @@ -44,7 +43,9 @@ pub struct DocumentHavingRequest<'a> { /// The `ORDER BY` clauses. Empty (ascending default) or exactly /// one, naming the selected aggregate. pub order_by: &'a [OrderClause], - /// Structured `where` clauses. Must be empty. + /// Structured `where` clauses. Empty for the single-property form; + /// equality pins on the covering compound index's leading + /// properties for the pinned-prefix form. pub where_clauses: &'a [WhereClause], /// Request `limit`. **Required**; `1 ..= MAX_HAVING_LIMIT`. pub limit: Option, diff --git a/packages/rs-drive/src/query/drive_document_having_query/executors.rs b/packages/rs-drive/src/query/drive_document_having_query/executors.rs index 97793ee923f..88347611e90 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/executors.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/executors.rs @@ -1,71 +1,21 @@ -//! Per-mode having-range executors on `impl Drive`, plus the shared -//! mode-to-query resolution. The dispatcher +//! Per-mode having-range executors on `impl Drive`. The dispatcher //! ([`super::drive_dispatcher`]) picks between the two executors on the //! request's `prove` flag. //! -//! Index resolution reuses the ranked surface's covering-index picker -//! ([`find_ranked_index_for_axis`]) — both surfaces read the same -//! indexed tree, and sharing the picker is what guarantees a proof and -//! an unproven read are about the same subtree. +//! Mode-to-query resolution — covering-index pick + equality-pin +//! encoding — is [`resolve_having_query_for_mode`], shared with the +//! SDK's proof helpers: both surfaces and both sides read the same +//! indexed tree, and sharing the resolution is what guarantees a proof +//! and an unproven read are about the same subtree. -use super::super::drive_document_ranked_query::index_picker::find_ranked_index_for_axis; use super::super::drive_document_ranked_query::RankedEntry; -use super::{DocumentHavingMode, DriveDocumentHavingQuery}; +use super::{resolve_having_query_for_mode, DocumentHavingMode}; use crate::drive::Drive; -use crate::error::query::QuerySyntaxError; use crate::error::Error; use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; -use dpp::data_contract::document_type::{DocumentTypeRef, Index}; +use dpp::data_contract::document_type::DocumentTypeRef; use dpp::version::PlatformVersion; use grovedb::TransactionArg; -use std::collections::BTreeMap; - -/// Resolve a validated [`DocumentHavingMode`] against a document type's -/// indexes into the executable [`DriveDocumentHavingQuery`]. -/// -/// `indexes` is threaded in separately for the same lifetime reason as -/// the ranked resolver: the returned query's `&'a Index` must outlive -/// this frame. Callers pass `document_type.indexes()`. -pub(super) fn having_query_for_mode<'a>( - contract_id: [u8; 32], - document_type: DocumentTypeRef<'a>, - document_type_name: String, - indexes: &'a BTreeMap, - mode: &DocumentHavingMode, -) -> Result, Error> { - let axis = mode.bounds.axis(); - let index = find_ranked_index_for_axis( - indexes, - &mode.group_by_property, - axis, - &mode.aggregate_field, - ) - .ok_or_else(|| { - Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty(format!( - "no ranked index covers `group_by = [{}]` on the {:?} axis: a `having` bound \ - is served from that axis's pre-sorted secondary, so the document type needs \ - a single-property index on `{}` declaring `{}`{}", - mode.group_by_property, - axis, - mode.group_by_property, - axis.required_index_keyword(), - if mode.aggregate_field.is_empty() { - String::new() - } else { - format!(" with `summable: \"{}\"`", mode.aggregate_field) - } - ))) - })?; - Ok(DriveDocumentHavingQuery { - document_type, - contract_id, - document_type_name, - index, - bounds: mode.bounds, - descending: mode.descending, - limit: mode.limit, - }) -} impl Drive { /// One page of groups matching a having bound, read without a proof. @@ -79,12 +29,13 @@ impl Drive { platform_version: &PlatformVersion, ) -> Result, Error> { let indexes = document_type.indexes(); - let having_query = having_query_for_mode( + let having_query = resolve_having_query_for_mode( contract_id, document_type, document_type_name, indexes, mode, + platform_version, )?; having_query.execute_range_no_proof(self, transaction, platform_version) } @@ -105,12 +56,13 @@ impl Drive { platform_version: &PlatformVersion, ) -> Result, Error> { let indexes = document_type.indexes(); - let having_query = having_query_for_mode( + let having_query = resolve_having_query_for_mode( contract_id, document_type, document_type_name, indexes, mode, + platform_version, )?; having_query.execute_range_with_proof(self, transaction, platform_version) } diff --git a/packages/rs-drive/src/query/drive_document_having_query/mod.rs b/packages/rs-drive/src/query/drive_document_having_query/mod.rs index c2f5d68b1d4..f3dfee688db 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/mod.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/mod.rs @@ -57,12 +57,24 @@ #[cfg(any(feature = "server", feature = "verify"))] use dpp::data_contract::document_type::{DocumentTypeRef, Index}; +#[cfg(any(feature = "server", feature = "verify"))] +use dpp::platform_value::Value; +#[cfg(any(feature = "server", feature = "verify"))] +use dpp::version::PlatformVersion; +#[cfg(any(feature = "server", feature = "verify"))] +use std::collections::BTreeMap; +#[cfg(any(feature = "server", feature = "verify"))] +use super::drive_document_ranked_query::index_picker::{ + encode_equality_prefix_values, find_ranked_index_for_axis, no_covering_index_message, +}; #[cfg(any(feature = "server", feature = "verify"))] use super::drive_document_ranked_query::{ path::indexed_property_name_tree_path_for_index, RankedAxis, }; #[cfg(any(feature = "server", feature = "verify"))] +use crate::error::query::QuerySyntaxError; +#[cfg(any(feature = "server", feature = "verify"))] use crate::error::Error; #[cfg(any(feature = "server", feature = "verify"))] use grovedb::element::indexed::{encode_avg_sort_key, encode_count_sort_key, encode_sum_sort_key}; @@ -203,7 +215,10 @@ impl AxisRangeBounds { /// /// Produced by [`mode_detection::detect_having_mode`]. Parallels /// [`super::drive_document_ranked_query::DocumentRankedMode`]. -#[derive(Debug, Clone, PartialEq, Eq)] +/// +/// Not `Eq`: the equality pins carry [`Value`]s, whose float variant +/// keeps the type at `PartialEq`. +#[derive(Debug, Clone, PartialEq)] #[cfg(any(feature = "server", feature = "verify"))] pub struct DocumentHavingMode { /// Inclusive bounds on the aggregate, in the axis's own domain. @@ -216,11 +231,16 @@ pub struct DocumentHavingMode { /// `1 ..= MAX_HAVING_LIMIT`, required. pub limit: u16, /// The single `GROUP BY` property; must be the covering ranked - /// index's only property. + /// index's **last** property. pub group_by_property: String, /// The field the aggregate applies to. Empty for `COUNT(*)`; the /// index's `summable` property for `SUM` / `AVG`. pub aggregate_field: String, + /// The equality `where` pins, `(property, value)` per clause — + /// exactly one per leading property of the covering compound index, + /// in request order (the resolver re-orders them into index order + /// when it encodes the path). Empty for the single-property form. + pub equality_pins: Vec<(String, Value)>, } /// A resolved having-range query. Shared by the prover and the verifier — @@ -238,10 +258,17 @@ pub struct DriveDocumentHavingQuery<'a> { pub contract_id: [u8; 32], /// The document type name — a path segment. pub document_type_name: String, - /// The covering ranked index. Single-property by construction; its - /// one property is both the `GROUP BY` property and the last path - /// segment. + /// The covering ranked index. Its **last** property is the `GROUP + /// BY` property and the final path segment; any leading properties + /// are pinned by [`Self::equality_prefix_values`]. pub index: &'a Index, + /// Encoded index-key bytes of each leading index property's pinned + /// value, in index-property order — empty for a single-property + /// index. Part of the prover/verifier agreement exactly as on the + /// ranked surface: the segments feed straight into the shared path + /// builder. Produced by + /// [`super::drive_document_ranked_query::index_picker::encode_equality_prefix_values`]. + pub equality_prefix_values: Vec>, /// Inclusive bounds on the aggregate. Carry the axis; the index must /// declare the matching `ranked_*` flag. pub bounds: AxisRangeBounds, @@ -260,7 +287,8 @@ pub struct DriveDocumentHavingQuery<'a> { #[cfg(any(feature = "server", feature = "verify"))] impl DriveDocumentHavingQuery<'_> { /// Path of the terminal property-name tree the axis secondary hangs - /// off — identical to the ranked surface's path, because both read + /// off — identical to the ranked surface's path (including the + /// pinned-prefix segments of a compound index), because both read /// the same indexed tree. See /// [`DriveDocumentRankedQuery::indexed_property_name_tree_path`](super::drive_document_ranked_query::DriveDocumentRankedQuery::indexed_property_name_tree_path). pub fn indexed_property_name_tree_path(&self) -> Result>, Error> { @@ -268,6 +296,69 @@ impl DriveDocumentHavingQuery<'_> { &self.contract_id, &self.document_type_name, self.index, + &self.equality_prefix_values, ) } } + +/// Resolve a validated [`DocumentHavingMode`] against a document type's +/// indexes into the executable [`DriveDocumentHavingQuery`]: pick the +/// covering index (shared with the ranked surface — both read the same +/// indexed tree), encode the equality pins into prefix-value path +/// segments, and assemble the query. +/// +/// The **one** resolution path for the having surface, mirroring +/// [`super::drive_document_ranked_query::index_picker::resolve_ranked_query_for_mode`]: +/// the server's executors and the SDK's proof helpers both call it, so a +/// proof and an unproven read (and the client's verification) are about +/// the same subtree by construction. +/// +/// `indexes` is threaded in separately for the same lifetime reason as +/// the ranked resolver: the returned query's `&'a Index` must outlive +/// this frame. Callers pass `document_type.indexes()`. +#[cfg(any(feature = "server", feature = "verify"))] +pub fn resolve_having_query_for_mode<'a>( + contract_id: [u8; 32], + document_type: DocumentTypeRef<'a>, + document_type_name: String, + indexes: &'a BTreeMap, + mode: &DocumentHavingMode, + platform_version: &PlatformVersion, +) -> Result, Error> { + let axis = mode.bounds.axis(); + let pin_fields: Vec = mode + .equality_pins + .iter() + .map(|(field, _)| field.clone()) + .collect(); + let index = find_ranked_index_for_axis( + indexes, + &mode.group_by_property, + &pin_fields, + axis, + &mode.aggregate_field, + ) + .ok_or_else(|| { + Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty( + no_covering_index_message( + "having-range", + axis, + &mode.group_by_property, + &mode.equality_pins, + &mode.aggregate_field, + ), + )) + })?; + let equality_prefix_values = + encode_equality_prefix_values(document_type, index, &mode.equality_pins, platform_version)?; + Ok(DriveDocumentHavingQuery { + document_type, + contract_id, + document_type_name, + index, + equality_prefix_values, + bounds: mode.bounds, + descending: mode.descending, + limit: mode.limit, + }) +} diff --git a/packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs b/packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs index 7231065fcd6..c1c536ee9e0 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs @@ -17,7 +17,9 @@ //! surface, so relaxing it later (multi-clause `HAVING`, `IN`, a //! pagination cursor) lands behind a method-version bump. -use super::super::drive_document_ranked_query::mode_detection::ranked_order_key; +use super::super::drive_document_ranked_query::mode_detection::{ + equality_pins_from_where_clauses, ranked_order_key, +}; use super::super::drive_document_ranked_query::{RankedAxis, RankedPaginationInputs}; use super::{AxisRangeBounds, DocumentHavingMode, MAX_HAVING_LIMIT}; use crate::error::query::QuerySyntaxError; @@ -76,8 +78,11 @@ pub fn detect_having_mode( /// SELECT AVG(f) GROUP BY p HAVING AVG(f) [ORDER BY f [ASC|DESC]] LIMIT n /// ``` /// -/// with no `WHERE`, no `OFFSET`, no `START AT` / `START AFTER`, exactly -/// one `GROUP BY` property, exactly one `HAVING` clause whose aggregate +/// with no `OFFSET`, no `START AT` / `START AFTER`, exactly one +/// `GROUP BY` property, `WHERE` clauses (when present) that are +/// equality pins on distinct properties — one per leading property of a +/// covering compound ranked index, selecting which prefix's secondary +/// the bound reads — exactly one `HAVING` clause whose aggregate /// **is the selected aggregate** (same function, same field), an operator /// from the contiguous-range family (`=`, `>`, `>=`, `<`, `<=`, and the /// four `BETWEEN*` variants — `!=` and `IN` describe non-contiguous @@ -120,15 +125,17 @@ pub fn detect_having_mode_v0( ) -> Result { // ---- GROUP BY: exactly one property ---------------------------- // - // Same contract as the ranked surface: ranked indexes are - // single-property, and the sole property is what the secondary's - // group keys are. + // Same contract as the ranked surface: the one property is the + // covering index's LAST property, whose distinct values are the + // secondary's group keys. A compound ranked index filters each + // prefix's groups separately — its leading properties are pinned by + // equality `where` clauses, never grouped over. if group_by.len() != 1 { return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( - "having-range queries require exactly one `group_by` property (the ranked \ - index's only property); got {}. Ranked indexes are single-property — compound \ - ranked indexes are rejected at contract-parse time — so there is no compound \ - grouping to filter over.", + "having-range queries require exactly one `group_by` property (the covering \ + ranked index's trailing property); got {}. A compound ranked index bounds \ + each prefix's groups separately — pin every leading index property with an \ + equality `where` clause and `group_by` the trailing property.", group_by.len() )))); } @@ -288,21 +295,14 @@ pub fn detect_having_mode_v0( } }; - // ---- WHERE: must be absent -------------------------------------- + // ---- WHERE: equality pins on the compound prefix ------------------ // - // Identical rationale to the ranked surface: single-property - // indexes have no equality prefix to narrow, and the secondary is - // ordered by aggregate, not by group key. - if !where_clauses.is_empty() { - return Err(Error::Query( - QuerySyntaxError::InvalidWhereClauseComponents( - "having-range queries do not accept `where` clauses: ranked indexes are \ - single-property, so there is no equality prefix to narrow, and the axis \ - secondary is ordered by aggregate rather than by group key — it cannot \ - bound a filtered subset. Bound the whole index, or add a narrower index.", - ), - )); - } + // Identical contract to the ranked surface: empty for the + // single-property form; for a compound ranked index, one equality + // pin per leading property selects which prefix's secondary the + // bound reads. Shape-only here; the index picker enforces the + // exact-cover rule. + let equality_pins = equality_pins_from_where_clauses(where_clauses)?; // ---- LIMIT: required, 1 ..= MAX_HAVING_LIMIT --------------------- // @@ -368,6 +368,7 @@ pub fn detect_having_mode_v0( limit, group_by_property, aggregate_field, + equality_pins, }) } diff --git a/packages/rs-drive/src/query/drive_document_having_query/tests.rs b/packages/rs-drive/src/query/drive_document_having_query/tests.rs index 159dd725f36..bbc422200f7 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/tests.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/tests.rs @@ -200,11 +200,13 @@ mod grammar { assert!(result.is_err(), "cross-aggregate having must fail"); } - /// `GROUP BY identityId, class HAVING AVG(grade) > 80` — compound - /// grouping — is rejected: ranked axes live on single-property - /// indexes (a contract declaring a ranked flag on a compound index - /// is already rejected at contract-parse time), so there is no - /// compound grouping for a bound to filter over. + /// `GROUP BY identityId, class HAVING AVG(grade) > 80` — an + /// **unpinned** two-field grouping — is still rejected: a compound + /// ranked index bounds each prefix's groups separately, so the + /// served form pins the leading property with an equality `where` + /// and groups over the trailing one (`WHERE identityId = X GROUP BY + /// class …` — exercised end to end in the `pinned_prefix` suite). + /// The rejection must steer the caller to that form. #[test] fn compound_group_by_is_rejected() { let result = detect_having_mode_v0( @@ -220,10 +222,12 @@ mod grammar { &[], pagination(10), ); - let error = result.expect_err("compound group_by must fail"); + let error = result.expect_err("unpinned compound group_by must fail"); + let message = format!("{error}"); assert!( - format!("{error}").contains("exactly one `group_by` property"), - "the rejection must say the surface is single-property, got: {error}" + message.contains("exactly one `group_by` property") + && message.contains("equality `where` clause"), + "the rejection must steer to the pinned-prefix form, got: {error}" ); } @@ -461,7 +465,7 @@ mod execution { use super::clause; use crate::drive::Drive; use crate::error::Error; - use crate::query::drive_document_ranked_query::index_picker::find_ranked_index_for_axis; + use crate::query::drive_document_having_query::resolve_having_query_for_mode; use crate::query::drive_document_ranked_query::{ RankedEntry, RankedEntryValue, RankedPaginationInputs, RANKED_COUNT_ORDER_KEY, }; @@ -708,24 +712,17 @@ mod execution { .get(case.document_type_name) .expect("doctype exists") .indexes(); - let index = find_ranked_index_for_axis( - indexes, - &mode.group_by_property, - mode.bounds.axis(), - &mode.aggregate_field, - ) - .expect("the fixture declares the axis"); - DriveDocumentHavingQuery { - document_type: contract + resolve_having_query_for_mode( + contract.id_ref().to_buffer(), + contract .document_type_for_name(case.document_type_name) .expect("doctype exists"), - contract_id: contract.id_ref().to_buffer(), - document_type_name: case.document_type_name.to_string(), - index, - bounds: mode.bounds, - descending: mode.descending, - limit: mode.limit, - } + case.document_type_name.to_string(), + indexes, + &mode, + platform_version(), + ) + .expect("the fixture declares the axis") } fn grovedb_root_hash(drive: &Drive) -> [u8; 32] { @@ -1152,7 +1149,7 @@ mod identifier_group_keys { use super::super::DriveDocumentHavingQuery; use super::clause; use crate::drive::Drive; - use crate::query::drive_document_ranked_query::index_picker::find_ranked_index_for_axis; + use crate::query::drive_document_having_query::resolve_having_query_for_mode; use crate::query::drive_document_ranked_query::{ RankedEntry, RankedEntryValue, RankedPaginationInputs, }; @@ -1309,24 +1306,17 @@ mod identifier_group_keys { .get(DOCUMENT_TYPE) .expect("grade doctype exists") .indexes(); - let index = find_ranked_index_for_axis( - indexes, - &mode.group_by_property, - mode.bounds.axis(), - &mode.aggregate_field, - ) - .expect("the fixture declares the avg axis"); - DriveDocumentHavingQuery { - document_type: contract + resolve_having_query_for_mode( + contract.id_ref().to_buffer(), + contract .document_type_for_name(DOCUMENT_TYPE) .expect("grade doctype exists"), - contract_id: contract.id_ref().to_buffer(), - document_type_name: DOCUMENT_TYPE.to_string(), - index, - bounds: mode.bounds, - descending: mode.descending, - limit: mode.limit, - } + DOCUMENT_TYPE.to_string(), + indexes, + &mode, + platform_version(), + ) + .expect("the fixture declares the avg axis") } fn assert_proof_round_trips( @@ -1423,3 +1413,433 @@ mod identifier_group_keys { assert_proof_round_trips(&drive, &contract, &descending, &flipped); } } + +mod pinned_prefix { + //! `SELECT AVG(grade) FROM grades WHERE identityId = X GROUP BY + //! class HAVING AVG(grade) > 80` — the compound ranked index + //! `[identityId, class]` with its leading property pinned by an + //! equality `where` clause. Per-prefix semantics: each identity's + //! terminal `class` property-name tree is its own indexed tree, so + //! the bound reads (and proves) only the pinned identity's class + //! groups. Documents are inserted through the real write path, so + //! these tests also pin that the document walkers create and + //! maintain the per-prefix secondaries for compound ranked indexes. + + use super::super::drive_dispatcher::{DocumentHavingRequest, DocumentHavingResponse}; + use super::super::mode_detection::detect_having_mode; + use super::super::DriveDocumentHavingQuery; + use super::clause; + use crate::drive::Drive; + use crate::error::query::QuerySyntaxError; + use crate::error::Error; + use crate::query::drive_document_having_query::resolve_having_query_for_mode; + use crate::query::drive_document_ranked_query::{ + RankedEntry, RankedEntryValue, RankedPaginationInputs, + }; + use crate::query::having::{HavingAggregateFunction, HavingOperator}; + use crate::query::projection::SelectProjection; + use crate::query::{OrderClause, WhereClause, WhereOperator}; + use crate::util::object_size_info::DocumentInfo::DocumentRefInfo; + use crate::util::object_size_info::{DocumentAndContractInfo, OwnedDocumentInfo}; + use crate::util::storage_flags::StorageFlags; + use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; + use dpp::block::block_info::BlockInfo; + use dpp::data_contract::accessors::v0::DataContractV0Getters; + use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; + use dpp::data_contract::document_type::random_document::CreateRandomDocument; + use dpp::document::{Document, DocumentV0Setters}; + use dpp::platform_value::Value; + use dpp::prelude::DataContract; + use dpp::tests::json_document::json_document_to_contract; + use dpp::version::PlatformVersion; + use grovedb::element::indexed::compute_avg_fixed_point; + use std::collections::BTreeMap; + + const PREFIX_PROPERTY: &str = "identityId"; + const GROUP_PROPERTY: &str = "class"; + const DOCUMENT_TYPE: &str = "grade"; + + fn platform_version() -> &'static PlatformVersion { + PlatformVersion::latest() + } + + fn setup_grades_compound_ranked() -> (Drive, DataContract) { + let drive = setup_drive_with_initial_state_structure(None); + let pv = platform_version(); + let contract = json_document_to_contract( + "tests/supporting_files/contract/grades/grades-compound-ranked-contract.json", + false, + pv, + ) + .expect("expected to parse the compound ranked grades contract"); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + pv, + ) + .expect("expected to apply the compound ranked grades contract"); + (drive, contract) + } + + fn insert_grades( + drive: &Drive, + contract: &DataContract, + first_seed: u64, + rows: &[([u8; 32], &str, i64)], + ) { + let pv = platform_version(); + let document_type = contract + .document_type_for_name(DOCUMENT_TYPE) + .expect("grade doctype exists"); + for (i, (identity, class, grade)) in rows.iter().enumerate() { + let mut doc: Document = document_type + .random_document(Some(first_seed + i as u64), pv) + .expect("random document"); + let mut props = BTreeMap::new(); + props.insert(PREFIX_PROPERTY.to_string(), Value::Identifier(*identity)); + props.insert(GROUP_PROPERTY.to_string(), Value::Text(class.to_string())); + props.insert("grade".to_string(), Value::I64(*grade)); + doc.set_properties(props); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo((&doc, None)), + owner_id: None, + }, + contract, + document_type, + }, + false, + BlockInfo::default(), + true, + None, + pv, + None, + ) + .expect("expected to insert a grade document"); + } + } + + fn pin(identity: [u8; 32]) -> Vec { + vec![WhereClause { + field: PREFIX_PROPERTY.to_string(), + operator: WhereOperator::Equal, + value: Value::Identifier(identity), + }] + } + + fn run( + drive: &Drive, + contract: &DataContract, + where_clauses: &[WhereClause], + order_by: &[OrderClause], + prove: bool, + ) -> Result { + let group_by = vec![GROUP_PROPERTY.to_string()]; + let having = vec![clause( + HavingAggregateFunction::Avg, + "grade", + HavingOperator::GreaterThan, + Value::U64(80), + )]; + drive.execute_document_having_request( + DocumentHavingRequest { + contract, + document_type: contract + .document_type_for_name(DOCUMENT_TYPE) + .expect("grade doctype exists"), + group_by: &group_by, + select: SelectProjection::avg("grade"), + having: &having, + order_by, + where_clauses, + limit: Some(10), + offset: None, + has_start_at: false, + prove, + }, + None, + platform_version(), + ) + } + + fn entries_of(response: DocumentHavingResponse) -> Vec { + match response { + DocumentHavingResponse::Entries(entries) => entries, + DocumentHavingResponse::Proof(_) => panic!("expected entries, got a proof"), + } + } + + fn client_side_query<'a>( + contract: &'a DataContract, + where_clauses: &[WhereClause], + order_by: &[OrderClause], + ) -> DriveDocumentHavingQuery<'a> { + let group_by = vec![GROUP_PROPERTY.to_string()]; + let having = vec![clause( + HavingAggregateFunction::Avg, + "grade", + HavingOperator::GreaterThan, + Value::U64(80), + )]; + let mode = detect_having_mode( + &SelectProjection::avg("grade"), + &group_by, + &having, + order_by, + where_clauses, + RankedPaginationInputs { + limit: Some(10), + offset: None, + has_start_at: false, + }, + platform_version(), + ) + .expect("the case is well-formed"); + let indexes = contract + .document_types() + .get(DOCUMENT_TYPE) + .expect("grade doctype exists") + .indexes(); + resolve_having_query_for_mode( + contract.id_ref().to_buffer(), + contract + .document_type_for_name(DOCUMENT_TYPE) + .expect("grade doctype exists"), + DOCUMENT_TYPE.to_string(), + indexes, + &mode, + platform_version(), + ) + .expect("the fixture's compound index covers the pinned request") + } + + fn assert_proof_round_trips( + drive: &Drive, + contract: &DataContract, + where_clauses: &[WhereClause], + order_by: &[OrderClause], + expected: &[RankedEntry], + ) { + let proof = match run(drive, contract, where_clauses, order_by, true) + .expect("prove must succeed") + { + DocumentHavingResponse::Proof(proof) => proof, + DocumentHavingResponse::Entries(_) => panic!("expected a proof, got entries"), + }; + let (root_hash, verified) = client_side_query(contract, where_clauses, order_by) + .verify_having_range_proof(&proof, platform_version()) + .expect("the proof must verify"); + assert_eq!( + verified, expected, + "verified entries must equal what the unproven read returned" + ); + assert_eq!( + root_hash, + drive + .grove + .root_hash(None, &platform_version().drive.grove_version) + .unwrap() + .expect("root hash must be readable"), + "the proof must reconstruct the live grovedb root hash" + ); + } + + const IDENTITY_X: [u8; 32] = [1u8; 32]; + const IDENTITY_Y: [u8; 32] = [2u8; 32]; + + /// The shared dataset: two identities whose class groups overlap by + /// name, so cross-prefix leakage is visible by construction — + /// `math` fails X's bound (avg 80, strict `>`) but passes Y's (avg + /// 92.5), and `science` exists only under Y. + fn insert_two_identities(drive: &Drive, contract: &DataContract) { + insert_grades( + drive, + contract, + 1000, + &[ + // Identity X: math avg 80 (at threshold, excluded under >), + // english avg 80.5 (fractional, just above), art avg 90, + // history avg 70. + (IDENTITY_X, "math", 80), + (IDENTITY_X, "math", 80), + (IDENTITY_X, "english", 80), + (IDENTITY_X, "english", 81), + (IDENTITY_X, "art", 85), + (IDENTITY_X, "art", 95), + (IDENTITY_X, "history", 60), + (IDENTITY_X, "history", 80), + // Identity Y: math avg 92.5, science avg 95 — both would + // qualify for X's bound too if prefixes leaked. + (IDENTITY_Y, "math", 90), + (IDENTITY_Y, "math", 95), + (IDENTITY_Y, "science", 95), + (IDENTITY_Y, "science", 95), + ], + ); + } + + /// Exact-threshold exclusion, fractional inclusion, byte-exact + /// string group keys, both walk directions, proof round-trips — + /// and isolation: identity Y's qualifying classes never appear in + /// X's result, even where the class *name* collides. + #[test] + fn avg_threshold_over_pinned_prefix_reads_and_proves() { + let (drive, contract) = setup_grades_compound_ranked(); + insert_two_identities(&drive, &contract); + + // Pinned to X, ascending: english (80.5) then art (90). math sits + // exactly at the threshold and stays out under `>`; Y's math + // (92.5) and science (95) must not leak in. + let x_pin = pin(IDENTITY_X); + let entries = + entries_of(run(&drive, &contract, &x_pin, &[], false).expect("read succeeds")); + assert_eq!( + entries, + vec![ + RankedEntry { + key: b"english".to_vec(), + value: RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(161, 2)), + }, + RankedEntry { + key: b"art".to_vec(), + value: RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(180, 2)), + }, + ], + "ascending walk over X's classes: 80.5 then 90, keyed by raw utf-8 class bytes" + ); + assert_proof_round_trips(&drive, &contract, &x_pin, &[], &entries); + + // Same pin, descending: art then english, with its own proof. + let descending = vec![OrderClause { + field: "grade".to_string(), + ascending: false, + }]; + let flipped = + entries_of(run(&drive, &contract, &x_pin, &descending, false).expect("read succeeds")); + assert_eq!( + flipped.iter().map(|e| &e.key).collect::>(), + vec![&b"art".to_vec(), &b"english".to_vec()], + "descending walk: 90 then 80.5" + ); + assert_proof_round_trips(&drive, &contract, &x_pin, &descending, &flipped); + + // Pinned to Y: math qualifies *here* (92.5) even though the same + // class name failed X's bound — the two prefixes rank separately. + let y_pin = pin(IDENTITY_Y); + let y_entries = + entries_of(run(&drive, &contract, &y_pin, &[], false).expect("read succeeds")); + assert_eq!( + y_entries, + vec![ + RankedEntry { + key: b"math".to_vec(), + value: RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(185, 2)), + }, + RankedEntry { + key: b"science".to_vec(), + value: RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(190, 2)), + }, + ], + "Y's own classes: math 92.5 then science 95" + ); + assert_proof_round_trips(&drive, &contract, &y_pin, &[], &y_entries); + } + + /// An unpinned prefix cannot be served: with `group_by = class` and + /// no `where`, no single-property ranked index on `class` exists, + /// and the compound index's per-prefix secondaries have no global + /// ordering to read. The rejection names the missing coverage. + #[test] + fn unpinned_prefix_is_rejected() { + let (drive, contract) = setup_grades_compound_ranked(); + insert_two_identities(&drive, &contract); + + let error = run(&drive, &contract, &[], &[], false) + .expect_err("an unpinned compound prefix must not resolve"); + match error { + Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty(message)) => { + assert!( + message.contains("no ranked index covers") + && message.contains("single-property index on `class`"), + "the rejection must explain the missing coverage, got: {message}" + ); + } + other => panic!("expected a no-covering-index rejection, got {other:?}"), + } + } + + /// `IN` on the prefix is rejected at detection with the + /// not-yet-supported message (v1 pins are equality-only), and a pin + /// on a property that is not the index's leading property fails + /// resolution. + #[test] + fn in_prefix_and_wrong_pins_are_rejected() { + let (drive, contract) = setup_grades_compound_ranked(); + insert_two_identities(&drive, &contract); + + let in_clause = vec![WhereClause { + field: PREFIX_PROPERTY.to_string(), + operator: WhereOperator::In, + value: Value::Array(vec![ + Value::Identifier(IDENTITY_X), + Value::Identifier(IDENTITY_Y), + ]), + }]; + let error = run(&drive, &contract, &in_clause, &[], false) + .expect_err("IN prefixes are not yet supported"); + match error { + Error::Query(QuerySyntaxError::Unsupported(message)) => { + assert!( + message.contains("IN") && message.contains("not yet supported"), + "the IN rejection must say it is a not-yet capability, got: {message}" + ); + } + other => panic!("expected Unsupported for an IN prefix, got {other:?}"), + } + + // A pin on the wrong property: `grade` is not the index's + // leading property, so nothing covers [grade, class]. + let wrong_pin = vec![WhereClause { + field: "grade".to_string(), + operator: WhereOperator::Equal, + value: Value::I64(80), + }]; + let error = run(&drive, &contract, &wrong_pin, &[], false) + .expect_err("a pin on a non-leading property must not resolve"); + assert!( + matches!( + error, + Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty(_)) + ), + "expected a no-covering-index rejection, got {error:?}" + ); + } + + /// A pin on an identity that never inserted a document addresses a + /// prefix value tree that does not exist. The read and the prover + /// both surface an error rather than fabricating an empty page — + /// same contract as the empty-secondary limitation on the + /// single-property surface (the abci layer maps these to a + /// client-visible rejection). + #[test] + fn unknown_prefix_value_errors_rather_than_fabricating_an_empty_page() { + let (drive, contract) = setup_grades_compound_ranked(); + insert_two_identities(&drive, &contract); + + let unknown = pin([9u8; 32]); + assert!( + run(&drive, &contract, &unknown, &[], false).is_err(), + "reading a never-written prefix value tree must error" + ); + assert!( + run(&drive, &contract, &unknown, &[], true).is_err(), + "proving a never-written prefix value tree must error" + ); + } +} diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/drive_dispatcher.rs b/packages/rs-drive/src/query/drive_document_ranked_query/drive_dispatcher.rs index 1783abbc925..23bd902fe6e 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/drive_dispatcher.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/drive_dispatcher.rs @@ -37,19 +37,18 @@ use grovedb::TransactionArg; /// canonicalized or rewritten the way count's where-clauses are, so /// taking ownership would just force the handler into a clone. /// -/// `where_clauses`, `having` and `start_at` are carried even though a -/// ranked request must leave all of them empty: drive owns the -/// rejection, so the contract is enforced identically no matter which -/// upstream path built the request. See -/// [`super::mode_detection::detect_ranked_mode_v0`] for why each is -/// refused rather than ignored. +/// `having` and `start_at` are carried even though a ranked request +/// must leave both empty: drive owns the rejection, so the contract is +/// enforced identically no matter which upstream path built the +/// request. See [`super::mode_detection::detect_ranked_mode_v0`] for +/// why each is refused rather than ignored. pub struct DocumentRankedRequest<'a> { /// Live contract (already loaded by the handler). pub contract: &'a DataContract, /// Resolved document type within `contract`. pub document_type: DocumentTypeRef<'a>, - /// The single `GROUP BY` property. Must be the ranked index's only - /// property. + /// The single `GROUP BY` property. Must be the covering ranked + /// index's trailing property. pub group_by: &'a [String], /// The projection being ranked: `COUNT(*)`, `SUM(field)` or /// `AVG(field)`. @@ -61,7 +60,9 @@ pub struct DocumentRankedRequest<'a> { /// aggregate (`$count` for `COUNT(*)`, otherwise the select's /// field); its direction is the ranking direction. pub order_by: &'a [OrderClause], - /// Structured `where` clauses. Must be empty. + /// Structured `where` clauses. Empty for the single-property form; + /// equality pins on the covering compound index's leading + /// properties for the pinned-prefix form. pub where_clauses: &'a [WhereClause], /// Request `limit` — the ranking's `k`. **Required**; there is no /// server default a verifying client could reproduce. diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/executors/mod.rs b/packages/rs-drive/src/query/drive_document_ranked_query/executors/mod.rs index b48bebc6560..56f31682687 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/executors/mod.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/executors/mod.rs @@ -15,55 +15,10 @@ pub mod top_k_no_proof; pub mod top_k_proof; -use super::index_picker::find_ranked_index_for_mode; -use super::{DocumentRankedMode, DriveDocumentRankedQuery}; -use crate::error::query::QuerySyntaxError; -use crate::error::Error; -use dpp::data_contract::document_type::{DocumentTypeRef, Index}; -use std::collections::BTreeMap; - -/// Resolve a validated [`DocumentRankedMode`] against a document type's -/// indexes into the executable [`DriveDocumentRankedQuery`]. -/// -/// `indexes` is threaded in separately rather than read off -/// `document_type` here because -/// [`DocumentTypeV0Getters::indexes`] borrows its receiver — taking the -/// map from the caller lets the returned query's `&'a Index` outlive this -/// frame. Callers pass `document_type.indexes()`. -/// -/// The only failure is "no index covers this", which is reported with the -/// exact contract keyword the index is missing so the caller can act on -/// it without reading the schema spec. -pub(super) fn ranked_query_for_mode<'a>( - contract_id: [u8; 32], - document_type: DocumentTypeRef<'a>, - document_type_name: String, - indexes: &'a BTreeMap, - mode: &DocumentRankedMode, -) -> Result, Error> { - let index = find_ranked_index_for_mode(indexes, mode).ok_or_else(|| { - Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty(format!( - "no ranked index covers `group_by = [{}]` on the {:?} axis: the document type \ - needs a single-property index on `{}` declaring `{}`{}", - mode.group_by_property, - mode.axis, - mode.group_by_property, - mode.axis.required_index_keyword(), - if mode.aggregate_field.is_empty() { - String::new() - } else { - format!(" with `summable: \"{}\"`", mode.aggregate_field) - } - ))) - })?; - Ok(DriveDocumentRankedQuery { - document_type, - contract_id, - document_type_name, - index, - axis: mode.axis, - descending: mode.descending, - k: mode.k, - offset: mode.offset, - }) -} +// Resolution — covering-index pick + equality-pin encoding — is shared +// with the SDK's proof helpers through +// [`super::index_picker::resolve_ranked_query_for_mode`]: both sides +// must land on the same index and the same prefix segments, or a client +// would verify a proof about a different subtree than the one an +// unproven read returned. +pub(super) use super::index_picker::resolve_ranked_query_for_mode as ranked_query_for_mode; diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_no_proof.rs b/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_no_proof.rs index 1b7b2b9d200..281fdfe2c2a 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_no_proof.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_no_proof.rs @@ -31,6 +31,7 @@ impl Drive { document_type_name, indexes, mode, + platform_version, )?; ranked_query.execute_top_k_no_proof(self, transaction, platform_version) } diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_proof.rs b/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_proof.rs index 478f498e05d..f276d7c0204 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_proof.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_proof.rs @@ -35,6 +35,7 @@ impl Drive { document_type_name, indexes, mode, + platform_version, )?; ranked_query.execute_top_k_with_proof(self, transaction, platform_version) } diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/index_picker.rs b/packages/rs-drive/src/query/drive_document_ranked_query/index_picker.rs index 251979e2b87..cfaab9be252 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/index_picker.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/index_picker.rs @@ -1,24 +1,35 @@ -//! Covering-index picker for the ranked query. +//! Covering-index picker for the ranked query, plus the shared +//! prefix-value encoding. //! -//! Pure function on the document type's index map plus the -//! `(group property, axis, aggregate field)` triple +//! Pure functions on the document type's index map plus the +//! `(group property, equality pins, axis, aggregate field)` tuple //! [`super::mode_detection`] resolved. No Drive, no proof — the server -//! and the SDK verifier both call it so they land on the same index (and -//! therefore the same grove path) for the same request. +//! and the SDK verifier both call these so they land on the same index +//! (and therefore the same grove path) for the same request. -use super::{DocumentRankedMode, RankedAxis}; -use dpp::data_contract::document_type::Index; +use super::{DocumentRankedMode, DriveDocumentRankedQuery, RankedAxis}; +use crate::error::query::QuerySyntaxError; +use crate::error::Error; +use dpp::data_contract::document_type::methods::DocumentTypeV0Methods; +use dpp::data_contract::document_type::{DocumentTypeRef, Index}; +use dpp::platform_value::Value; +use dpp::version::PlatformVersion; use std::collections::BTreeMap; /// Find the index that can serve `axis` ranking grouped by -/// `group_by_property`, aggregating `aggregate_field`. +/// `group_by_property` with the given equality pins, aggregating +/// `aggregate_field`. /// /// An index qualifies when **all** of: /// -/// - it has exactly one property, and that property is -/// `group_by_property` — the ranked secondary's group keys are the -/// values of an index's *last* property, and rs-dpp only allows -/// single-property ranked indexes, so "last" and "only" coincide; +/// - it has exactly one more property than there are pins, and its +/// **last** property is `group_by_property` — the ranked secondary's +/// group keys are the values of an index's last property; +/// - every **leading** property is pinned: each appears (by name) among +/// `equality_pin_fields`. Lengths matching plus the pins being +/// distinct (enforced upstream by +/// [`super::mode_detection::equality_pins_from_where_clauses`]) makes +/// this set equality, so no pin is left over either; /// - it declares the ranking keyword for `axis` /// ([`RankedAxis::required_index_keyword`]); /// - for [`RankedAxis::Sum`] / [`RankedAxis::Avg`], its `summable` @@ -27,17 +38,25 @@ use std::collections::BTreeMap; /// group's count), so summing a *different* field than the one the /// index accumulates would silently answer about the wrong property. /// +/// With no pins this degenerates to the original single-property rule. +/// A partial pin (some but not all leading properties) matches nothing — +/// the per-prefix secondary lives under one value tree per leading +/// property, so there is no subtree an unpinned prefix could address — +/// and callers turn the `None` into a loud +/// [`crate::error::query::QuerySyntaxError`] naming what is missing. +/// /// Returns `None` when nothing qualifies; callers turn that into /// [`crate::error::query::QuerySyntaxError::WhereClauseOnNonIndexedProperty`] /// with a message naming the missing keyword. /// -/// At most one index can qualify for a given `(group property, axis, -/// field)` triple — rs-dpp rejects two indexes over the same property set -/// on one document type — so "first match wins" is not a tie-break in -/// practice. Should that ever change, the `BTreeMap` iteration order -/// (index name, ascending) keeps the choice deterministic, which is what -/// prover/verifier agreement actually requires: both sides run this same -/// function over the same contract and must land on the same grove path. +/// At most one index can qualify for a given `(group property, pins, +/// axis, field)` tuple — rs-dpp rejects two indexes over the same +/// property set on one document type — so "first match wins" is not a +/// tie-break in practice. Should that ever change, the `BTreeMap` +/// iteration order (index name, ascending) keeps the choice +/// deterministic, which is what prover/verifier agreement actually +/// requires: both sides run this same function over the same contract +/// and must land on the same grove path. /// /// Note that axis availability is decided from the index's `ranked_*` /// flags, **not** from the element variant the write path laid down: a @@ -47,12 +66,23 @@ use std::collections::BTreeMap; pub fn find_ranked_index_for_axis<'b>( indexes: &'b BTreeMap, group_by_property: &str, + equality_pin_fields: &[String], axis: RankedAxis, aggregate_field: &str, ) -> Option<&'b Index> { indexes.values().find(|index| { - // Single-property, and that property is the grouping property. - if index.properties.len() != 1 || index.properties[0].name != group_by_property { + // Trailing property is the grouping property; every leading + // property is pinned exactly once (length equality + distinct + // pins ⇒ set equality). + let Some((terminal, leading)) = index.properties.split_last() else { + return false; + }; + if terminal.name != group_by_property + || leading.len() != equality_pin_fields.len() + || !leading + .iter() + .all(|property| equality_pin_fields.iter().any(|f| f == &property.name)) + { return false; } match axis { @@ -73,10 +103,165 @@ pub fn find_ranked_index_for_mode<'b>( indexes: &'b BTreeMap, mode: &DocumentRankedMode, ) -> Option<&'b Index> { + let pin_fields: Vec = mode + .equality_pins + .iter() + .map(|(field, _)| field.clone()) + .collect(); find_ranked_index_for_axis( indexes, &mode.group_by_property, + &pin_fields, mode.axis, &mode.aggregate_field, ) } + +/// Resolve a validated [`DocumentRankedMode`] against a document type's +/// indexes into the executable [`DriveDocumentRankedQuery`]: pick the +/// covering index, encode the equality pins into prefix-value path +/// segments, and assemble the query. +/// +/// This is the **one** resolution path — the server's executors and the +/// SDK's proof helpers both call it, which is what guarantees a proof +/// and an unproven read (and the client's verification) are about the +/// same subtree. +/// +/// `indexes` is threaded in separately rather than read off +/// `document_type` here because +/// [`DocumentTypeV0Getters::indexes`](dpp::data_contract::document_type::accessors::DocumentTypeV0Getters::indexes) +/// borrows its receiver — taking the map from the caller lets the +/// returned query's `&'a Index` outlive this frame. Callers pass +/// `document_type.indexes()`. +/// +/// The main failure is "no index covers this", reported with the exact +/// contract keyword (and, for pinned requests, the exact index shape) +/// the request needs, so the caller can act on it without reading the +/// schema spec. +pub fn resolve_ranked_query_for_mode<'a>( + contract_id: [u8; 32], + document_type: DocumentTypeRef<'a>, + document_type_name: String, + indexes: &'a BTreeMap, + mode: &DocumentRankedMode, + platform_version: &PlatformVersion, +) -> Result, Error> { + let index = find_ranked_index_for_mode(indexes, mode).ok_or_else(|| { + Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty( + no_covering_index_message( + "ranked", + mode.axis, + &mode.group_by_property, + &mode.equality_pins, + &mode.aggregate_field, + ), + )) + })?; + let equality_prefix_values = + encode_equality_prefix_values(document_type, index, &mode.equality_pins, platform_version)?; + Ok(DriveDocumentRankedQuery { + document_type, + contract_id, + document_type_name, + index, + equality_prefix_values, + axis: mode.axis, + descending: mode.descending, + k: mode.k, + offset: mode.offset, + }) +} + +/// The "no index covers this request" rejection text, shared by the +/// ranked and having-range resolutions (and the SDK's mirrors of them) +/// so a rejected request reads identically everywhere. Names the exact +/// index the request needs: property list (pins first, in request +/// order, then the grouping property), ranking keyword, and `summable` +/// field where applicable. +pub fn no_covering_index_message( + surface: &str, + axis: RankedAxis, + group_by_property: &str, + equality_pins: &[(String, Value)], + aggregate_field: &str, +) -> String { + let pin_fields = || { + equality_pins + .iter() + .map(|(field, _)| field.as_str()) + .collect::>() + .join(", ") + }; + let index_shape = if equality_pins.is_empty() { + format!("a single-property index on `{group_by_property}`") + } else { + format!( + "a compound index on [{}, {group_by_property}] (every leading property pinned \ + by an equality `where` clause, the trailing property grouped over)", + pin_fields() + ) + }; + format!( + "no ranked index covers `group_by = [{group_by_property}]`{} on the {axis:?} axis \ + for this {surface} query: the document type needs {index_shape} declaring `{}`{}", + if equality_pins.is_empty() { + String::new() + } else { + format!(" with equality pins on [{}]", pin_fields()) + }, + axis.required_index_keyword(), + if aggregate_field.is_empty() { + String::new() + } else { + format!(" with `summable: \"{aggregate_field}\"`") + } + ) +} + +/// Encode the equality pins into the grove path's prefix-value +/// segments: for each **leading** property of `index`, in index order, +/// the pinned value's index-key bytes +/// (`DocumentType::serialize_value_for_key` — the same encoding the +/// write path used to key that prefix's value tree). +/// +/// This is part of the prover/verifier agreement: server executors and +/// the SDK's proof helpers both come through here, so a pinned value +/// can only ever name one subtree, identically on both sides. +/// +/// `index` must have been picked by [`find_ranked_index_for_axis`] +/// against these same pins — every leading property is then guaranteed +/// a pin. A value the property's type cannot encode (a string against +/// an integer property, an out-of-range integer) is a caller error, +/// reported as a query-syntax rejection naming the property. +pub fn encode_equality_prefix_values( + document_type: DocumentTypeRef, + index: &Index, + equality_pins: &[(String, Value)], + platform_version: &PlatformVersion, +) -> Result>, Error> { + let leading = &index.properties[..index.properties.len().saturating_sub(1)]; + leading + .iter() + .map(|property| { + let (_, value) = equality_pins + .iter() + .find(|(field, _)| field == &property.name) + .ok_or_else(|| { + Error::Query(QuerySyntaxError::InvalidWhereClauseComponents( + "internal resolution mismatch: the picked compound ranked index has \ + a leading property with no equality pin — the index picker and the \ + prefix encoder disagreed on the pins", + )) + })?; + document_type + .serialize_value_for_key(&property.name, value, platform_version) + .map_err(|e| { + Error::Query(QuerySyntaxError::InvalidParameter(format!( + "the equality pin on `{}` does not encode as that property's \ + index key: {e}", + property.name + ))) + }) + }) + .collect() +} diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/mod.rs b/packages/rs-drive/src/query/drive_document_ranked_query/mod.rs index 70cc2dd2339..fed2b5dffbd 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/mod.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/mod.rs @@ -38,15 +38,20 @@ //! secondary Merk, keyed by `(sort_key ‖ group_key)`. Three consequences //! shape the API: //! -//! 1. **No `where` clauses.** Ranked indexes are single-property only -//! (compound ones are rejected at contract-parse time in rs-dpp — -//! their terminal level is created lazily by the same batch that -//! populates it, which grovedb rejects for indexed trees). With one -//! property there is no equality prefix to narrow, and a `where` on -//! the ranked property itself would ask for a *filtered* ranking, -//! which the secondary cannot express — it is sorted by aggregate, -//! not by group key. Non-empty `where` is therefore rejected rather -//! than silently ignored. +//! 1. **`where` clauses are equality pins on a compound prefix — or +//! absent.** A single-property ranked index has no prefix to narrow, +//! so its requests carry no `where`. A compound ranked index +//! `[p1, …, pn]` maintains one secondary **per prefix value** +//! (per-prefix semantics: each terminal `pn` property-name tree, +//! inside the `[p1, …, pn-1]` value trees, is its own indexed tree +//! — grovedb creates and populates it in the same document batch), +//! so a request must pin every leading property with an equality +//! clause to name which prefix's secondary the walk reads. A `where` +//! on the grouped (terminal) property itself would ask for a +//! *filtered* ranking, which no secondary can express — it is sorted +//! by aggregate, not by group key — and is rejected rather than +//! silently ignored, as is any non-equality prefix clause (`IN` +//! included: one walk per element is a future multi-`IN` capability). //! 2. **`limit` is mandatory, `offset` is free, `start_at` is refused.** //! `limit` is the `k` of the walk and the ranked surface has no //! server default for it, so it must be supplied. `offset` is the @@ -63,6 +68,8 @@ #[cfg(any(feature = "server", feature = "verify"))] use dpp::data_contract::document_type::{DocumentTypeRef, Index}; +#[cfg(any(feature = "server", feature = "verify"))] +use dpp::platform_value::Value; /// The fixed-point scale grovedb's Avg axis sorts by: /// `avg_fixed_point = floor(sum * RANKED_AVG_SCALE / count)` with @@ -268,10 +275,19 @@ pub struct DriveDocumentRankedQuery<'a> { pub contract_id: [u8; 32], /// The document type name — a path segment. pub document_type_name: String, - /// The covering ranked index. Single-property by construction; its - /// one property is both the `GROUP BY` property and the last path - /// segment. + /// The covering ranked index. Its **last** property is the `GROUP + /// BY` property and the final path segment; any leading properties + /// are pinned by [`Self::equality_prefix_values`]. pub index: &'a Index, + /// Encoded index-key bytes of each leading index property's pinned + /// value, in index-property order — empty for a single-property + /// index. Together with `index` these determine the grove path + /// (each leading property contributes a name segment and a value + /// segment), so they are as much a part of the prover/verifier + /// agreement as the path builder itself. Produced by + /// [`index_picker::encode_equality_prefix_values`] from the + /// request's equality `where` pins. + pub equality_prefix_values: Vec>, /// Which aggregate the groups are ranked by. Must be covered by /// `index`'s matching `ranked_*` flag. pub axis: RankedAxis, @@ -371,8 +387,11 @@ pub struct RankedPaginationInputs { /// the versioned classification of a request — but carries data rather /// than being a bare discriminant, because the ranked surface has exactly /// one executor pair (no-proof / proof) and all of its variation is in -/// these five values. -#[derive(Debug, Clone, PartialEq, Eq)] +/// these values. +/// +/// Not `Eq`: the equality pins carry [`Value`]s, whose float variant +/// keeps the type at `PartialEq`. +#[derive(Debug, Clone, PartialEq)] #[cfg(any(feature = "server", feature = "verify"))] pub struct DocumentRankedMode { /// The ranking axis, from the `SELECT` function. @@ -383,11 +402,19 @@ pub struct DocumentRankedMode { pub k: u16, /// Ranks to skip — the `OFFSET`, `0` when unset. pub offset: u32, - /// The single `GROUP BY` property; must be the ranked index's only - /// property. + /// The single `GROUP BY` property; must be the covering ranked + /// index's **last** property. pub group_by_property: String, /// The field the aggregate applies to. Empty for /// [`RankedAxis::Count`] (`COUNT(*)`); the index's `summable` /// property for [`RankedAxis::Sum`] / [`RankedAxis::Avg`]. pub aggregate_field: String, + /// The equality `where` pins, `(property, value)` per clause — + /// exactly one per leading property of the covering compound index, + /// in whatever order the request supplied them (the resolver + /// re-orders them into index-property order when it encodes the + /// path). Empty for the single-property form. Shape-validated only: + /// the index-aware checks (does a compound index exist whose leading + /// properties these pin?) live in [`index_picker`]. + pub equality_pins: Vec<(String, Value)>, } diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection.rs b/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection.rs index f42de99cfe8..a96dd287f5e 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection.rs @@ -28,7 +28,8 @@ use crate::error::query::QuerySyntaxError; use crate::error::Error; use crate::query::having::HavingClause; use crate::query::projection::{SelectFunction, SelectProjection}; -use crate::query::{OrderClause, WhereClause}; +use crate::query::{OrderClause, WhereClause, WhereOperator}; +use dpp::platform_value::Value; use dpp::version::PlatformVersion; /// Versioned entry point. Routes through @@ -85,20 +86,92 @@ pub fn ranked_order_key(select: &SelectProjection) -> &str { } } +/// Translate a request's `where` clauses into equality pins — +/// `(property, value)` pairs, one per clause — for the ranked and +/// having-range surfaces. +/// +/// Both surfaces read a compound index's per-prefix secondary by +/// descending through one prefix value tree per **leading** index +/// property, and only an equality clause names a single value tree to +/// descend into. So the grammar is: every `where` clause must be an +/// equality (`==`), each on a distinct property. `IN` is rejected +/// separately from the other operators because it *will* eventually be +/// serviceable (one branch per element, once multi-`IN` branching lands +/// on the document query surface) — the message says so — while a range +/// operator on a prefix property can never pin a single subtree. +/// +/// Shape-only, like everything in this module: whether the pinned +/// properties are exactly the leading properties of a covering compound +/// index is the index picker's call. +pub fn equality_pins_from_where_clauses( + where_clauses: &[WhereClause], +) -> Result, Error> { + let mut pins: Vec<(String, Value)> = Vec::with_capacity(where_clauses.len()); + for clause in where_clauses { + match clause.operator { + WhereOperator::Equal => {} + WhereOperator::In => { + return Err(Error::Query(QuerySyntaxError::Unsupported(format!( + "`{} IN …` is not yet supported on a ranked / having-range query's \ + prefix properties: each `IN` element names a different prefix value \ + tree, so serving it means one secondary walk per element and a merged \ + result — a future capability layered on multi-`IN` branching. Pin \ + each leading index property with `==`, or issue one request per \ + value.", + clause.field + )))); + } + _ => { + return Err(Error::Query( + QuerySyntaxError::InvalidWhereClauseComponents( + "a ranked / having-range query's `where` clauses must pin the covering \ + compound index's leading properties with `==`: the per-prefix \ + secondary lives under one prefix value tree per leading property, and \ + only an equality names a single value tree to descend into — a range \ + operator cannot pin a prefix", + ), + )); + } + } + if clause.field.is_empty() { + return Err(Error::Query( + QuerySyntaxError::InvalidWhereClauseComponents( + "a ranked / having-range query's `where` clause names an empty property", + ), + )); + } + if pins.iter().any(|(field, _)| field == &clause.field) { + return Err(Error::Query( + QuerySyntaxError::InvalidWhereClauseComponents( + "a ranked / having-range query pins the same property twice: each leading \ + index property takes exactly one equality pin", + ), + )); + } + pins.push((clause.field.clone(), clause.value.clone())); + } + Ok(pins) +} + /// v0 of the ranked request grammar. /// /// Accepts exactly: /// /// ```text -/// SELECT COUNT(*) GROUP BY p ORDER BY $count [ASC|DESC] LIMIT n [OFFSET m] -/// SELECT SUM(f) GROUP BY p ORDER BY f [ASC|DESC] LIMIT n [OFFSET m] -/// SELECT AVG(f) GROUP BY p ORDER BY f [ASC|DESC] LIMIT n [OFFSET m] +/// SELECT COUNT(*) [WHERE q1 = v1 [AND …]] GROUP BY p ORDER BY $count [ASC|DESC] LIMIT n [OFFSET m] +/// SELECT SUM(f) [WHERE q1 = v1 [AND …]] GROUP BY p ORDER BY f [ASC|DESC] LIMIT n [OFFSET m] +/// SELECT AVG(f) [WHERE q1 = v1 [AND …]] GROUP BY p ORDER BY f [ASC|DESC] LIMIT n [OFFSET m] /// ``` /// -/// with no `WHERE`, no `HAVING`, no `START AT` / `START AFTER`, exactly -/// one `GROUP BY` property, exactly one `ORDER BY` clause naming the +/// with no `HAVING`, no `START AT` / `START AFTER`, exactly one +/// `GROUP BY` property, exactly one `ORDER BY` clause naming the /// selected aggregate, `1 ≤ n ≤` [`MAX_RANKED_LIMIT`], and any -/// `m ≥ 0`. +/// `m ≥ 0`. `WHERE` clauses, when present, must be **equality pins** on +/// distinct properties — one per leading property of a covering +/// compound ranked index (see +/// [`equality_pins_from_where_clauses`]); the ranking then reads that +/// pinned prefix's own secondary. With no `where` the covering index is +/// single-property, exactly as before. /// /// `DESC` walks the axis from the largest aggregate down (the "top n" /// reading), `ASC` from the smallest up (the "bottom n" reading). @@ -134,16 +207,19 @@ pub fn detect_ranked_mode_v0( ) -> Result { // ---- GROUP BY: exactly one property ---------------------------- // - // Ranked indexes are single-property (rs-dpp rejects compound ones - // at parse time), and the sole property is what the secondary's - // group keys are. Zero group_by would ask to rank a single global - // aggregate against itself; two or more would need a compound index. + // The one property is the covering ranked index's LAST property — + // the level whose distinct values are the secondary's group keys. + // Zero group_by would ask to rank a single global aggregate against + // itself. Two or more is rejected because a compound ranked index + // ranks per prefix, not across a compound grouping: its leading + // properties are pinned by equality `where` clauses, and only the + // trailing property is grouped over. if group_by.len() != 1 { return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( - "ranked queries require exactly one `group_by` property (the ranked index's \ - only property); got {}. Ranked indexes are single-property — compound ranked \ - indexes are rejected at contract-parse time — so there is no compound \ - grouping to rank over.", + "ranked queries require exactly one `group_by` property (the covering ranked \ + index's trailing property); got {}. A compound ranked index ranks each \ + prefix's groups separately — pin every leading index property with an \ + equality `where` clause and `group_by` the trailing property.", group_by.len() )))); } @@ -249,24 +325,16 @@ pub fn detect_ranked_mode_v0( )))); } - // ---- WHERE: must be absent -------------------------------------- + // ---- WHERE: equality pins on the compound prefix ------------------ // - // Rejected rather than ignored. A single-property ranked index has no - // equality prefix a clause could narrow, and a clause on the ranked - // property itself asks for a ranking over a filtered subset — which - // the secondary cannot answer, because it is ordered by aggregate, - // not by group key. Silently dropping the filter would return the - // global ranking under the guise of a filtered one. - if !where_clauses.is_empty() { - return Err(Error::Query( - QuerySyntaxError::InvalidWhereClauseComponents( - "ranked queries do not accept `where` clauses: ranked indexes are \ - single-property, so there is no equality prefix to narrow, and the axis \ - secondary is ordered by aggregate rather than by group key — it cannot \ - rank a filtered subset. Rank the whole index, or add a narrower index.", - ), - )); - } + // Empty for the single-property form. For a compound ranked index, + // each `where` clause must pin one leading index property with `==` + // — that is what selects which prefix's secondary the walk reads + // (per-prefix semantics: there is no global cross-prefix ordering to + // serve). Anything other than a distinct-property equality is + // rejected loudly here; whether the pinned set matches a covering + // index's leading properties exactly is the index picker's call. + let equality_pins = equality_pins_from_where_clauses(where_clauses)?; // ---- LIMIT: required, 1 ..= MAX_RANKED_LIMIT --------------------- // @@ -331,5 +399,6 @@ pub fn detect_ranked_mode_v0( offset, group_by_property, aggregate_field, + equality_pins, }) } diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/path.rs b/packages/rs-drive/src/query/drive_document_ranked_query/path.rs index 2475f080a8c..4f1404b71ec 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/path.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/path.rs @@ -19,32 +19,50 @@ use crate::error::drive::DriveError; use crate::error::Error; use dpp::data_contract::document_type::Index; -/// Path of a single-property index's terminal property-name tree — -/// shared by the ranked and having-range query surfaces, which read -/// the same indexed tree. See -/// [`DriveDocumentRankedQuery::indexed_property_name_tree_path`] for -/// the segment layout and the single-property requirement. +/// Path of an index's terminal property-name tree — shared by the +/// ranked and having-range query surfaces, which read the same indexed +/// tree. See [`DriveDocumentRankedQuery::indexed_property_name_tree_path`] +/// for the segment layout. +/// +/// `equality_prefix_values` carries the **encoded index-key bytes** of +/// each leading property's pinned value, in index-property order — one +/// per property before the terminal one. Empty for a single-property +/// index. The arity must match exactly: a compound index's terminal +/// tree sits under one prefix value tree per leading property, and only +/// an equality `where` clause can name those values, so a missing or +/// surplus value means the caller resolved the wrong index — a typed +/// error, not a guess. pub(crate) fn indexed_property_name_tree_path_for_index( contract_id: &[u8; 32], document_type_name: &str, index: &Index, + equality_prefix_values: &[Vec], ) -> Result>, Error> { - let [property] = index.properties.as_slice() else { + let Some((terminal_property, leading_properties)) = index.properties.split_last() else { return Err(Error::Drive(DriveError::NotSupported( - "ranked queries require a single-property index: the ranked secondary \ - lives on the index's terminal property-name tree, and for a compound \ - index that tree sits under a prefix value tree whose value only a \ - `where` clause could name — but ranked queries accept no `where` \ - clauses", + "ranked queries require an index with at least one property", ))); }; - Ok(vec![ - vec![RootTree::DataContractDocuments as u8], - contract_id.to_vec(), - vec![1u8], - document_type_name.as_bytes().to_vec(), - property.name.as_bytes().to_vec(), - ]) + if leading_properties.len() != equality_prefix_values.len() { + return Err(Error::Drive(DriveError::NotSupported( + "ranked queries over a compound index require exactly one encoded equality \ + value per leading index property: the ranked secondary lives on the index's \ + terminal property-name tree, which for a compound index sits under one prefix \ + value tree per leading property, and only an equality `where` clause can name \ + those values", + ))); + } + let mut path = Vec::with_capacity(5 + 2 * leading_properties.len()); + path.push(vec![RootTree::DataContractDocuments as u8]); + path.push(contract_id.to_vec()); + path.push(vec![1u8]); + path.push(document_type_name.as_bytes().to_vec()); + for (property, value) in leading_properties.iter().zip(equality_prefix_values) { + path.push(property.name.as_bytes().to_vec()); + path.push(value.clone()); + } + path.push(terminal_property.name.as_bytes().to_vec()); + Ok(path) } impl DriveDocumentRankedQuery<'_> { @@ -52,6 +70,8 @@ impl DriveDocumentRankedQuery<'_> { /// whose primary holds one value tree per group and whose per-axis /// secondaries hold the ranking. /// + /// For a single-property index: + /// /// ```text /// [ RootTree::DataContractDocuments as u8 ] // 0x01 /// / @@ -60,23 +80,32 @@ impl DriveDocumentRankedQuery<'_> { /// / /// ``` /// - /// The children of that tree are the groups, keyed by the raw - /// index-key bytes of the property value — the same bytes that come - /// back as [`super::RankedEntry::key`]. + /// For a compound index `[p1, …, pn]`, each leading property + /// contributes two segments — its name and the **encoded index-key + /// bytes of its pinned value** (from + /// [`Self::equality_prefix_values`]) — and the terminal property + /// name closes the path: + /// + /// ```text + /// … / / / … / + /// ``` + /// + /// so the ranking read lands on **that prefix's** indexed tree: the + /// per-prefix secondary orders only the pinned prefix's groups. + /// + /// The children of the terminal tree are the groups, keyed by the + /// raw index-key bytes of the terminal property value — the same + /// bytes that come back as [`super::RankedEntry::key`]. /// - /// Errors when the index is not single-property. A compound ranked - /// index would terminate one level *below* a prefix value tree, so - /// its path would need the prefix property's value — which only a - /// `where` clause could supply, and ranked queries take none. rs-dpp - /// rejects compound ranked indexes at contract-parse time, so this is - /// a fail-closed backstop for the day that grammar relaxes: better a - /// typed error than a path pointing at a prefix level whose element - /// is not an indexed tree at all. + /// Errors when the number of encoded prefix values does not match + /// the index's leading-property count — the fail-closed backstop + /// for a caller that resolved the query against the wrong index. pub fn indexed_property_name_tree_path(&self) -> Result>, Error> { indexed_property_name_tree_path_for_index( &self.contract_id, &self.document_type_name, self.index, + &self.equality_prefix_values, ) } } diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs b/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs index 07bc14ccb23..cacce78a1d5 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs @@ -31,7 +31,7 @@ //! `adjustment` exists for the write-path suite's signed-average coverage //! (`delta` admits negatives, `grade` does not) and is unused here. -use super::index_picker::find_ranked_index_for_axis; +use super::index_picker::{find_ranked_index_for_axis, resolve_ranked_query_for_mode}; use super::mode_detection::{detect_ranked_mode, detect_ranked_mode_v0}; use super::*; use crate::drive::Drive; @@ -490,26 +490,104 @@ fn sum_and_avg_selects_require_a_field() { } } -/// A `where` clause is refused, not ignored. The axis secondary is -/// ordered by aggregate, not by group key, so it cannot rank a filtered -/// subset — silently dropping the filter would answer the unfiltered -/// question under the guise of the filtered one. +/// `where` clauses are equality pins on a compound index's leading +/// properties. Detection is shape-only: an equality clause becomes a +/// pin (whether a compound index actually covers it is the resolver's +/// call — see `pins_without_a_covering_compound_index_are_rejected`); +/// anything that is not a distinct-property equality is refused +/// loudly, `IN` with its own not-yet message because it will become +/// serviceable once multi-`IN` branching lands. #[test] -fn where_clauses_are_rejected() { - let where_clauses = vec![WhereClause { - field: GROUP_PROPERTY.to_string(), +fn where_clauses_resolve_to_equality_pins_and_reject_everything_else() { + // Equality pin: accepted at detection, carried in the mode. + let pinned = vec![WhereClause { + field: "chefId".to_string(), operator: WhereOperator::Equal, value: Value::Text("alpha".to_string()), }]; + let mode = detect_ranked_mode_v0( + &SelectProjection::avg("grade"), + &group_by(), + &[], + &order_by("grade", false), + &pinned, + page(Some(2), None), + ) + .expect("an equality pin is a well-formed prefix pin"); + assert_eq!( + mode.equality_pins, + vec![("chefId".to_string(), Value::Text("alpha".to_string()))] + ); + + // A range operator can never pin a single prefix value tree. + let ranged = vec![WhereClause { + field: "chefId".to_string(), + operator: WhereOperator::GreaterThan, + value: Value::Text("alpha".to_string()), + }]; let error = detect_ranked_mode_v0( &SelectProjection::avg("grade"), &group_by(), &[], &order_by("grade", false), - &where_clauses, + &ranged, page(Some(2), None), ) - .expect_err("ranked queries take no where clauses"); + .expect_err("a range operator cannot pin a prefix"); + assert!(matches!( + error, + Error::Query(QuerySyntaxError::InvalidWhereClauseComponents(_)) + )); + + // `IN` is rejected with its own message: v1 pins are equality-only, + // multi-`IN` branching is a future capability. + let in_clause = vec![WhereClause { + field: "chefId".to_string(), + operator: WhereOperator::In, + value: Value::Array(vec![Value::Text("alpha".to_string())]), + }]; + let error = detect_ranked_mode_v0( + &SelectProjection::avg("grade"), + &group_by(), + &[], + &order_by("grade", false), + &in_clause, + page(Some(2), None), + ) + .expect_err("IN prefixes are not yet supported"); + match error { + Error::Query(QuerySyntaxError::Unsupported(message)) => { + assert!( + message.contains("IN") && message.contains("not yet supported"), + "the IN rejection must say it is a not-yet capability, got: {message}" + ); + } + other => panic!("expected Unsupported for an IN prefix, got {other:?}"), + } + + // The same property pinned twice is a caller error, not a silent + // last-write-wins. + let duplicated = vec![ + WhereClause { + field: "chefId".to_string(), + operator: WhereOperator::Equal, + value: Value::Text("alpha".to_string()), + }, + WhereClause { + field: "chefId".to_string(), + operator: WhereOperator::Equal, + value: Value::Text("beta".to_string()), + }, + ]; + let error = detect_ranked_mode_v0( + &SelectProjection::avg("grade"), + &group_by(), + &[], + &order_by("grade", false), + &duplicated, + page(Some(2), None), + ) + .expect_err("duplicate pins on one property must fail"); assert!(matches!( error, Error::Query(QuerySyntaxError::InvalidWhereClauseComponents(_)) @@ -633,12 +711,12 @@ fn picker_requires_the_index_to_declare_the_requested_axis() { let indexes = index_map(vec![index]); assert!( - find_ranked_index_for_axis(&indexes, GROUP_PROPERTY, RankedAxis::Count, "").is_some(), + find_ranked_index_for_axis(&indexes, GROUP_PROPERTY, &[], RankedAxis::Count, "").is_some(), "the declared axis resolves" ); for (axis, field) in [(RankedAxis::Sum, "grade"), (RankedAxis::Avg, "grade")] { assert!( - find_ranked_index_for_axis(&indexes, GROUP_PROPERTY, axis, field).is_none(), + find_ranked_index_for_axis(&indexes, GROUP_PROPERTY, &[], axis, field).is_none(), "{axis:?} is not declared even though the index is summable and the stored \ element could host that secondary" ); @@ -657,11 +735,11 @@ fn picker_requires_the_select_field_to_be_the_indexed_summable() { for axis in [RankedAxis::Sum, RankedAxis::Avg] { assert!( - find_ranked_index_for_axis(&indexes, GROUP_PROPERTY, axis, "grade").is_some(), + find_ranked_index_for_axis(&indexes, GROUP_PROPERTY, &[], axis, "grade").is_some(), "{axis:?} on the indexed summable resolves" ); assert!( - find_ranked_index_for_axis(&indexes, GROUP_PROPERTY, axis, "tipAmount").is_none(), + find_ranked_index_for_axis(&indexes, GROUP_PROPERTY, &[], axis, "tipAmount").is_none(), "{axis:?} on a different field must not resolve" ); } @@ -676,7 +754,7 @@ fn picker_rejects_an_unknown_group_property() { let indexes = index_map(vec![index]); assert!( - find_ranked_index_for_axis(&indexes, "chefId", RankedAxis::Avg, "grade").is_none(), + find_ranked_index_for_axis(&indexes, "chefId", &[], RankedAxis::Avg, "grade").is_none(), "no index groups by `chefId`" ); } @@ -697,7 +775,8 @@ fn picker_rejects_compound_indexes() { let indexes = index_map(vec![index]); assert!( - find_ranked_index_for_axis(&indexes, GROUP_PROPERTY, RankedAxis::Avg, "price").is_none() + find_ranked_index_for_axis(&indexes, GROUP_PROPERTY, &[], RankedAxis::Avg, "price") + .is_none() ); } @@ -943,25 +1022,17 @@ fn client_side_query<'a>( .get(case.document_type_name) .expect("doctype exists") .indexes(); - let index = find_ranked_index_for_axis( - indexes, - &mode.group_by_property, - mode.axis, - &mode.aggregate_field, - ) - .expect("the fixture declares the axis"); - DriveDocumentRankedQuery { - document_type: contract + resolve_ranked_query_for_mode( + contract.id_ref().to_buffer(), + contract .document_type_for_name(case.document_type_name) .expect("doctype exists"), - contract_id: contract.id_ref().to_buffer(), - document_type_name: case.document_type_name.to_string(), - index, - axis: mode.axis, - descending: mode.descending, - k: mode.k, - offset: mode.offset, - } + case.document_type_name.to_string(), + indexes, + &mode, + platform_version(), + ) + .expect("the fixture declares the axis") } fn grovedb_root_hash(drive: &Drive) -> [u8; 32] { @@ -1958,13 +2029,13 @@ fn a_doctype_with_several_indexes_ranks_each_group_property_on_its_own_index() { ); assert_eq!(client_side_query(&contract, &by_chef).index.name, "byChef"); assert_eq!( - find_ranked_index_for_axis(indexes, GROUP_PROPERTY, RankedAxis::Avg, "grade") + find_ranked_index_for_axis(indexes, GROUP_PROPERTY, &[], RankedAxis::Avg, "grade") .expect("the Avg ranking resolves") .name, "byRestaurant", ); assert_eq!( - find_ranked_index_for_axis(indexes, CHEF_PROPERTY, RankedAxis::Count, "") + find_ranked_index_for_axis(indexes, CHEF_PROPERTY, &[], RankedAxis::Count, "") .expect("the Count ranking resolves") .name, "byChef", @@ -2015,7 +2086,7 @@ fn a_doctype_with_several_indexes_ranks_each_group_property_on_its_own_index() { // ranking over chefs has no index — and `byChefRestaurant` must not // be press-ganged into serving it. assert!( - find_ranked_index_for_axis(indexes, CHEF_PROPERTY, RankedAxis::Avg, "grade").is_none(), + find_ranked_index_for_axis(indexes, CHEF_PROPERTY, &[], RankedAxis::Avg, "grade").is_none(), "`byChefRestaurant` leads with chefId but is compound and unranked" ); let avg_by_chef = RankedCase { @@ -2033,3 +2104,299 @@ fn a_doctype_with_several_indexes_ranks_each_group_property_on_its_own_index() { "the error must name the missing keyword, got: {error}" ); } + +mod pinned_prefix { + //! `SELECT AVG(grade) FROM grades WHERE identityId = X GROUP BY + //! class ORDER BY AVG(grade) DESC LIMIT k` — the ranked top-k + //! surface over a compound ranked index `[identityId, class]` with + //! its leading property pinned. Per-prefix semantics: the walk + //! reads (and proves) the pinned identity's own secondary, so two + //! identities' class rankings never mix. The having-path sibling + //! (`drive_document_having_query::tests::pinned_prefix`) shares the + //! fixture and pins the write path; this module pins the rank walk + //! and its paginated proof. + + use super::super::drive_dispatcher::{DocumentRankedRequest, DocumentRankedResponse}; + use super::super::index_picker::resolve_ranked_query_for_mode; + use super::super::mode_detection::detect_ranked_mode; + use super::super::{DriveDocumentRankedQuery, RankedEntry, RankedEntryValue}; + use crate::drive::Drive; + use crate::error::query::QuerySyntaxError; + use crate::error::Error; + use crate::query::drive_document_ranked_query::RankedPaginationInputs; + use crate::query::projection::SelectProjection; + use crate::query::{OrderClause, WhereClause, WhereOperator}; + use crate::util::object_size_info::DocumentInfo::DocumentRefInfo; + use crate::util::object_size_info::{DocumentAndContractInfo, OwnedDocumentInfo}; + use crate::util::storage_flags::StorageFlags; + use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; + use dpp::block::block_info::BlockInfo; + use dpp::data_contract::accessors::v0::DataContractV0Getters; + use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; + use dpp::data_contract::document_type::random_document::CreateRandomDocument; + use dpp::document::{Document, DocumentV0Setters}; + use dpp::platform_value::Value; + use dpp::prelude::DataContract; + use dpp::tests::json_document::json_document_to_contract; + use dpp::version::PlatformVersion; + use grovedb::element::indexed::compute_avg_fixed_point; + use std::collections::BTreeMap; + + const PREFIX_PROPERTY: &str = "identityId"; + const CLASS_PROPERTY: &str = "class"; + const DOCUMENT_TYPE: &str = "grade"; + const IDENTITY_X: [u8; 32] = [1u8; 32]; + const IDENTITY_Y: [u8; 32] = [2u8; 32]; + + fn platform_version() -> &'static PlatformVersion { + PlatformVersion::latest() + } + + fn setup_grades_compound_ranked() -> (Drive, DataContract) { + let drive = setup_drive_with_initial_state_structure(None); + let pv = platform_version(); + let contract = json_document_to_contract( + "tests/supporting_files/contract/grades/grades-compound-ranked-contract.json", + false, + pv, + ) + .expect("expected to parse the compound ranked grades contract"); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + pv, + ) + .expect("expected to apply the compound ranked grades contract"); + (drive, contract) + } + + fn insert_grades(drive: &Drive, contract: &DataContract, rows: &[([u8; 32], &str, i64)]) { + let pv = platform_version(); + let document_type = contract + .document_type_for_name(DOCUMENT_TYPE) + .expect("grade doctype exists"); + for (i, (identity, class, grade)) in rows.iter().enumerate() { + let mut doc: Document = document_type + .random_document(Some(4000 + i as u64), pv) + .expect("random document"); + let mut props = BTreeMap::new(); + props.insert(PREFIX_PROPERTY.to_string(), Value::Identifier(*identity)); + props.insert(CLASS_PROPERTY.to_string(), Value::Text(class.to_string())); + props.insert("grade".to_string(), Value::I64(*grade)); + doc.set_properties(props); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo((&doc, None)), + owner_id: None, + }, + contract, + document_type, + }, + false, + BlockInfo::default(), + true, + None, + pv, + None, + ) + .expect("expected to insert a grade document"); + } + } + + fn pin(identity: [u8; 32]) -> Vec { + vec![WhereClause { + field: PREFIX_PROPERTY.to_string(), + operator: WhereOperator::Equal, + value: Value::Identifier(identity), + }] + } + + fn run( + drive: &Drive, + contract: &DataContract, + where_clauses: &[WhereClause], + limit: u32, + prove: bool, + ) -> Result { + let group_by = vec![CLASS_PROPERTY.to_string()]; + let order_by = vec![OrderClause { + field: "grade".to_string(), + ascending: false, + }]; + drive.execute_document_ranked_request( + DocumentRankedRequest { + contract, + document_type: contract + .document_type_for_name(DOCUMENT_TYPE) + .expect("grade doctype exists"), + group_by: &group_by, + select: SelectProjection::avg("grade"), + having: &[], + order_by: &order_by, + where_clauses, + limit: Some(limit), + offset: None, + has_start_at: false, + prove, + }, + None, + platform_version(), + ) + } + + fn client_side_query<'a>( + contract: &'a DataContract, + where_clauses: &[WhereClause], + limit: u32, + ) -> DriveDocumentRankedQuery<'a> { + let group_by = vec![CLASS_PROPERTY.to_string()]; + let order_by = vec![OrderClause { + field: "grade".to_string(), + ascending: false, + }]; + let mode = detect_ranked_mode( + &SelectProjection::avg("grade"), + &group_by, + &[], + &order_by, + where_clauses, + RankedPaginationInputs { + limit: Some(limit), + offset: None, + has_start_at: false, + }, + platform_version(), + ) + .expect("the case is well-formed"); + let indexes = contract + .document_types() + .get(DOCUMENT_TYPE) + .expect("grade doctype exists") + .indexes(); + resolve_ranked_query_for_mode( + contract.id_ref().to_buffer(), + contract + .document_type_for_name(DOCUMENT_TYPE) + .expect("grade doctype exists"), + DOCUMENT_TYPE.to_string(), + indexes, + &mode, + platform_version(), + ) + .expect("the fixture's compound index covers the pinned request") + } + + /// Top-k per pinned prefix, with the paginated proof round-tripped + /// against the live root hash — and isolation between prefixes: + /// X's and Y's class rankings come off different secondaries. + #[test] + fn top_k_over_pinned_prefix_reads_and_proves() { + let (drive, contract) = setup_grades_compound_ranked(); + insert_grades( + &drive, + &contract, + &[ + // X: art 90, english 80.5, math 80, history 70. + (IDENTITY_X, "math", 80), + (IDENTITY_X, "math", 80), + (IDENTITY_X, "english", 80), + (IDENTITY_X, "english", 81), + (IDENTITY_X, "art", 85), + (IDENTITY_X, "art", 95), + (IDENTITY_X, "history", 60), + (IDENTITY_X, "history", 80), + // Y: science 95, math 92.5. + (IDENTITY_Y, "math", 90), + (IDENTITY_Y, "math", 95), + (IDENTITY_Y, "science", 95), + (IDENTITY_Y, "science", 95), + ], + ); + + // X's top 2 by average, best first: art (90) then english (80.5). + // Y's science (95) and math (92.5) would both outrank them if the + // prefixes shared a secondary. + let x_pin = pin(IDENTITY_X); + let page = match run(&drive, &contract, &x_pin, 2, false).expect("read succeeds") { + DocumentRankedResponse::Entries(page) => page, + DocumentRankedResponse::Proof(_) => panic!("expected entries, got a proof"), + }; + assert_eq!(page.skipped, 0); + assert_eq!( + page.entries, + vec![ + RankedEntry { + key: b"art".to_vec(), + value: RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(180, 2)), + }, + RankedEntry { + key: b"english".to_vec(), + value: RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(161, 2)), + }, + ], + "X's own top 2: art 90 then english 80.5" + ); + + let proof = match run(&drive, &contract, &x_pin, 2, true).expect("prove succeeds") { + DocumentRankedResponse::Proof(proof) => proof, + DocumentRankedResponse::Entries(_) => panic!("expected a proof, got entries"), + }; + let (root_hash, verified) = client_side_query(&contract, &x_pin, 2) + .verify_ranked_top_k_proof(&proof, platform_version()) + .expect("the proof must verify"); + assert_eq!( + verified.entries, page.entries, + "verified entries must equal the unproven read" + ); + assert_eq!( + root_hash, + drive + .grove + .root_hash(None, &platform_version().drive.grove_version) + .unwrap() + .expect("root hash must be readable"), + "the proof must reconstruct the live grovedb root hash" + ); + + // Y's own ranking, proving the prefixes are separate secondaries. + let y_pin = pin(IDENTITY_Y); + let y_page = match run(&drive, &contract, &y_pin, 2, false).expect("read succeeds") { + DocumentRankedResponse::Entries(page) => page, + DocumentRankedResponse::Proof(_) => panic!("expected entries, got a proof"), + }; + assert_eq!( + y_page + .entries + .iter() + .map(|e| e.key.as_slice()) + .collect::>(), + vec![b"science".as_slice(), b"math".as_slice()], + "Y's own top 2: science 95 then math 92.5" + ); + } + + /// An unpinned request over the compound-only contract has no + /// covering index — there is no global cross-prefix ordering to + /// serve, so the rejection names the missing coverage. + #[test] + fn unpinned_prefix_is_rejected() { + let (drive, contract) = setup_grades_compound_ranked(); + insert_grades(&drive, &contract, &[(IDENTITY_X, "math", 80)]); + + let error = run(&drive, &contract, &[], 2, false) + .expect_err("an unpinned compound prefix must not resolve"); + assert!( + matches!( + error, + Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty(_)) + ), + "expected a no-covering-index rejection, got {error:?}" + ); + } +} diff --git a/packages/rs-drive/tests/supporting_files/contract/grades/grades-compound-ranked-contract.json b/packages/rs-drive/tests/supporting_files/contract/grades/grades-compound-ranked-contract.json new file mode 100644 index 00000000000..364d13703de --- /dev/null +++ b/packages/rs-drive/tests/supporting_files/contract/grades/grades-compound-ranked-contract.json @@ -0,0 +1,52 @@ +{ + "$formatVersion": "0", + "id": "AgradesC5w7Y9R4nDqJk2vHpL3uM6tF1xE8cA2bN7zXq", + "ownerId": "7m6mTfWqkrCnvLLPK3eqxQM2x2RDpYV6dsAyhVKsAEAQ", + "version": 1, + "documentSchemas": { + "grade": { + "type": "object", + "documentsMutable": false, + "canBeDeleted": false, + "indices": [ + { + "name": "byIdentityAndClass", + "properties": [ + { "identityId": "asc" }, + { "class": "asc" } + ], + "averageable": "grade", + "rangeAverageable": true, + "rankedAverageable": true + } + ], + "properties": { + "identityId": { + "type": "array", + "byteArray": true, + "minItems": 32, + "maxItems": 32, + "position": 0, + "contentMediaType": "application/x.dash.dpp.identifier" + }, + "class": { + "type": "string", + "maxLength": 32, + "position": 1 + }, + "grade": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "position": 2 + } + }, + "required": [ + "identityId", + "class", + "grade" + ], + "additionalProperties": false + } + } +} diff --git a/packages/rs-sdk/src/platform/documents/document_having_entries.rs b/packages/rs-sdk/src/platform/documents/document_having_entries.rs index 876a034c041..930e660aa47 100644 --- a/packages/rs-sdk/src/platform/documents/document_having_entries.rs +++ b/packages/rs-sdk/src/platform/documents/document_having_entries.rs @@ -22,17 +22,20 @@ //! a contiguous-range operator (`=`, `>`, `>=`, `<`, `<=`, `BETWEEN*` — //! `!=` and `IN` are rejected), and a `LIMIT`. `ORDER BY` is optional: //! omitted means ascending by the aggregate; naming the selected -//! aggregate sets the direction. No `where`, no `offset`, no -//! `start_at`. +//! aggregate sets the direction. `where` clauses are equality pins on a +//! covering compound ranked index's leading properties (one per leading +//! property, selecting which prefix's groups the bound reads) — absent +//! for a single-property index. No `offset`, no `start_at`. //! //! ## Contract prerequisites //! //! Same as the ranked surface: the index must opt in with //! `rankedCountable` / `rankedSummable` / `rankedAverageable` -//! (meta-schema v3, **protocol version 14+**), and ranked indexes are -//! single-property. Against a pre-v14 node the request is refused with -//! "HAVING clause is not yet implemented" — the intended activation -//! gate. +//! (meta-schema v3, **protocol version 14+**). The index may be +//! single-property (`group_by` its property, no `where`) or compound +//! (`group_by` its trailing property, equality-pin every leading one). +//! Against a pre-v14 node the request is refused with "HAVING clause +//! is not yet implemented" — the intended activation gate. //! //! ## Reading the result //! diff --git a/packages/rs-sdk/src/platform/documents/document_ranked_entries.rs b/packages/rs-sdk/src/platform/documents/document_ranked_entries.rs index b0cd0d41f95..f5a80ee3a59 100644 --- a/packages/rs-sdk/src/platform/documents/document_ranked_entries.rs +++ b/packages/rs-sdk/src/platform/documents/document_ranked_entries.rs @@ -18,7 +18,10 @@ //! //! Exactly one aggregate `select`, exactly one `group_by` property, //! exactly one `ORDER BY` clause naming that select's aggregate, and a -//! `LIMIT` — plus an optional `OFFSET`. No `where`, no `having`, no +//! `LIMIT` — plus an optional `OFFSET`. `where` clauses are equality +//! pins on a covering compound ranked index's leading properties (one +//! per leading property, selecting which prefix's own ranking the walk +//! reads) — absent for a single-property index. No `having`, no //! `start_at`: each of those is rejected rather than ignored, on both //! sides, because a ranked walk cannot honour them and silently //! answering a different question is worse than an error. @@ -32,8 +35,10 @@ //! ## Contract prerequisites //! //! The index must opt in with `rankedCountable` / `rankedSummable` / -//! `rankedAverageable` (meta-schema v3, **protocol version 14+**), and -//! ranked indexes are single-property. Against a protocol-version-13 +//! `rankedAverageable` (meta-schema v3, **protocol version 14+**). The +//! index may be single-property (`group_by` its property, no `where`) +//! or compound (`group_by` its trailing property, equality-pin every +//! leading one). Against a protocol-version-13 //! node the request is refused — v13's query table has no ranked path //! and rejects the ordering as `Unsupported`. That is the intended //! activation gate, not a bug: a v13 node and a v14 node must disagree diff --git a/packages/rs-sdk/src/platform/documents/having_proof_helpers.rs b/packages/rs-sdk/src/platform/documents/having_proof_helpers.rs index 2958f7ab0e7..d989c2ab254 100644 --- a/packages/rs-sdk/src/platform/documents/having_proof_helpers.rs +++ b/packages/rs-sdk/src/platform/documents/having_proof_helpers.rs @@ -4,9 +4,10 @@ //! caller-built [`DocumentQuery`] plus the node's response into a //! verified entry list. The routing decisions — which axis, which //! inclusive bounds the operator translates to, which direction, which -//! index covers them — are **not** re-derived here. They come from -//! rs-drive's own [`detect_having_mode`] and -//! [`find_ranked_index_for_axis`], the same two functions the server +//! index covers them, which prefix-value segments a pinned compound +//! request descends through — are **not** re-derived here. They come +//! from rs-drive's own [`detect_having_mode`] and +//! [`resolve_having_query_for_mode`], the same two functions the server //! calls, so client and server land on the same grove path and the same //! bounds by construction rather than by two copies of a grammar //! agreeing. The bounds matter doubly here: the verifier rebuilds the @@ -25,10 +26,8 @@ use dpp::{ data_contract::document_type::accessors::DocumentTypeV0Getters, }; use drive::query::drive_document_having_query::mode_detection::detect_having_mode; -use drive::query::drive_document_ranked_query::index_picker::find_ranked_index_for_axis; -use drive::query::{ - DocumentHavingMode, DriveDocumentHavingQuery, RankedEntry, RankedPaginationInputs, -}; +use drive::query::drive_document_having_query::resolve_having_query_for_mode; +use drive::query::{DocumentHavingMode, RankedEntry, RankedPaginationInputs}; use drive_proof_verifier::verify_having_range_proof; /// Validate that the caller-built [`DocumentQuery`] really describes a @@ -71,8 +70,9 @@ pub(super) fn assert_having_shape( query is `.with_select()`, `.with_group_by()`, \ `.with_having()` and `.with_limit(n)`, optionally \ - `.order_by_selected_aggregate()`, with no where clauses, no offset \ - and no start_at." + `.order_by_selected_aggregate()`, with no offset and no start_at; \ + where clauses, when present, must be equality pins on the covering compound \ + index's leading properties." ), }) } @@ -110,41 +110,29 @@ pub(super) fn verify_having_query( .or(Err(drive_proof_verifier::Error::EmptyResponseMetadata))?; let mode = assert_having_shape(&request, platform_version)?; - let axis = mode.bounds.axis(); - // Pick the index the prover picked — rs-drive's own picker, shared - // with the ranked surface because both read the same indexed tree. - let index = find_ranked_index_for_axis( + // Resolve the query exactly as the prover did — rs-drive's own + // resolution (covering-index pick, shared with the ranked surface + // because both read the same indexed tree, plus the equality-pin + // encoding into prefix path segments for pinned compound requests). + let having_query = resolve_having_query_for_mode( + request.data_contract.id().to_buffer(), + document_type, + request.document_type_name.clone(), document_type.indexes(), - &mode.group_by_property, - axis, - &mode.aggregate_field, + &mode, + platform_version, ) - .ok_or_else(|| drive_proof_verifier::Error::RequestError { + .map_err(|e| drive_proof_verifier::Error::RequestError { error: format!( - "no index on document type `{}` can serve a `{:?}` having bound grouped on \ - `{}`: a having-range query needs a single-property index over `{}` declaring \ - `{}` (and, for SUM / AVG, `summable: \"{}\"`). Ranked indexes are opt-in \ - contract grammar (meta-schema v3, protocol version 14+).", + "document type `{}` cannot serve this having-range query: {e}. Ranked indexes \ + are opt-in contract grammar (meta-schema v3, protocol version 14+); a pinned \ + (compound-index) bound additionally needs every leading index property pinned \ + by an equality where clause.", request.document_type_name, - axis, - mode.group_by_property, - mode.group_by_property, - axis.required_index_keyword(), - mode.aggregate_field, ), })?; - let having_query = DriveDocumentHavingQuery { - document_type, - contract_id: request.data_contract.id().to_buffer(), - document_type_name: request.document_type_name.clone(), - index, - bounds: mode.bounds, - descending: mode.descending, - limit: mode.limit, - }; - // Binds the reconstructed grovedb root hash to the quorum-signed // app hash before returning — see the module docs. let (root_hash, entries) = diff --git a/packages/rs-sdk/src/platform/documents/ranked_proof_helpers.rs b/packages/rs-sdk/src/platform/documents/ranked_proof_helpers.rs index 9211efbce89..1aa2fb817eb 100644 --- a/packages/rs-sdk/src/platform/documents/ranked_proof_helpers.rs +++ b/packages/rs-sdk/src/platform/documents/ranked_proof_helpers.rs @@ -4,9 +4,10 @@ //! caller-built [`DocumentQuery`] plus the node's response into a //! verified [`RankedPage`]. The routing decisions — which ranking axis, //! which direction, how many groups, how many ranks to skip, which -//! index covers them — are **not** re-derived here. They come from -//! rs-drive's own [`detect_ranked_mode`] and -//! [`find_ranked_index_for_axis`], the same two functions the server +//! index covers them, which prefix-value segments a pinned compound +//! request descends through — are **not** re-derived here. They come +//! from rs-drive's own [`detect_ranked_mode`] and +//! [`resolve_ranked_query_for_mode`], the same two functions the server //! calls, so client and server land on the same grove path and the same //! `(axis, k, descending, offset)` tuple by construction rather than by //! two copies of a grammar agreeing. @@ -27,11 +28,9 @@ use dpp::{ data_contract::accessors::v0::DataContractV0Getters, data_contract::document_type::accessors::DocumentTypeV0Getters, }; -use drive::query::drive_document_ranked_query::index_picker::find_ranked_index_for_axis; +use drive::query::drive_document_ranked_query::index_picker::resolve_ranked_query_for_mode; use drive::query::drive_document_ranked_query::mode_detection::detect_ranked_mode; -use drive::query::{ - DocumentRankedMode, DriveDocumentRankedQuery, RankedPage, RankedPaginationInputs, -}; +use drive::query::{DocumentRankedMode, RankedPage, RankedPaginationInputs}; use drive_proof_verifier::verify_ranked_top_k_proof; /// Validate that the caller-built [`DocumentQuery`] really describes a @@ -88,8 +87,9 @@ pub(super) fn assert_ranked_shape( "this DocumentQuery is not a well-formed ranked query: {e}. A ranked query is \ `.with_select()`, `.with_group_by()`, \ `.order_by_selected_aggregate()` and `.with_limit(n)`, \ - optionally `.with_offset(m)`, with no where clauses, no having and no \ - start_at." + optionally `.with_offset(m)`, with no having and no start_at; where clauses, \ + when present, must be equality pins on the covering compound index's leading \ + properties." ), }) } @@ -139,45 +139,31 @@ pub(super) fn verify_ranked_query( let mode = assert_ranked_shape(&request, platform_version)?; - // Pick the index the prover picked. Availability is decided from - // the index's `ranked_*` flags, not from the stored element - // variant, and this is rs-drive's own picker — a second - // implementation here could choose a different index on a - // contract with several candidates and then verify a proof of the - // wrong subtree's ranking (or, more likely, fail to verify at all - // with a confusing merk error). - let index = find_ranked_index_for_axis( + // Resolve the query exactly as the prover did — rs-drive's own + // resolution (covering-index pick + equality-pin encoding into + // prefix path segments). A second implementation here could choose + // a different index on a contract with several candidates, or + // encode a pinned prefix value differently, and then verify a proof + // of the wrong subtree's ranking (or, more likely, fail to verify + // at all with a confusing merk error). + let ranked_query = resolve_ranked_query_for_mode( + request.data_contract.id().to_buffer(), + document_type, + request.document_type_name.clone(), document_type.indexes(), - &mode.group_by_property, - mode.axis, - &mode.aggregate_field, + &mode, + platform_version, ) - .ok_or_else(|| drive_proof_verifier::Error::RequestError { + .map_err(|e| drive_proof_verifier::Error::RequestError { error: format!( - "no index on document type `{}` can rank by `{:?}` grouped on `{}`: a ranked \ - query needs a single-property index over `{}` declaring `{}` (and, for SUM / \ - AVG, `summable: \"{}\"`). Ranked indexes are opt-in contract grammar \ - (meta-schema v3, protocol version 14+).", + "document type `{}` cannot serve this ranked query: {e}. Ranked indexes are \ + opt-in contract grammar (meta-schema v3, protocol version 14+); a pinned \ + (compound-index) ranking additionally needs every leading index property \ + pinned by an equality where clause.", request.document_type_name, - mode.axis, - mode.group_by_property, - mode.group_by_property, - mode.axis.required_index_keyword(), - mode.aggregate_field, ), })?; - let ranked_query = DriveDocumentRankedQuery { - document_type, - contract_id: request.data_contract.id().to_buffer(), - document_type_name: request.document_type_name.clone(), - index, - axis: mode.axis, - descending: mode.descending, - k: mode.k, - offset: mode.offset, - }; - // Binds the reconstructed grovedb root hash to the quorum-signed // app hash before returning — see the module docs. let (root_hash, page) = From 9e72bdc54eb8ee6db3f3e601fa90d1de237f4120 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 08:06:51 +0700 Subject: [PATCH 05/12] fix(drive): exact float AVG bound translation and honest having continuation contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes for the having-range surface: - Float AVG operands now translate through the exact IEEE-754 value with operator-aware floor/ceiling (scaled_avg_operand + avg_bounds_for_operator) instead of f64-multiply-and-truncate, which lost sub-tick precision at the 10^19 scale and could move an inclusive bound by one tick — including the sign-dependent cases around zero. An equality bound between ticks is rejected loudly instead of silently becoming a point lookup on the truncated tick. - The continuation-by-bound story is stated honestly everywhere: a page cut at the limit continues past distinct aggregate values only; a cut inside a tie cannot be continued without a composite-key cursor (future capability), so callers size the limit above the widest expected tie. - The abci empty-axis mapping keeps its typed InvalidArgument but now describes both ranking and HAVING-range shapes, and the having dispatcher's comment no longer claims the path is unreachable (the empty-secondary prove failure is pinned by test). - Unexpected getDocuments result variants are reported by variant name only (shared result_variant_name helper) so error strings and logs cannot grow with — or leak — an untrusted response payload; the shared single-property path error now names both query surfaces. Co-Authored-By: Claude Fable 5 --- .../src/query/document_query/v1/mod.rs | 23 +- .../src/proof/document_having.rs | 15 +- .../src/proof/document_ranked.rs | 27 +- .../query/drive_document_having_query/mod.rs | 25 +- .../mode_detection.rs | 249 ++++++++++++++++-- .../drive_document_having_query/tests.rs | 120 ++++++++- .../query/drive_document_ranked_query/path.rs | 10 +- .../documents/document_having_entries.rs | 12 +- 8 files changed, 418 insertions(+), 63 deletions(-) diff --git a/packages/rs-drive-abci/src/query/document_query/v1/mod.rs b/packages/rs-drive-abci/src/query/document_query/v1/mod.rs index e79ee540a3e..2c54d7634e4 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/mod.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/mod.rs @@ -1510,11 +1510,14 @@ impl Platform { Err(drive::error::Error::Query(qe)) => { return Ok(QueryValidationResult::new_with_error(QueryError::Query(qe))); } - // Same empty-tree backstop as the ranked path: the - // range prover emits a guaranteed-empty range against - // an empty secondary, so this should be unreachable, - // but the failure class is merk-level and could - // surface from anywhere in the ancestor chain. + // Same empty-tree mapping as the ranked path — and + // genuinely reachable here: the range prover has no + // empty-range shape for a completely empty axis + // secondary (pinned by + // `an_empty_match_set_reads_empty_and_proves_empty` in + // rs-drive's having suite), so a proved HAVING request + // against an index with no documents yet surfaces this + // merk-level failure. Err(e) => match empty_ranking_proof_rejection(&e) { Some(rejection) => { return Ok(QueryValidationResult::new_with_error(rejection)); @@ -1638,11 +1641,11 @@ fn empty_ranking_proof_rejection(error: &drive::error::Error) -> Option, +) -> &'static str { + match result { + None => "an absent result", + Some(get_documents_response_v1::Result::Proof(_)) => "a proof", + Some(get_documents_response_v1::Result::Data(ResultData { variant })) => match variant { + None => "a ResultData with no variant", + Some(result_data::Variant::Documents(_)) => "a ResultData.documents payload", + Some(result_data::Variant::Counts(_)) => "a ResultData.counts payload", + Some(result_data::Variant::Sums(_)) => "a ResultData.sums payload", + Some(result_data::Variant::Averages(_)) => "a ResultData.averages payload", + Some(result_data::Variant::Ranked(_)) => "a ResultData.ranked payload", + }, + } +} + /// Decode one wire [`ProtoRankedEntry`] into rs-drive's /// [`RankedEntry`]. /// diff --git a/packages/rs-drive/src/query/drive_document_having_query/mod.rs b/packages/rs-drive/src/query/drive_document_having_query/mod.rs index c2f5d68b1d4..1b554131b35 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/mod.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/mod.rs @@ -44,11 +44,18 @@ //! share one bounds-to-query translation, exactly as they share the //! grove path. Completeness comes from the Merk range proof: the //! boundary commitments show no in-range group was omitted. -//! 2. **No `OFFSET`, no `start_at`.** The range primitives take a limit -//! but no skip; pagination of an over-long result set is a future -//! capability (a cursor on the `(sort_key ‖ group_key)` composite -//! keyspace), not an emulated one. A request carrying either is -//! rejected loudly. +//! 2. **No `OFFSET`, no `start_at` — and no full pagination.** The +//! range primitives take a limit but no skip, and a request carrying +//! either knob is rejected loudly. A page cut at `limit` can only be +//! continued past **distinct** aggregate values, by tightening the +//! bound past the last value seen; a cut that lands **inside a tie** +//! (several groups sharing the boundary aggregate) cannot be +//! continued at all — moving the threshold past the tied value skips +//! the uncollected tied groups, and keeping it returns the same +//! page. Enumerating through a tie wider than [`MAX_HAVING_LIMIT`] +//! needs a cursor on the `(sort_key ‖ group_key)` composite +//! keyspace, a future capability; until then, size `limit` above the +//! widest tie the data can produce, or accept the cut. //! 3. **Entry order is axis order in the walk direction.** Ascending by //! default (`ORDER BY` is optional here — the bound, not the //! ordering, is the point of the query); an explicit `ORDER BY` on @@ -252,8 +259,12 @@ pub struct DriveDocumentHavingQuery<'a> { /// Maximum number of matching groups to return. Fewer entries come /// back when fewer groups fall inside the bounds; that is not an /// error. **More matching groups than `limit` are silently cut at - /// `limit`** — the walk stops, and nothing marks the cut; a caller - /// that needs the full set must widen the limit or narrow the bound. + /// `limit`** — the walk stops, and nothing marks the cut. A caller + /// can continue past *distinct* aggregate values by tightening the + /// bound, but a cut inside a **tie** cannot be continued (see the + /// module docs): groups tied at the boundary aggregate that fell + /// past the limit stay unreachable until a composite-key cursor + /// exists, so size the limit above the widest expected tie. pub limit: u16, } diff --git a/packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs b/packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs index 7231065fcd6..ba909d86be4 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/mode_detection.rs @@ -246,15 +246,7 @@ pub fn detect_having_mode_v0( AxisRangeBounds::Sum { lo, hi } } RankedAxis::Avg => { - let (lo, hi) = bounds_for_operator( - clause.operator, - right, - avg_operand, - i128::MIN, - i128::MAX, - |v| v.checked_add(1), - |v| v.checked_sub(1), - )?; + let (lo, hi) = avg_bounds_for_operator(clause.operator, right)?; AxisRangeBounds::Avg { lo, hi } } }; @@ -349,7 +341,9 @@ pub fn detect_having_mode_v0( return Err(Error::Query(QuerySyntaxError::InvalidLimit( "having-range queries do not accept `offset`: matching groups are read from \ the bound's start and cut at `limit`. To reach deeper matches, tighten the \ - bound (e.g. move the threshold past the last aggregate value already seen)." + bound past the last aggregate value already seen — noting that a page cut \ + inside a tie (several groups sharing the boundary aggregate) cannot be \ + continued that way; size `limit` above the widest expected tie." .to_string(), ))); } @@ -483,26 +477,49 @@ fn sum_operand(value: &Value) -> Result { }) } +/// An `AVG` operand scaled into the axis's fixed-point domain, kept as +/// `(⌊t × SCALE⌋, is the product exactly that integer?)` so the operator +/// translation can pick the correct floor or ceiling per bound. +/// +/// A plain truncated `i128` would be wrong for floats: truncation is +/// toward zero, but an inclusive lower bound needs the *ceiling* and an +/// upper bound the *floor* — and around zero the two diverge in +/// opposite directions (`AVG >= 0.5-tick` must start at tick 1, while +/// truncation says 0; `AVG > -0.5-tick` must start at tick 0, while +/// truncate-then-increment says 1). +#[derive(Debug, Clone, Copy)] +struct ScaledAvgOperand { + /// `⌊t × SCALE⌋` — the floor (toward −∞) of the exact real product. + floor: i128, + /// Whether `t × SCALE` is exactly `floor` (the operand lands on a + /// fixed-point tick). Always true for integer operands. + exact: bool, +} + /// Extract an average operand and scale it into the axis's fixed-point /// domain (see -/// [`super::super::drive_document_ranked_query::RANKED_AVG_SCALE`]). +/// [`super::super::drive_document_ranked_query::RANKED_AVG_SCALE`]) +/// **exactly**. /// /// Integer operands scale exactly (`v × SCALE` — the product of any i64 /// with the scale fits in `i128` by the compile-time bound next to the -/// scale constant). Float operands are scaled through `f64` -/// multiplication and truncated toward zero; that conversion is -/// deterministic (IEEE 754) but inexact above 2^53, which is fine for a -/// *threshold* — callers needing exact fixed-point bounds pass integers -/// or pre-scaled values. -fn avg_operand(value: &Value) -> Result { +/// scale constant). Float operands are decomposed into their IEEE-754 +/// `±mantissa × 2^exponent` form and the product `±mantissa × SCALE × +/// 2^exponent` is floored with integer arithmetic — never through an +/// `f64` multiplication, which loses sub-tick precision long before +/// this scale (`SCALE = 10^19 > 2^53`): the nearest-f64 rounding of +/// `t × SCALE` can land on the wrong side of a tick and silently move +/// an inclusive bound by one. +fn scaled_avg_operand(value: &Value) -> Result { if let Some(int) = value.as_integer::() { - return (int as i128) + let floor = (int as i128) .checked_mul(AVG_FIXED_POINT_SCALE) .ok_or_else(|| { Error::Query(QuerySyntaxError::InvalidParameter(format!( "the `AVG(field)` having bound {int} does not fit the fixed-point domain" ))) - }); + })?; + return Ok(ScaledAvgOperand { floor, exact: true }); } let float = value.to_float().map_err(|_| { Error::Query(QuerySyntaxError::InvalidParameter(format!( @@ -514,14 +531,194 @@ fn avg_operand(value: &Value) -> Result { "an `AVG(field)` having bound must be finite; got {float}" )))); } - let scaled = float * AVG_FIXED_POINT_SCALE as f64; - // `f64 as i128` saturates rather than wrapping; the explicit range - // check keeps out-of-domain thresholds a loud caller error instead - // of a silent clamp to the domain edge. - if scaled <= i128::MIN as f64 || scaled >= i128::MAX as f64 { - return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + + // IEEE-754 double decomposition: float = ±mantissa × 2^exponent, + // with the implicit leading bit restored for normal numbers and the + // subnormal exponent pinned at 2^-1074. + let bits = float.to_bits(); + let negative = bits >> 63 == 1; + let raw_exponent = ((bits >> 52) & 0x7ff) as i64; + let fraction = bits & 0x000f_ffff_ffff_ffff; + let (mantissa, exponent) = if raw_exponent == 0 { + (fraction, -1074i64) + } else { + (fraction | 0x0010_0000_0000_0000, raw_exponent - 1075) + }; + if mantissa == 0 { + // ±0.0 — exactly tick zero. + return Ok(ScaledAvgOperand { + floor: 0, + exact: true, + }); + } + let out_of_domain = || { + Error::Query(QuerySyntaxError::InvalidParameter(format!( "the `AVG(field)` having bound {float} does not fit the fixed-point domain" + ))) + }; + + // mantissa < 2^54 and SCALE < 2^64, so the product stays well under + // i128::MAX (< 2^118); only the 2^exponent factor can overflow. + let magnitude = (mantissa as i128) * AVG_FIXED_POINT_SCALE; + let signed = if negative { -magnitude } else { magnitude }; + if exponent >= 0 { + // × 2^exponent, exactly. |signed| ≥ SCALE ≥ 1, so a factor the + // domain cannot hold means the bound itself is out of domain. + if exponent >= 127 { + return Err(out_of_domain()); + } + let floor = signed + .checked_mul(1i128 << exponent) + .ok_or_else(out_of_domain)?; + Ok(ScaledAvgOperand { floor, exact: true }) + } else { + // ÷ 2^-exponent with euclidean (toward −∞) division — exactly + // the floor, with the remainder deciding exactness. + let shift = -exponent as u32; + if shift >= 127 { + // |signed| < 2^118 < 2^shift ⇒ 0 < |t × SCALE| < 1: the + // product floors to 0 (positive) or −1 (negative), and is + // never exact (mantissa is non-zero). + return Ok(ScaledAvgOperand { + floor: if negative { -1 } else { 0 }, + exact: false, + }); + } + let divisor = 1i128 << shift; + Ok(ScaledAvgOperand { + floor: signed.div_euclid(divisor), + exact: signed.rem_euclid(divisor) == 0, + }) + } +} + +/// Translate `(operator, right operand)` into inclusive `[lo, hi]` +/// bounds in the Avg axis's fixed-point domain. +/// +/// The Avg counterpart of [`bounds_for_operator`], separate because Avg +/// operands may be floats that do not land on a fixed-point tick, and +/// the correct translation is then **operator-aware**: an inclusive +/// lower bound takes the ceiling of the exact product `t × SCALE`, an +/// upper bound its floor, and the exclusive translations collapse onto +/// the inclusive ones whenever `t` sits strictly between two ticks +/// (`v > t` and `v ≥ t` admit exactly the same integers there). All of +/// it works off [`scaled_avg_operand`]'s exact `(floor, exact)` pair: +/// +/// | operator | lower bound | upper bound | +/// |---------------|------------------------|------------------------| +/// | `= t` | `t` exact on a tick — otherwise rejected: nothing can match | +/// | `> t` | `⌊t⌋ + 1` | domain max | +/// | `>= t` | `⌈t⌉` | domain max | +/// | `< t` | domain min | `⌈t⌉ − 1` | +/// | `<= t` | domain min | `⌊t⌋` | +/// | `BETWEEN*` | per-end combination of the four rows above | +/// +/// Empty translations (`> MAX`, a between pair that inverts, an +/// equality between ticks) are rejected loudly, matching +/// [`bounds_for_operator`]'s contract: a bound that cannot match any +/// group is a caller error, and silently proving an empty page would +/// hide it. +fn avg_bounds_for_operator(operator: HavingOperator, right: &Value) -> Result<(i128, i128), Error> { + let scalar = || scaled_avg_operand(right); + let pair = || -> Result<(ScaledAvgOperand, ScaledAvgOperand), Error> { + let Some(items) = right.as_array() else { + return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + "`{operator:?}` requires a 2-element list operand `[lower, upper]`; got a \ + non-list value" + )))); + }; + let [lower, upper] = items.as_slice() else { + return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + "`{operator:?}` requires a 2-element list operand `[lower, upper]`; got {} \ + element(s)", + items.len() + )))); + }; + Ok((scaled_avg_operand(lower)?, scaled_avg_operand(upper)?)) + }; + let past_max = || { + Error::Query(QuerySyntaxError::InvalidParameter(format!( + "the `{operator:?}` bound matches no possible aggregate value: it lies at or \ + above the largest value the aggregate can take" + ))) + }; + let past_min = || { + Error::Query(QuerySyntaxError::InvalidParameter(format!( + "the `{operator:?}` bound matches no possible aggregate value: it lies at or \ + below the smallest value the aggregate can take" + ))) + }; + + // v > t ⇔ v ≥ ⌊t⌋ + 1 whether or not t is a tick (for a tick, + // strictly above it; between ticks, the ceiling). + let exclusive_lower = |bound: ScaledAvgOperand| bound.floor.checked_add(1).ok_or_else(past_max); + // v ≥ t ⇔ v ≥ ⌈t⌉. + let inclusive_lower = |bound: ScaledAvgOperand| { + if bound.exact { + Ok(bound.floor) + } else { + bound.floor.checked_add(1).ok_or_else(past_max) + } + }; + // v < t ⇔ v ≤ ⌈t⌉ − 1 (t on a tick: strictly below it; between + // ticks: the floor). + let exclusive_upper = |bound: ScaledAvgOperand| { + if bound.exact { + bound.floor.checked_sub(1).ok_or_else(past_min) + } else { + Ok(bound.floor) + } + }; + // v ≤ t ⇔ v ≤ ⌊t⌋, which never overflows. + let inclusive_upper = |bound: ScaledAvgOperand| bound.floor; + + let (lo, hi) = match operator { + HavingOperator::Equal => { + let bound = scalar()?; + if !bound.exact { + return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + "the `AVG(field)` equality bound {right} does not land on a fixed-point \ + tick, so no group's average can equal it; use a range operator (e.g. \ + `BETWEEN`) around the intended value, or an operand that scales exactly" + )))); + } + (bound.floor, bound.floor) + } + HavingOperator::GreaterThan => (exclusive_lower(scalar()?)?, i128::MAX), + HavingOperator::GreaterThanOrEquals => (inclusive_lower(scalar()?)?, i128::MAX), + HavingOperator::LessThan => (i128::MIN, exclusive_upper(scalar()?)?), + HavingOperator::LessThanOrEquals => (i128::MIN, inclusive_upper(scalar()?)), + HavingOperator::Between => { + let (lower, upper) = pair()?; + (inclusive_lower(lower)?, inclusive_upper(upper)) + } + HavingOperator::BetweenExcludeBounds => { + let (lower, upper) = pair()?; + (exclusive_lower(lower)?, exclusive_upper(upper)?) + } + HavingOperator::BetweenExcludeLeft => { + let (lower, upper) = pair()?; + (exclusive_lower(lower)?, inclusive_upper(upper)) + } + HavingOperator::BetweenExcludeRight => { + let (lower, upper) = pair()?; + (inclusive_lower(lower)?, exclusive_upper(upper)?) + } + HavingOperator::NotEqual | HavingOperator::In => { + return Err(Error::Query(QuerySyntaxError::Unsupported(format!( + "`{operator:?}` is not yet supported in having-range queries: it describes \ + a non-contiguous set of aggregate values, and the axis secondary serves \ + one contiguous range per request. Use a range operator, or issue one \ + request per contiguous range." + )))); + } + }; + + if lo > hi { + return Err(Error::Query(QuerySyntaxError::InvalidParameter(format!( + "the `having` bound resolves to the empty range [{lo}, {hi}] (lower above \ + upper), which matches no group; fix the operand" )))); } - Ok(scaled as i128) + Ok((lo, hi)) } diff --git a/packages/rs-drive/src/query/drive_document_having_query/tests.rs b/packages/rs-drive/src/query/drive_document_having_query/tests.rs index 159dd725f36..924fcafc275 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/tests.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/tests.rs @@ -139,6 +139,116 @@ mod grammar { ); } + /// Resolve one `AVG(grade)` clause and return the bounds. + fn avg_bounds( + operator: HavingOperator, + right: Value, + ) -> Result { + detect_having_mode_v0( + &SelectProjection::avg("grade"), + &["restaurantId".to_string()], + &[clause( + HavingAggregateFunction::Avg, + "grade", + operator, + right, + )], + &[], + &[], + pagination(50), + ) + .map(|mode| mode.bounds) + } + + /// Float thresholds translate through the **exact** IEEE-754 value + /// with operator-aware floor/ceiling — never through truncation. + /// `80.5` is exactly representable (`161 × 2⁻¹`), so its scaled + /// product lands on a tick and the inclusive/exclusive translations + /// differ by exactly one, on both ends. + #[test] + fn avg_float_threshold_on_a_tick_translates_like_an_integer() { + use crate::query::drive_document_ranked_query::RANKED_AVG_SCALE; + let tick = 161 * RANKED_AVG_SCALE / 2; // 80.5 × SCALE, exact + let max = i128::MAX; + let min = i128::MIN; + for (operator, expected_lo, expected_hi) in [ + (HavingOperator::GreaterThanOrEquals, tick, max), + (HavingOperator::GreaterThan, tick + 1, max), + (HavingOperator::LessThanOrEquals, min, tick), + (HavingOperator::LessThan, min, tick - 1), + (HavingOperator::Equal, tick, tick), + ] { + assert_eq!( + avg_bounds(operator, Value::Float(80.5)).expect("80.5 scales exactly"), + AxisRangeBounds::Avg { + lo: expected_lo, + hi: expected_hi + }, + "wrong translation for {operator:?} 80.5" + ); + } + } + + /// A float threshold that falls **between** two ticks: the + /// inclusive and exclusive translations collapse onto the same + /// integer bound — the ceiling for lower bounds, the floor for + /// upper bounds. `5e-20` scales to ≈0.5 of a tick, the exact case + /// truncation used to get wrong (`AVG >= 0.5-tick` must start at + /// tick 1, not 0), and its negation exercises the + /// negative-threshold direction (`AVG > -0.5-tick` must start at + /// tick 0, not 1 — truncate-then-increment lands on 1). + #[test] + fn avg_float_threshold_between_ticks_takes_operator_aware_bounds() { + let half_tick = Value::Float(5e-20); // ≈ 0.5 of a fixed-point tick + let neg_half_tick = Value::Float(-5e-20); + let max = i128::MAX; + let min = i128::MIN; + for (operator, right, expected_lo, expected_hi) in [ + ( + HavingOperator::GreaterThanOrEquals, + half_tick.clone(), + 1, + max, + ), + (HavingOperator::GreaterThan, half_tick.clone(), 1, max), + (HavingOperator::LessThanOrEquals, half_tick.clone(), min, 0), + (HavingOperator::LessThan, half_tick.clone(), min, 0), + (HavingOperator::GreaterThan, neg_half_tick.clone(), 0, max), + ( + HavingOperator::GreaterThanOrEquals, + neg_half_tick.clone(), + 0, + max, + ), + (HavingOperator::LessThan, neg_half_tick.clone(), min, -1), + ( + HavingOperator::LessThanOrEquals, + neg_half_tick.clone(), + min, + -1, + ), + ] { + assert_eq!( + avg_bounds(operator, right.clone()).expect("between-tick thresholds resolve"), + AxisRangeBounds::Avg { + lo: expected_lo, + hi: expected_hi + }, + "wrong translation for {operator:?} {right:?}" + ); + } + + // An equality on a value between ticks can never match a + // group's average; it is rejected loudly rather than silently + // converted into a point lookup on the truncated tick. + let error = avg_bounds(HavingOperator::Equal, half_tick) + .expect_err("equality between ticks matches nothing"); + assert!( + format!("{error}").contains("does not land on a fixed-point tick"), + "the rejection must explain the tick mismatch, got: {error}" + ); + } + #[test] fn order_by_the_selected_aggregate_sets_direction() { let mode = detect_having_mode_v0( @@ -1096,9 +1206,13 @@ mod execution { assert_proof_round_trips(&drive, &contract, &case, &entries); } - /// Pagination-by-bound: after a page cut at the limit, the caller - /// tightens the bound past the last seen value and continues — - /// the documented substitute for `OFFSET` on this surface. + /// Continuation-by-bound: after a page cut at the limit, the + /// caller tightens the bound past the last seen value and picks up + /// at the next *distinct* aggregate value. This is deliberately not + /// full pagination — a cut inside a tie cannot be continued (the + /// tied groups past the limit are unreachable without a + /// composite-key cursor); this fixture's counts are distinct, which + /// is the case the continuation serves. #[test] fn tightening_the_bound_continues_past_a_cut_page() { let (drive, contract) = setup_restaurants(); diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/path.rs b/packages/rs-drive/src/query/drive_document_ranked_query/path.rs index 2475f080a8c..a66c13b08c6 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/path.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/path.rs @@ -31,11 +31,11 @@ pub(crate) fn indexed_property_name_tree_path_for_index( ) -> Result>, Error> { let [property] = index.properties.as_slice() else { return Err(Error::Drive(DriveError::NotSupported( - "ranked queries require a single-property index: the ranked secondary \ - lives on the index's terminal property-name tree, and for a compound \ - index that tree sits under a prefix value tree whose value only a \ - `where` clause could name — but ranked queries accept no `where` \ - clauses", + "ranked and having-range queries require a single-property index: the \ + axis secondary lives on the index's terminal property-name tree, and \ + for a compound index that tree sits under a prefix value tree whose \ + value only a `where` clause could name — but these queries accept no \ + `where` clauses", ))); }; Ok(vec![ diff --git a/packages/rs-sdk/src/platform/documents/document_having_entries.rs b/packages/rs-sdk/src/platform/documents/document_having_entries.rs index 876a034c041..d8349ca5ef5 100644 --- a/packages/rs-sdk/src/platform/documents/document_having_entries.rs +++ b/packages/rs-sdk/src/platform/documents/document_having_entries.rs @@ -38,10 +38,14 @@ //! //! Entries come back in axis order in the walk direction; **do not //! re-sort**. Fewer than `n` entries means fewer groups matched. -//! **Exactly `n` may mean the match set was cut at the limit** — to -//! continue, tighten the bound past the last aggregate value seen and -//! ask again. Averages are fixed-point integers, exact on this (proved) -//! path; see the ranked module's notes, which apply verbatim. +//! **Exactly `n` may mean the match set was cut at the limit.** +//! Tightening the bound past the last aggregate value seen continues +//! past *distinct* values only: a cut inside a tie (several groups +//! sharing the boundary aggregate) cannot be continued — the tied +//! groups past the limit stay unreachable until a composite-key cursor +//! exists — so size the limit above the widest expected tie. Averages +//! are fixed-point integers, exact on this (proved) path; see the +//! ranked module's notes, which apply verbatim. //! //! ## Example: hashtags with more than 100 posts //! From 0c80416c7b86b94db5d0ce396f0d6cd72f9db556 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 10:29:00 +0700 Subject: [PATCH 06/12] docs(dapi-grpc): document the PV14 having-range grammar on the wire The canonical platform.proto comments still described the pre-PV14 behavior (every non-empty having rejected at every protocol version, having cannot combine with an aggregate ORDER BY). They now document the served single-clause COUNT/SUM/AVG range shape, the required ranked-axis index and limit, the absence of offset and cursor pagination with the distinct-value continuation and its tie limitation, and the unchanged rejection on v13 and earlier. Clients regenerated (only the Objective-C header embeds comments). The having-range route also gets its own OFFSET rejection message: the legacy one recommends `start_after` / `start_at`, which that surface rejects too, so it now explains continuation-by-bound instead. The legacy message stays byte-identical on every other route. Co-Authored-By: Claude Fable 5 --- .../platform/v0/objective-c/Platform.pbobjc.h | 53 ++++++++++++------- .../protos/platform/v0/platform.proto | 53 ++++++++++++------- .../src/query/document_query/v1/mod.rs | 36 +++++++++---- .../src/query/document_query/v1/tests.rs | 20 +++++-- 4 files changed, 110 insertions(+), 52 deletions(-) diff --git a/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h b/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h index 90b34897e29..c8c08853c0d 100644 --- a/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h +++ b/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h @@ -2648,12 +2648,18 @@ typedef GPB_ENUM(GetDocumentsRequest_HavingClause_Right_OneOfCase) { * before release rather than deprecated, because it invented * non-SQL grammar for something SQL already expresses. * - * **`HAVING` cannot yet combine with an aggregate `ORDER BY`.** - * The ranked executor reads a pre-sorted per-axis secondary and - * has no way to drop groups from the middle of that walk, so a - * request carrying both a non-empty `having` and a ranking - * `order_by` is rejected with `Unsupported` rather than served - * with one of the two silently ignored. + * **From protocol v14 a single `HAVING` clause is served as a + * bounded range read** (having-range mode): `SELECT GROUP BY + * p HAVING [ORDER BY ASC|DESC] LIMIT n` + * answers from the same per-axis secondary as ranked mode, on an + * index declaring the matching ranked axis. The clause's aggregate + * must be the selected aggregate, the operator must describe one + * contiguous range (`NOT_EQUAL` / `IN` are rejected), and the + * optional `ORDER BY` names the same aggregate to pick the walk + * direction. See the supported-shape table on + * `GetDocumentsRequestV1`. On protocol v13 and earlier every + * non-empty `having` stays rejected with `Unsupported`, exactly as + * before. * * The operator set mirrors `WhereOperator` minus `STARTS_WITH` * (prefix matching has no natural meaning against a scalar @@ -2897,12 +2903,16 @@ typedef GPB_ENUM(GetDocumentsRequest_GetDocumentsRequestV1_Start_OneOfCase) { * It returns `ResultData.ranked`. See `order_by` and the * supported-shape table below. * - * `having` is a boolean per-group predicate and is **still** - * `Unsupported` at every protocol version, ranked mode or not - * (`"HAVING clause is not yet implemented"`). It carries no ranking - * spelling: an earlier draft put cross-group ranking on the right of - * a `HAVING` (`HAVING AVG(grade) IN TOP(5)`) and that grammar was - * removed before release in favour of `ORDER BY` + `LIMIT`. + * **Having-range mode** is served from protocol v14: a single + * `having` clause whose aggregate is the selected aggregate turns + * the request into a bounded range read over the same per-axis + * secondary ranked mode walks, answered in `ResultData.ranked`. + * On protocol v13 and earlier every non-empty `having` is rejected + * (`"HAVING clause is not yet implemented"`). `having` carries no + * ranking spelling: an earlier draft put cross-group ranking on the + * right of a `HAVING` (`HAVING AVG(grade) IN TOP(5)`) and that + * grammar was removed before release in favour of `ORDER BY` + + * `LIMIT`. See the supported-shape table below. * * **Supported shapes** (everything else rejects with a typed * `QuerySyntaxError::Unsupported` so callers can detect un-wired @@ -2931,8 +2941,12 @@ typedef GPB_ENUM(GetDocumentsRequest_GetDocumentsRequestV1_Start_OneOfCase) { * - exactly one `group_by` property, exactly one `order_by` clause naming the select's aggregate (`f` for `SUM(f)` / `AVG(f)`, the `$count` sentinel for `COUNT(*)`), a `limit` in `1 ..= 100`, an optional `offset`, and no `where` / `having` / `start_at`, on an index declaring the matching `rankedCountable` / `rankedSummable` / `rankedAverageable` axis → ranked executor, answered in `ResultData.ranked`. * - `DESC` is the "top n" reading (walk the axis from the largest aggregate down), `ASC` the "bottom n" reading. Worked example: `SELECT AVG(grade) GROUP BY restaurantId ORDER BY grade DESC LIMIT 1 OFFSET 4` is the 5th-best restaurant. * + * `select=, group_by=[p], having=[ ]` (protocol v14+) — **having-range mode**: + * - exactly one `group_by` property, exactly one `having` clause whose aggregate is the select's aggregate, an operator describing one contiguous range (`EQUAL`, `GREATER_THAN[_OR_EQUALS]`, `LESS_THAN[_OR_EQUALS]`, `BETWEEN*`; `NOT_EQUAL` / `IN` rejected), a `limit` in `1 ..= 100`, an optional `order_by` naming the same aggregate (walk direction; ascending by default), and no `where` / `offset` / `start_at` / `start_after`, on an index declaring the matching ranked axis → having-range executor, answered in `ResultData.ranked`. + * - no offset or cursor pagination: a page cut at `limit` continues only by tightening the bound past the last *distinct* aggregate value seen; a cut inside a tie (several groups sharing the boundary aggregate) cannot be continued, so size `limit` above the widest expected tie. + * * **Rejected shapes** (return `Unsupported`): - * - any non-empty `having`, at every protocol version. + * - any non-empty `having` on protocol v13 and earlier; at v14+, any `having` shape outside having-range mode above (multiple clauses, an aggregate other than the select's, `NOT_EQUAL` / `IN`, or a carried `where` / `offset` / cursor). * - at v14+: a ranked-shaped request carrying a `where` clause, a `start_at` / `start_after` cursor, more than one `order_by`, or an `order_by` naming anything but the selected aggregate. * - `select=DOCUMENTS` with non-empty `group_by`. * - `select=COUNT` with `group_by` on a field that is not constrained by an `In` or range where clause. @@ -3110,12 +3124,13 @@ GPB_FINAL @interface GetDocumentsRequest_GetDocumentsRequestV1 : GPBMessage * `HavingClause` / `HavingAggregate` for the operator and * aggregate-function catalogs. * - * **Every non-empty `having` is rejected**, at every protocol - * version, with `Unsupported("HAVING clause is not yet - * implemented")`. The wire shape ships ahead of evaluation so - * callers can construct full `HAVING COUNT(*) > 5 AND - * SUM(amount) > 100` requests in their builders, and so the - * capability can land without another version bump. + * **From protocol v14 a single clause is served** as a bounded + * range read — having-range mode; see the message-level + * supported-shape table. On v13 and earlier every non-empty + * `having` is rejected with `Unsupported("HAVING clause is not + * yet implemented")`. Multi-clause `HAVING COUNT(*) > 5 AND + * SUM(amount) > 100` requests can still be constructed on the + * wire, but stay rejected until a multi-clause evaluator lands. * * **`having` does not express ranking.** "The n highest-scoring * groups" is `ORDER BY DESC LIMIT n` diff --git a/packages/dapi-grpc/protos/platform/v0/platform.proto b/packages/dapi-grpc/protos/platform/v0/platform.proto index c93f36b0a19..687c683431b 100644 --- a/packages/dapi-grpc/protos/platform/v0/platform.proto +++ b/packages/dapi-grpc/protos/platform/v0/platform.proto @@ -703,12 +703,18 @@ message GetDocumentsRequest { // before release rather than deprecated, because it invented // non-SQL grammar for something SQL already expresses. // - // **`HAVING` cannot yet combine with an aggregate `ORDER BY`.** - // The ranked executor reads a pre-sorted per-axis secondary and - // has no way to drop groups from the middle of that walk, so a - // request carrying both a non-empty `having` and a ranking - // `order_by` is rejected with `Unsupported` rather than served - // with one of the two silently ignored. + // **From protocol v14 a single `HAVING` clause is served as a + // bounded range read** (having-range mode): `SELECT GROUP BY + // p HAVING [ORDER BY ASC|DESC] LIMIT n` + // answers from the same per-axis secondary as ranked mode, on an + // index declaring the matching ranked axis. The clause's aggregate + // must be the selected aggregate, the operator must describe one + // contiguous range (`NOT_EQUAL` / `IN` are rejected), and the + // optional `ORDER BY` names the same aggregate to pick the walk + // direction. See the supported-shape table on + // `GetDocumentsRequestV1`. On protocol v13 and earlier every + // non-empty `having` stays rejected with `Unsupported`, exactly as + // before. // // The operator set mirrors `WhereOperator` minus `STARTS_WITH` // (prefix matching has no natural meaning against a scalar @@ -849,12 +855,16 @@ message GetDocumentsRequest { // It returns `ResultData.ranked`. See `order_by` and the // supported-shape table below. // - // `having` is a boolean per-group predicate and is **still** - // `Unsupported` at every protocol version, ranked mode or not - // (`"HAVING clause is not yet implemented"`). It carries no ranking - // spelling: an earlier draft put cross-group ranking on the right of - // a `HAVING` (`HAVING AVG(grade) IN TOP(5)`) and that grammar was - // removed before release in favour of `ORDER BY` + `LIMIT`. + // **Having-range mode** is served from protocol v14: a single + // `having` clause whose aggregate is the selected aggregate turns + // the request into a bounded range read over the same per-axis + // secondary ranked mode walks, answered in `ResultData.ranked`. + // On protocol v13 and earlier every non-empty `having` is rejected + // (`"HAVING clause is not yet implemented"`). `having` carries no + // ranking spelling: an earlier draft put cross-group ranking on the + // right of a `HAVING` (`HAVING AVG(grade) IN TOP(5)`) and that + // grammar was removed before release in favour of `ORDER BY` + + // `LIMIT`. See the supported-shape table below. // // **Supported shapes** (everything else rejects with a typed // `QuerySyntaxError::Unsupported` so callers can detect un-wired @@ -883,8 +893,12 @@ message GetDocumentsRequest { // - exactly one `group_by` property, exactly one `order_by` clause naming the select's aggregate (`f` for `SUM(f)` / `AVG(f)`, the `$count` sentinel for `COUNT(*)`), a `limit` in `1 ..= 100`, an optional `offset`, and no `where` / `having` / `start_at`, on an index declaring the matching `rankedCountable` / `rankedSummable` / `rankedAverageable` axis → ranked executor, answered in `ResultData.ranked`. // - `DESC` is the "top n" reading (walk the axis from the largest aggregate down), `ASC` the "bottom n" reading. Worked example: `SELECT AVG(grade) GROUP BY restaurantId ORDER BY grade DESC LIMIT 1 OFFSET 4` is the 5th-best restaurant. // + // `select=, group_by=[p], having=[ ]` (protocol v14+) — **having-range mode**: + // - exactly one `group_by` property, exactly one `having` clause whose aggregate is the select's aggregate, an operator describing one contiguous range (`EQUAL`, `GREATER_THAN[_OR_EQUALS]`, `LESS_THAN[_OR_EQUALS]`, `BETWEEN*`; `NOT_EQUAL` / `IN` rejected), a `limit` in `1 ..= 100`, an optional `order_by` naming the same aggregate (walk direction; ascending by default), and no `where` / `offset` / `start_at` / `start_after`, on an index declaring the matching ranked axis → having-range executor, answered in `ResultData.ranked`. + // - no offset or cursor pagination: a page cut at `limit` continues only by tightening the bound past the last *distinct* aggregate value seen; a cut inside a tie (several groups sharing the boundary aggregate) cannot be continued, so size `limit` above the widest expected tie. + // // **Rejected shapes** (return `Unsupported`): - // - any non-empty `having`, at every protocol version. + // - any non-empty `having` on protocol v13 and earlier; at v14+, any `having` shape outside having-range mode above (multiple clauses, an aggregate other than the select's, `NOT_EQUAL` / `IN`, or a carried `where` / `offset` / cursor). // - at v14+: a ranked-shaped request carrying a `where` clause, a `start_at` / `start_after` cursor, more than one `order_by`, or an `order_by` naming anything but the selected aggregate. // - `select=DOCUMENTS` with non-empty `group_by`. // - `select=COUNT` with `group_by` on a field that is not constrained by an `In` or range where clause. @@ -1084,12 +1098,13 @@ message GetDocumentsRequest { // `HavingClause` / `HavingAggregate` for the operator and // aggregate-function catalogs. // - // **Every non-empty `having` is rejected**, at every protocol - // version, with `Unsupported("HAVING clause is not yet - // implemented")`. The wire shape ships ahead of evaluation so - // callers can construct full `HAVING COUNT(*) > 5 AND - // SUM(amount) > 100` requests in their builders, and so the - // capability can land without another version bump. + // **From protocol v14 a single clause is served** as a bounded + // range read — having-range mode; see the message-level + // supported-shape table. On v13 and earlier every non-empty + // `having` is rejected with `Unsupported("HAVING clause is not + // yet implemented")`. Multi-clause `HAVING COUNT(*) > 5 AND + // SUM(amount) > 100` requests can still be constructed on the + // wire, but stay rejected until a multi-clause evaluator lands. // // **`having` does not express ranking.** "The n highest-scoring // groups" is `ORDER BY DESC LIMIT n` diff --git a/packages/rs-drive-abci/src/query/document_query/v1/mod.rs b/packages/rs-drive-abci/src/query/document_query/v1/mod.rs index 2c54d7634e4..d5b55335db2 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/mod.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/mod.rs @@ -408,21 +408,37 @@ enum RoutingDecision { /// had, **message for message**: those callers paginate with /// `start_after` / `start_at`, or by narrowing the range clause. /// -/// The message below is load-bearing and must not be reworded: clients -/// match on it, and on a protocol version whose routing table has no -/// ranked path (v13 and earlier) it is the *only* answer an offset can -/// get, exactly as it was before the ranked surface existed. +/// The legacy message below is load-bearing and must not be reworded: +/// clients match on it, and on a protocol version whose routing table +/// has no ranked path (v13 and earlier) it is the *only* answer an +/// offset can get, exactly as it was before the ranked surface existed. +/// +/// The having-range route gets its own message instead, because the +/// legacy one gives that caller wrong advice: the having surface has +/// neither offset nor cursor pagination (`start_after` / `start_at` +/// are rejected by mode detection — a document-ID cursor cannot +/// address the aggregate-sorted secondary). The only continuation is +/// tightening the bound past the last distinct aggregate value, with +/// the documented tie limitation. fn reject_offset_off_the_ranked_path( offset: Option, decision: &RoutingDecision, ) -> Result<(), QueryError> { - if offset.is_some() && !matches!(decision, RoutingDecision::Ranked) { - return Err(not_yet_implemented( + match decision { + _ if offset.is_none() => Ok(()), + RoutingDecision::Ranked => Ok(()), + RoutingDecision::HavingRange => Err(not_yet_implemented( + "OFFSET on a having-range query; this surface has no offset or cursor \ + pagination — to continue past a page cut at the limit, tighten the \ + `having` bound past the last aggregate value seen (this cannot cross \ + a tie: several groups sharing the boundary aggregate must fit inside \ + one limit)", + )), + _ => Err(not_yet_implemented( "OFFSET pagination (use cursor pagination via `start_after` / \ `start_at` instead)", - )); + )), } - Ok(()) } /// Test-only: expose the routing decision for unit tests without @@ -563,7 +579,9 @@ impl Platform { // once `validate_and_route` has answered, in // `reject_offset_off_the_ranked_path`. Off the ranked path the // rejection is byte-identical to the one this block used to - // emit. + // emit, except on the having-range route, which gets its own + // message (the legacy one recommends cursors that route also + // rejects). // Decode the proto-typed `repeated WhereClause` / `repeated // OrderClause` into drive's structured forms once, up diff --git a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs index de57aae029b..e282048d1cc 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs @@ -3279,9 +3279,11 @@ mod having_range_tests { } } - /// `OFFSET` stays ranked-only: the having-range walk has no skip, - /// so the post-routing offset gate fires with its long-standing - /// message. + /// `OFFSET` stays ranked-only, and the having-range route gets its + /// own rejection: the legacy message recommends `start_after` / + /// `start_at`, which this surface also rejects, so the having + /// message explains continuation-by-bound instead of pointing at + /// an unsupported cursor. #[test] fn offset_is_rejected_on_the_having_path() { let (platform, state, version) = setup_platform(None, Network::Testnet, None); @@ -3306,8 +3308,16 @@ mod having_range_tests { match ranked_error(&platform, &state, request, version) { QueryError::Query(QuerySyntaxError::Unsupported(message)) => { assert!( - message.contains("OFFSET pagination"), - "expected the offset gate's message, got: {message}" + message.contains("OFFSET on a having-range query"), + "expected the having-specific offset message, got: {message}" + ); + assert!( + message.contains("tighten the `having` bound"), + "the message must explain continuation-by-bound, got: {message}" + ); + assert!( + !message.contains("start_after"), + "the message must not recommend cursors this surface rejects, got: {message}" ); } other => panic!("expected Unsupported, got {other:?}"), From c418afc9a4bed534ee7e8840184208ceedc6bd6c Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 10:31:40 +0700 Subject: [PATCH 07/12] docs(dapi-grpc): wire docs cover compound-index equality pins The having-range and ranked mode tables inherited "no where" wording from the base branch; on this branch a compound ranked index requires exactly one EQUAL pin per leading property, so the supported and rejected shape bullets now say that. Objective-C client regenerated. Co-Authored-By: Claude Fable 5 --- .../clients/platform/v0/objective-c/Platform.pbobjc.h | 8 ++++---- packages/dapi-grpc/protos/platform/v0/platform.proto | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h b/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h index c8c08853c0d..edd18ed09b6 100644 --- a/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h +++ b/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h @@ -2938,16 +2938,16 @@ typedef GPB_ENUM(GetDocumentsRequest_GetDocumentsRequestV1_Start_OneOfCase) { * - a is the In field AND b is the range field, in that order → existing compound distinct shape; entries carry both `in_key` (= a's value) and `key` (= b's value). * * `select=, group_by=[p], order_by=[]` (protocol v14+) — **ranked mode**: - * - exactly one `group_by` property, exactly one `order_by` clause naming the select's aggregate (`f` for `SUM(f)` / `AVG(f)`, the `$count` sentinel for `COUNT(*)`), a `limit` in `1 ..= 100`, an optional `offset`, and no `where` / `having` / `start_at`, on an index declaring the matching `rankedCountable` / `rankedSummable` / `rankedAverageable` axis → ranked executor, answered in `ResultData.ranked`. + * - exactly one `group_by` property, exactly one `order_by` clause naming the select's aggregate (`f` for `SUM(f)` / `AVG(f)`, the `$count` sentinel for `COUNT(*)`), a `limit` in `1 ..= 100`, an optional `offset`, and no `having` / `start_at`, on an index declaring the matching `rankedCountable` / `rankedSummable` / `rankedAverageable` axis → ranked executor, answered in `ResultData.ranked`. On a single-property ranked index no `where` is accepted; on a compound ranked index every leading index property must be pinned with an `EQUAL` where clause (one per property, `group_by` names the trailing property), selecting which prefix's ranking is read. * - `DESC` is the "top n" reading (walk the axis from the largest aggregate down), `ASC` the "bottom n" reading. Worked example: `SELECT AVG(grade) GROUP BY restaurantId ORDER BY grade DESC LIMIT 1 OFFSET 4` is the 5th-best restaurant. * * `select=, group_by=[p], having=[ ]` (protocol v14+) — **having-range mode**: - * - exactly one `group_by` property, exactly one `having` clause whose aggregate is the select's aggregate, an operator describing one contiguous range (`EQUAL`, `GREATER_THAN[_OR_EQUALS]`, `LESS_THAN[_OR_EQUALS]`, `BETWEEN*`; `NOT_EQUAL` / `IN` rejected), a `limit` in `1 ..= 100`, an optional `order_by` naming the same aggregate (walk direction; ascending by default), and no `where` / `offset` / `start_at` / `start_after`, on an index declaring the matching ranked axis → having-range executor, answered in `ResultData.ranked`. + * - exactly one `group_by` property, exactly one `having` clause whose aggregate is the select's aggregate, an operator describing one contiguous range (`EQUAL`, `GREATER_THAN[_OR_EQUALS]`, `LESS_THAN[_OR_EQUALS]`, `BETWEEN*`; `NOT_EQUAL` / `IN` rejected), a `limit` in `1 ..= 100`, an optional `order_by` naming the same aggregate (walk direction; ascending by default), and no `offset` / `start_at` / `start_after`, on an index declaring the matching ranked axis → having-range executor, answered in `ResultData.ranked`. `where` follows the same rule as ranked mode: none on a single-property ranked index; exactly one `EQUAL` pin per leading index property on a compound ranked index. * - no offset or cursor pagination: a page cut at `limit` continues only by tightening the bound past the last *distinct* aggregate value seen; a cut inside a tie (several groups sharing the boundary aggregate) cannot be continued, so size `limit` above the widest expected tie. * * **Rejected shapes** (return `Unsupported`): - * - any non-empty `having` on protocol v13 and earlier; at v14+, any `having` shape outside having-range mode above (multiple clauses, an aggregate other than the select's, `NOT_EQUAL` / `IN`, or a carried `where` / `offset` / cursor). - * - at v14+: a ranked-shaped request carrying a `where` clause, a `start_at` / `start_after` cursor, more than one `order_by`, or an `order_by` naming anything but the selected aggregate. + * - any non-empty `having` on protocol v13 and earlier; at v14+, any `having` shape outside having-range mode above (multiple clauses, an aggregate other than the select's, `NOT_EQUAL` / `IN`, a `where` shape other than the compound-index equality pins above, or a carried `offset` / cursor). + * - at v14+: a ranked-shaped request carrying a `where` shape other than the compound-index equality pins above (a non-`EQUAL` operator, a repeated or non-leading property, or a missing pin), a `start_at` / `start_after` cursor, more than one `order_by`, or an `order_by` naming anything but the selected aggregate. * - `select=DOCUMENTS` with non-empty `group_by`. * - `select=COUNT` with `group_by` on a field that is not constrained by an `In` or range where clause. * - `select=COUNT` with `group_by.len() > 2`. diff --git a/packages/dapi-grpc/protos/platform/v0/platform.proto b/packages/dapi-grpc/protos/platform/v0/platform.proto index 687c683431b..caf4ca62861 100644 --- a/packages/dapi-grpc/protos/platform/v0/platform.proto +++ b/packages/dapi-grpc/protos/platform/v0/platform.proto @@ -890,16 +890,16 @@ message GetDocumentsRequest { // - a is the In field AND b is the range field, in that order → existing compound distinct shape; entries carry both `in_key` (= a's value) and `key` (= b's value). // // `select=, group_by=[p], order_by=[]` (protocol v14+) — **ranked mode**: - // - exactly one `group_by` property, exactly one `order_by` clause naming the select's aggregate (`f` for `SUM(f)` / `AVG(f)`, the `$count` sentinel for `COUNT(*)`), a `limit` in `1 ..= 100`, an optional `offset`, and no `where` / `having` / `start_at`, on an index declaring the matching `rankedCountable` / `rankedSummable` / `rankedAverageable` axis → ranked executor, answered in `ResultData.ranked`. + // - exactly one `group_by` property, exactly one `order_by` clause naming the select's aggregate (`f` for `SUM(f)` / `AVG(f)`, the `$count` sentinel for `COUNT(*)`), a `limit` in `1 ..= 100`, an optional `offset`, and no `having` / `start_at`, on an index declaring the matching `rankedCountable` / `rankedSummable` / `rankedAverageable` axis → ranked executor, answered in `ResultData.ranked`. On a single-property ranked index no `where` is accepted; on a compound ranked index every leading index property must be pinned with an `EQUAL` where clause (one per property, `group_by` names the trailing property), selecting which prefix's ranking is read. // - `DESC` is the "top n" reading (walk the axis from the largest aggregate down), `ASC` the "bottom n" reading. Worked example: `SELECT AVG(grade) GROUP BY restaurantId ORDER BY grade DESC LIMIT 1 OFFSET 4` is the 5th-best restaurant. // // `select=, group_by=[p], having=[ ]` (protocol v14+) — **having-range mode**: - // - exactly one `group_by` property, exactly one `having` clause whose aggregate is the select's aggregate, an operator describing one contiguous range (`EQUAL`, `GREATER_THAN[_OR_EQUALS]`, `LESS_THAN[_OR_EQUALS]`, `BETWEEN*`; `NOT_EQUAL` / `IN` rejected), a `limit` in `1 ..= 100`, an optional `order_by` naming the same aggregate (walk direction; ascending by default), and no `where` / `offset` / `start_at` / `start_after`, on an index declaring the matching ranked axis → having-range executor, answered in `ResultData.ranked`. + // - exactly one `group_by` property, exactly one `having` clause whose aggregate is the select's aggregate, an operator describing one contiguous range (`EQUAL`, `GREATER_THAN[_OR_EQUALS]`, `LESS_THAN[_OR_EQUALS]`, `BETWEEN*`; `NOT_EQUAL` / `IN` rejected), a `limit` in `1 ..= 100`, an optional `order_by` naming the same aggregate (walk direction; ascending by default), and no `offset` / `start_at` / `start_after`, on an index declaring the matching ranked axis → having-range executor, answered in `ResultData.ranked`. `where` follows the same rule as ranked mode: none on a single-property ranked index; exactly one `EQUAL` pin per leading index property on a compound ranked index. // - no offset or cursor pagination: a page cut at `limit` continues only by tightening the bound past the last *distinct* aggregate value seen; a cut inside a tie (several groups sharing the boundary aggregate) cannot be continued, so size `limit` above the widest expected tie. // // **Rejected shapes** (return `Unsupported`): - // - any non-empty `having` on protocol v13 and earlier; at v14+, any `having` shape outside having-range mode above (multiple clauses, an aggregate other than the select's, `NOT_EQUAL` / `IN`, or a carried `where` / `offset` / cursor). - // - at v14+: a ranked-shaped request carrying a `where` clause, a `start_at` / `start_after` cursor, more than one `order_by`, or an `order_by` naming anything but the selected aggregate. + // - any non-empty `having` on protocol v13 and earlier; at v14+, any `having` shape outside having-range mode above (multiple clauses, an aggregate other than the select's, `NOT_EQUAL` / `IN`, a `where` shape other than the compound-index equality pins above, or a carried `offset` / cursor). + // - at v14+: a ranked-shaped request carrying a `where` shape other than the compound-index equality pins above (a non-`EQUAL` operator, a repeated or non-leading property, or a missing pin), a `start_at` / `start_after` cursor, more than one `order_by`, or an `order_by` naming anything but the selected aggregate. // - `select=DOCUMENTS` with non-empty `group_by`. // - `select=COUNT` with `group_by` on a field that is not constrained by an `In` or range where clause. // - `select=COUNT` with `group_by.len() > 2`. From 39e14e5771f199e27d1aaefc0e99b83febf1f4cc Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 10:35:57 +0700 Subject: [PATCH 08/12] docs(dapi-grpc): having-range ORDER BY uses the ranked order-key spelling `ORDER BY ` read as an explicit OrderClause.aggregate target, which the wire rejects; the accepted spelling is the field name for SUM/AVG and the $count sentinel for COUNT(*), same as ranked mode. Objective-C client regenerated. Co-Authored-By: Claude Fable 5 --- .../platform/v0/objective-c/Platform.pbobjc.h | 17 ++++++++++------- .../dapi-grpc/protos/platform/v0/platform.proto | 17 ++++++++++------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h b/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h index c8c08853c0d..295993f3f5e 100644 --- a/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h +++ b/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h @@ -2650,13 +2650,16 @@ typedef GPB_ENUM(GetDocumentsRequest_HavingClause_Right_OneOfCase) { * * **From protocol v14 a single `HAVING` clause is served as a * bounded range read** (having-range mode): `SELECT GROUP BY - * p HAVING [ORDER BY ASC|DESC] LIMIT n` - * answers from the same per-axis secondary as ranked mode, on an - * index declaring the matching ranked axis. The clause's aggregate - * must be the selected aggregate, the operator must describe one - * contiguous range (`NOT_EQUAL` / `IN` are rejected), and the - * optional `ORDER BY` names the same aggregate to pick the walk - * direction. See the supported-shape table on + * p HAVING [ORDER BY ASC|DESC] + * LIMIT n` answers from the same per-axis secondary as ranked + * mode, on an index declaring the matching ranked axis. The + * clause's aggregate must be the selected aggregate, the operator + * must describe one contiguous range (`NOT_EQUAL` / `IN` are + * rejected), and the optional `ORDER BY` picks the walk direction + * using the same order-key spelling as ranked mode: `f` for + * `SUM(f)` / `AVG(f)`, the `$count` sentinel for `COUNT(*)` — + * never an explicit `OrderClause.aggregate` target, which is + * rejected. See the supported-shape table on * `GetDocumentsRequestV1`. On protocol v13 and earlier every * non-empty `having` stays rejected with `Unsupported`, exactly as * before. diff --git a/packages/dapi-grpc/protos/platform/v0/platform.proto b/packages/dapi-grpc/protos/platform/v0/platform.proto index 687c683431b..bfa0b5cf68b 100644 --- a/packages/dapi-grpc/protos/platform/v0/platform.proto +++ b/packages/dapi-grpc/protos/platform/v0/platform.proto @@ -705,13 +705,16 @@ message GetDocumentsRequest { // // **From protocol v14 a single `HAVING` clause is served as a // bounded range read** (having-range mode): `SELECT GROUP BY - // p HAVING [ORDER BY ASC|DESC] LIMIT n` - // answers from the same per-axis secondary as ranked mode, on an - // index declaring the matching ranked axis. The clause's aggregate - // must be the selected aggregate, the operator must describe one - // contiguous range (`NOT_EQUAL` / `IN` are rejected), and the - // optional `ORDER BY` names the same aggregate to pick the walk - // direction. See the supported-shape table on + // p HAVING [ORDER BY ASC|DESC] + // LIMIT n` answers from the same per-axis secondary as ranked + // mode, on an index declaring the matching ranked axis. The + // clause's aggregate must be the selected aggregate, the operator + // must describe one contiguous range (`NOT_EQUAL` / `IN` are + // rejected), and the optional `ORDER BY` picks the walk direction + // using the same order-key spelling as ranked mode: `f` for + // `SUM(f)` / `AVG(f)`, the `$count` sentinel for `COUNT(*)` — + // never an explicit `OrderClause.aggregate` target, which is + // rejected. See the supported-shape table on // `GetDocumentsRequestV1`. On protocol v13 and earlier every // non-empty `having` stays rejected with `Unsupported`, exactly as // before. From 9ec210b6288bf9f7604594a1c71acab8e8fa7825 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 11:45:09 +0700 Subject: [PATCH 09/12] fix(dpp): satisfy clippy type_complexity on the compound-ranked test helper Co-Authored-By: Claude Fable 5 --- .../document_type/class_methods/try_from_schema/v3/mod.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs index e30d3e52bb7..715b0214f06 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs @@ -988,11 +988,15 @@ mod tests { // prefix-overlap conflict // ------------------------------------------------------------------- + /// One extra single-property index for [`compound_ranked_schema`]: + /// `(name, property, keys)`. + type ExtraIndexSpec<'a> = (&'a str, &'a str, Vec<(&'a str, Value)>); + /// A doctype with a compound ranked index `[region, restaurantId]` /// (avg axis on `grade`), plus optional extra single-property /// indexes to provoke — or fail to provoke — the prefix-overlap - /// conflict. `extra_index` entries are `(name, property, keys)`. - fn compound_ranked_schema(extra_indexes: Vec<(&str, &str, Vec<(&str, Value)>)>) -> Value { + /// conflict. + fn compound_ranked_schema(extra_indexes: Vec) -> Value { let compound_entry: Vec<(Value, Value)> = vec![ ( Value::Text("name".to_string()), From 869dfba3596e552269d395b47a13188b5413cdb4 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 20:10:53 +0700 Subject: [PATCH 10/12] fix(drive): adapt post-merge tests to pinned-prefix query APIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The base branch's final refactor (file-based mode-detection versioning, abci dispatch split, trust-boundary and batched-drain tests) landed after this branch's last sync, so its new call sites used the pre-compound signatures. find_ranked_index_for_axis callers now pass the (empty) pin set and the query structs their (empty) equality_prefix_values — both fixtures are single-property indexes. Co-Authored-By: Claude Fable 5 --- packages/rs-drive-abci/src/query/document_query/v1/tests.rs | 2 ++ .../insert/insert_contract/v0/tests/batched_group_drain.rs | 2 ++ .../query/drive_document_ranked_query/mode_detection/v0/mod.rs | 1 + 3 files changed, 5 insertions(+) diff --git a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs index 83bb3a2ce54..475419587a2 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs @@ -3991,6 +3991,7 @@ mod having_trust_boundary { .expect("grade doctype exists") .indexes(), &mode.group_by_property, + &[], mode.bounds.axis(), &mode.aggregate_field, ) @@ -4003,6 +4004,7 @@ mod having_trust_boundary { document_type_name: "grade".to_string(), index, bounds: mode.bounds, + equality_prefix_values: Vec::new(), descending: mode.descending, limit: mode.limit, } diff --git a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/batched_group_drain.rs b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/batched_group_drain.rs index f178e6bf7da..d81d73c4267 100644 --- a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/batched_group_drain.rs +++ b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/batched_group_drain.rs @@ -369,11 +369,13 @@ fn verified_page( index: find_ranked_index_for_axis( indexes, GROUP_PROPERTY, + &[], axis.ranked, axis.aggregate_field(), ) .expect("the fixture declares this axis"), axis: axis.ranked, + equality_prefix_values: Vec::new(), descending: !ascending, k: 100, offset: 0, diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection/v0/mod.rs b/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection/v0/mod.rs index d4b05f80aa5..8a96f033b47 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection/v0/mod.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection/v0/mod.rs @@ -13,6 +13,7 @@ use crate::error::Error; use crate::query::having::HavingClause; use crate::query::projection::{SelectFunction, SelectProjection}; use crate::query::{OrderClause, WhereClause, WhereOperator}; +use dpp::platform_value::Value; /// Translate a request's `where` clauses into equality pins — /// `(property, value)` pairs, one per clause — for the ranked and From b17dfcf62066bfb55558f11268e4d53ab9cdac4f Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 21:24:02 +0700 Subject: [PATCH 11/12] fix(dpp)!: ranked key ceiling binds only the terminal property; null pins address absent prefixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review blockers on the compound-ranked surface: - The ranked item-key ceiling (247/239 bytes) applies only to the terminal index property — the one whose encoded value becomes the indexed tree's item key behind the sort key. Leading prefix properties are ordinary grovedb path segments bound by the generic limits, so a 63-character leading string on an avg-ranked compound index now parses. Boundary-pinned in both directions. - A null equality pin now encodes as the empty path segment the write walkers store for an absent optional leading property (get_raw_for_document_type(..).unwrap_or_default()), instead of failing in the system-property encoders. WHERE tag == null GROUP BY class round-trips read, proof, and client verification on both the ranked and having surfaces, pinned with a new optional-leading-tag doctype in the compound fixture. Also per review: validate_no_ranked_prefix_overlap moves out of the shared parse core into generation 3 as the ranked_index_structure_check callback (same pattern as ranked_index_key_length_check, no flag branch in common); the v3 query-table doc names the tables shipped before PV14 (V0 for 1-11, V1 for 12-13); the SDK limit test's doc comment describes the 1..=100 contract it pins. Co-Authored-By: Claude Fable 5 --- .../try_from_schema/common/mod.rs | 112 +++------- .../class_methods/try_from_schema/v1/mod.rs | 1 + .../class_methods/try_from_schema/v3/mod.rs | 209 +++++++++++++++++- .../drive_document_having_query/tests.rs | 152 +++++++++++++ .../index_picker.rs | 11 + .../drive_document_ranked_query/tests.rs | 160 ++++++++++++++ .../grades-compound-ranked-contract.json | 54 ++++- .../drive_abci_query_versions/v3.rs | 8 +- .../documents/document_having_entries.rs | 7 +- 9 files changed, 619 insertions(+), 95 deletions(-) diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs index 15d7a8dc152..c334f4b586f 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs @@ -129,6 +129,27 @@ pub(super) fn no_ranked_index_key_length_check( Ok(()) } +/// RANKED: the cross-index structural check a generation runs over a +/// document type's parsed indices, before the merged index tree is built. +/// +/// Generation 3's implementation rejects the compound-ranked prefix-overlap +/// shape the storage layer cannot lay out; earlier generations pass +/// [`no_ranked_index_structure_check`] — their grammar rejects the +/// `ranked*` keywords, so no index they parse can carry a ranking axis. +/// Unlike [`RankedIndexKeyLengthCheck`] this runs on **every** parse path, +/// not only under `full_validation`: a contract admitted through a +/// non-validating parse would brick the first document insert. +pub(super) type RankedIndexStructureCheck = + fn(&BTreeMap) -> Result<(), ProtocolError>; + +/// The [`RankedIndexStructureCheck`] for a generation that has no ranking +/// axes to constrain. +pub(super) fn no_ranked_index_structure_check( + _indices: &BTreeMap, +) -> Result<(), ProtocolError> { + Ok(()) +} + /// Everything the shared parsing steps need to know about *which* generation is /// running them. /// @@ -169,6 +190,8 @@ pub(super) struct ParserGeneration { pub admit_ranked: bool, /// See [`RankedIndexKeyLengthCheck`]. pub ranked_index_key_length_check: RankedIndexKeyLengthCheck, + /// See [`RankedIndexStructureCheck`]. + pub ranked_index_structure_check: RankedIndexStructureCheck, } /// Reject a document type whose name is not a non-empty ASCII @@ -947,13 +970,11 @@ fn parse_indices( .transpose()? .unwrap_or_default(); - // Cross-index structural check for the ranked grammar. Gated on the - // generation constant only to skip the scan where it cannot fire: - // without `admit_ranked` the `ranked*` keywords do not parse at all, - // so no index below can carry a ranking axis. - if ctx.generation.admit_ranked { - validate_no_ranked_prefix_overlap(&indices)?; - } + // Cross-index structural check owned by the generation, exactly like + // the per-property key-length check above: generations whose index + // grammar rejects the `ranked*` keywords pass the no-op, so the shared + // core never branches on a version. + (ctx.generation.ranked_index_structure_check)(&indices)?; let index_structure = IndexLevel::try_from_indices(indices.values(), ctx.name, ctx.platform_version)?; @@ -961,83 +982,6 @@ fn parse_indices( Ok((indices, index_structure)) } -/// Rejects the one compound-ranked shape the storage layer cannot lay -/// out: a compound ranked index whose **full leading prefix** also -/// terminates a separate countable and/or summable index. -/// -/// A ranked flag on a compound index `[p1, …, pn]` puts an indexed tree -/// at each prefix's terminal `pn` property-name level — inside the value -/// trees of the `[p1, …, pn-1]` level. When another countable/summable -/// index terminates at exactly that prefix, those value trees are -/// aggregating (`CountTree` / `SumTree` / …), and every continuation -/// subtree inside them must be wrapped in a `NonCounted` / `NotSummed` -/// shell so its contents don't pollute the prefix index's aggregates. -/// grovedb structurally rejects that shell around an indexed tree — the -/// wrapper would neutralize the very aggregates the ranking indexes — -/// so the write path fails closed at document insert. Rejecting the -/// contract here surfaces the conflict at registration instead. -/// -/// Only the **exact** `n-1` prefix conflicts. An aggregating index -/// terminating at a shorter prefix wraps a plain intermediate -/// property-name tree (fine), and one extending *past* the ranked -/// terminal lives inside the indexed tree's value trees, which the -/// storage layer supports (see rs-drive's -/// `ranked_terminator_with_a_compound_continuation_gets_both_treatments`). -/// -/// Property comparison is by name, positionally: the merged index-level -/// tree keys sub-levels by property name in declaration order, so -/// `[a, b]` and `[b, a]` never share a level and cannot conflict. -/// -/// Unconditional (not gated on `full_validation`): the same structural -/// impossibility must reject the contract on every parse path — a -/// contract admitted through a non-validating parse would brick the -/// first document insert under the ranked index. -fn validate_no_ranked_prefix_overlap( - indices: &BTreeMap, -) -> Result<(), ProtocolError> { - for ranked in indices.values() { - let is_ranked = - ranked.ranked_countable || ranked.ranked_summable || ranked.ranked_averageable; - if !is_ranked || ranked.properties.len() < 2 { - continue; - } - let prefix = &ranked.properties[..ranked.properties.len() - 1]; - for other in indices.values() { - if other.name == ranked.name { - continue; - } - let terminates_at_prefix = other.properties.len() == prefix.len() - && other - .properties - .iter() - .zip(prefix.iter()) - .all(|(a, b)| a.name == b.name); - let aggregates = other.countable.is_countable() || other.summable.is_some(); - if terminates_at_prefix && aggregates { - return Err(consensus_or_protocol_data_contract_error( - DataContractError::InvalidContractStructure(format!( - "compound ranked index `{}` conflicts with index `{}`: the ranked \ - index's leading prefix [{}] also terminates a countable/summable \ - index, so the ranked terminal tree would sit inside aggregating \ - value trees and need a NonCounted/NotSummed shell — which the \ - storage layer rejects for indexed trees because the wrapper would \ - neutralize the aggregates the ranking indexes. Drop the ranked \ - flags, or drop the aggregate flags from the prefix index", - ranked.name, - other.name, - prefix - .iter() - .map(|p| p.name.as_str()) - .collect::>() - .join(", "), - )), - )); - } - } - } - Ok(()) -} - /// The per-property half of index validation: an already-indexed system /// property may not be indexed again, a user property must be defined, and an /// indexed property's type must be one the index encoding supports within its diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs index 41a56c52db9..86107721e1f 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v1/mod.rs @@ -99,6 +99,7 @@ impl DocumentTypeV1 { // therefore has no ranked key ceiling to enforce. admit_ranked: false, ranked_index_key_length_check: common::no_ranked_index_key_length_check, + ranked_index_structure_check: common::no_ranked_index_structure_check, }, platform_version, ) diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs index 715b0214f06..92a8b13fcde 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs @@ -15,13 +15,15 @@ //! index-key length ceilings, and the constants they are derived from. use crate::data_contract::config::DataContractConfig; -// Only the ranked key-length rule below names these, and it is validation-only. -#[cfg(feature = "validation")] +use crate::data_contract::document_type::class_methods::consensus_or_protocol_data_contract_error; +// The prefix-overlap rule below runs on every parse path; only the +// key-length rule is validation-only. use crate::data_contract::document_type::index::Index; #[cfg(feature = "validation")] use crate::data_contract::document_type::property::DocumentPropertyType; use crate::data_contract::document_type::v2::DocumentTypeV2; use crate::data_contract::document_type::DocumentType; +use crate::data_contract::errors::DataContractError; use crate::data_contract::{TokenConfiguration, TokenContractPosition}; use crate::validation::operations::ProtocolValidationOperation; use crate::version::PlatformVersion; @@ -110,6 +112,15 @@ fn validate_ranked_index_property_key_length( property_type: &DocumentPropertyType, platform_version: &PlatformVersion, ) -> Result<(), ProtocolError> { + // Only the **terminal** property's encoded value becomes an + // indexed-tree item key, mirrored into the ordered secondary behind + // the sort key — that is where the tightened ceiling comes from. + // Leading properties of a compound ranked index are ordinary grovedb + // path segments, bound by the generic limits checked after this. + if index.properties.last().map(|p| p.name.as_str()) != Some(index_property_name) { + return Ok(()); + } + let Some(limit) = ranked_index_key_length_limit(index) else { return Ok(()); }; @@ -179,6 +190,83 @@ fn validate_ranked_index_property_key_length( ))) } +/// Rejects the one compound-ranked shape the storage layer cannot lay +/// out: a compound ranked index whose **full leading prefix** also +/// terminates a separate countable and/or summable index. +/// +/// A ranked flag on a compound index `[p1, …, pn]` puts an indexed tree +/// at each prefix's terminal `pn` property-name level — inside the value +/// trees of the `[p1, …, pn-1]` level. When another countable/summable +/// index terminates at exactly that prefix, those value trees are +/// aggregating (`CountTree` / `SumTree` / …), and every continuation +/// subtree inside them must be wrapped in a `NonCounted` / `NotSummed` +/// shell so its contents don't pollute the prefix index's aggregates. +/// grovedb structurally rejects that shell around an indexed tree — the +/// wrapper would neutralize the very aggregates the ranking indexes — +/// so the write path fails closed at document insert. Rejecting the +/// contract here surfaces the conflict at registration instead. +/// +/// Only the **exact** `n-1` prefix conflicts. An aggregating index +/// terminating at a shorter prefix wraps a plain intermediate +/// property-name tree (fine), and one extending *past* the ranked +/// terminal lives inside the indexed tree's value trees, which the +/// storage layer supports (see rs-drive's +/// `ranked_terminator_with_a_compound_continuation_gets_both_treatments`). +/// +/// Property comparison is by name, positionally: the merged index-level +/// tree keys sub-levels by property name in declaration order, so +/// `[a, b]` and `[b, a]` never share a level and cannot conflict. +/// +/// Unconditional (not gated on `full_validation`): the same structural +/// impossibility must reject the contract on every parse path — a +/// contract admitted through a non-validating parse would brick the +/// first document insert under the ranked index. +fn validate_no_ranked_prefix_overlap( + indices: &BTreeMap, +) -> Result<(), ProtocolError> { + for ranked in indices.values() { + let is_ranked = + ranked.ranked_countable || ranked.ranked_summable || ranked.ranked_averageable; + if !is_ranked || ranked.properties.len() < 2 { + continue; + } + let prefix = &ranked.properties[..ranked.properties.len() - 1]; + for other in indices.values() { + if other.name == ranked.name { + continue; + } + let terminates_at_prefix = other.properties.len() == prefix.len() + && other + .properties + .iter() + .zip(prefix.iter()) + .all(|(a, b)| a.name == b.name); + let aggregates = other.countable.is_countable() || other.summable.is_some(); + if terminates_at_prefix && aggregates { + return Err(consensus_or_protocol_data_contract_error( + DataContractError::InvalidContractStructure(format!( + "compound ranked index `{}` conflicts with index `{}`: the ranked \ + index's leading prefix [{}] also terminates a countable/summable \ + index, so the ranked terminal tree would sit inside aggregating \ + value trees and need a NonCounted/NotSummed shell — which the \ + storage layer rejects for indexed trees because the wrapper would \ + neutralize the aggregates the ranking indexes. Drop the ranked \ + flags, or drop the aggregate flags from the prefix index", + ranked.name, + other.name, + prefix + .iter() + .map(|p| p.name.as_str()) + .collect::>() + .join(", "), + )), + )); + } + } + } + Ok(()) +} + /// The [`common::RankedIndexKeyLengthCheck`] generation 3 runs on every indexed /// property, resolved at compile time. /// @@ -246,6 +334,7 @@ fn try_from_schema_generation_3( // RANKED: the constants that make this generation 3. admit_ranked: true, ranked_index_key_length_check: RANKED_INDEX_KEY_LENGTH_CHECK, + ranked_index_structure_check: validate_no_ranked_prefix_overlap, }, platform_version, )?; @@ -872,6 +961,122 @@ mod tests { ); } + /// A compound `[region, restaurantId]` avg-ranked index over the same + /// doctype shape as [`ranked_bound_schema`], with independent control + /// of the leading and terminal string properties' `maxLength`. + fn compound_ranked_bound_schema(leading: Value, terminal: Value) -> Value { + let mut index_entry: Vec<(Value, Value)> = vec![ + ( + Value::Text("name".to_string()), + Value::Text("byRegionRestaurant".to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![ + Value::Map(vec![( + Value::Text("region".to_string()), + Value::Text("asc".to_string()), + )]), + Value::Map(vec![( + Value::Text("restaurantId".to_string()), + Value::Text("asc".to_string()), + )]), + ]), + ), + ]; + index_entry.extend( + avg_ranked_extras() + .into_iter() + .map(|(key, value)| (Value::Text(key.to_string()), value)), + ); + + Value::Map(vec![ + ( + Value::Text("type".to_string()), + Value::Text("object".to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Map(vec![ + (Value::Text("region".to_string()), leading), + (Value::Text("restaurantId".to_string()), terminal), + ( + Value::Text("grade".to_string()), + platform_value!({ + "type": "integer", + "minimum": 0, + "maximum": 100, + "position": 2, + }), + ), + ]), + ), + ( + Value::Text("required".to_string()), + Value::Array(vec![ + Value::Text("region".to_string()), + Value::Text("restaurantId".to_string()), + Value::Text("grade".to_string()), + ]), + ), + ( + Value::Text("additionalProperties".to_string()), + Value::Bool(false), + ), + ( + Value::Text("indices".to_string()), + Value::Array(vec![Value::Map(index_entry)]), + ), + ]) + } + + fn string_property_at(max_length: u32, position: u32) -> Value { + platform_value!({ + "type": "string", + "maxLength": max_length, + "position": position, + }) + } + + /// The ranked ceiling binds only the **terminal** property — the one + /// whose encoded value becomes the indexed tree's item key. A leading + /// prefix property is an ordinary grovedb path segment: the generic + /// 63-character indexed-string limit is what binds it, not the + /// 59-character avg-ranked ceiling. + #[test] + fn ranked_ceiling_binds_the_terminal_property_not_the_leading_prefix() { + // Leading at 63 (over the 59 ranked bound, at the generic bound) + // with a terminal that fits the ranked bound: accepted. + parse_bound(compound_ranked_bound_schema( + string_property_at(63, 0), + string_property_at(59, 1), + )) + .expect("a 63-character leading prefix is a path segment, not an item key"); + + // The same 60-character string the single-property tests reject is + // still rejected when it is the terminal property of a compound + // ranked index. + let error = parse_bound(compound_ranked_bound_schema( + string_property_at(30, 0), + string_property_at(60, 1), + )) + .expect_err("the terminal property keeps the 239-byte avg ceiling"); + let msg = format!("{error:?}"); + assert!( + msg.contains("maxLength") && msg.contains("59") && msg.contains("239"), + "the error must still name the ranked bound for the terminal; got {msg}" + ); + + // And the generic indexed-string limit still binds the leading + // property: 64 characters is over the 63-character cap whether or + // not the index is ranked. + parse_bound(compound_ranked_bound_schema( + string_property_at(64, 0), + string_property_at(59, 1), + )) + .expect_err("the generic 63-character limit still binds the leading prefix"); + } + /// Avg is strictly tighter than Count/Sum, and an index carrying Avg /// *alongside* the other axes has to satisfy the tightest of them: the /// 60-character string that a count-only ranked index accepts is refused diff --git a/packages/rs-drive/src/query/drive_document_having_query/tests.rs b/packages/rs-drive/src/query/drive_document_having_query/tests.rs index 91aea75512f..9bfd3277323 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/tests.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/tests.rs @@ -1956,4 +1956,156 @@ mod pinned_prefix { "proving a never-written prefix value tree must error" ); } + + /// A **null** pin addresses the empty-segment prefix the write path + /// creates for an absent optional leading property — the having + /// bound reads (and proves) the tagless subtree, and tagged rows + /// stay in their own prefix. Sibling of the ranked suite's + /// `a_null_pin_addresses_the_absent_value_prefix`. + #[test] + fn a_null_pin_bounds_the_absent_value_prefix() { + const TAGGED_DOCTYPE: &str = "taggedGrade"; + let (drive, contract) = setup_grades_compound_ranked(); + let pv = platform_version(); + let document_type = contract + .document_type_for_name(TAGGED_DOCTYPE) + .expect("taggedGrade doctype exists"); + + for (i, (tag, class, grade)) in [ + (None, "math", 80i64), + (None, "math", 90), + (None, "science", 60), + (Some("honors"), "science", 100), + ] + .iter() + .enumerate() + { + let mut doc: Document = document_type + .random_document(Some(8000 + i as u64), pv) + .expect("random document"); + let mut props = BTreeMap::new(); + if let Some(tag) = tag { + props.insert("tag".to_string(), Value::Text(tag.to_string())); + } + props.insert(GROUP_PROPERTY.to_string(), Value::Text(class.to_string())); + props.insert("grade".to_string(), Value::I64(*grade)); + doc.set_properties(props); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo((&doc, None)), + owner_id: None, + }, + contract: &contract, + document_type, + }, + false, + BlockInfo::default(), + true, + None, + pv, + None, + ) + .expect("expected to insert a tagged grade document"); + } + + let null_pin = vec![WhereClause { + field: "tag".to_string(), + operator: WhereOperator::Equal, + value: Value::Null, + }]; + let group_by = vec![GROUP_PROPERTY.to_string()]; + let having = vec![clause( + HavingAggregateFunction::Avg, + "grade", + HavingOperator::GreaterThan, + Value::U64(70), + )]; + let request = |prove: bool| DocumentHavingRequest { + contract: &contract, + document_type, + group_by: &group_by, + select: SelectProjection::avg("grade"), + having: &having, + order_by: &[], + where_clauses: &null_pin, + limit: Some(10), + offset: None, + has_start_at: false, + prove, + }; + + // Only the tagless math group (avg 85) clears the bound: the + // tagless science (60) misses it, and the tagged science (100) + // lives under its own prefix. + let entries = match drive + .execute_document_having_request(request(false), None, pv) + .expect("the null-pinned read succeeds") + { + DocumentHavingResponse::Entries(entries) => entries, + DocumentHavingResponse::Proof(_) => panic!("expected entries, got a proof"), + }; + assert_eq!( + entries, + vec![RankedEntry { + key: b"math".to_vec(), + value: RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(170, 2)), + }], + "the tagless bound: math 85 only — the honors science row \ + must not leak into the null prefix" + ); + + let proof = match drive + .execute_document_having_request(request(true), None, pv) + .expect("the null-pinned prove succeeds") + { + DocumentHavingResponse::Proof(proof) => proof, + DocumentHavingResponse::Entries(_) => panic!("expected a proof, got entries"), + }; + let mode = detect_having_mode( + &SelectProjection::avg("grade"), + &group_by, + &having, + &[], + &null_pin, + RankedPaginationInputs { + limit: Some(10), + offset: None, + has_start_at: false, + }, + pv, + ) + .expect("the null-pinned case is well-formed"); + let query = resolve_having_query_for_mode( + contract.id_ref().to_buffer(), + document_type, + TAGGED_DOCTYPE.to_string(), + contract + .document_types() + .get(TAGGED_DOCTYPE) + .expect("taggedGrade doctype exists") + .indexes(), + &mode, + pv, + ) + .expect("the compound index covers the null-pinned request"); + assert_eq!( + query.equality_prefix_values, + vec![Vec::::new()], + "a null pin must encode as the write path's empty segment" + ); + let (root_hash, verified) = query + .verify_having_range_proof(&proof, pv) + .expect("the null-pinned proof must verify"); + assert_eq!(verified, entries); + assert_eq!( + root_hash, + drive + .grove + .root_hash(None, &pv.drive.grove_version) + .unwrap() + .expect("root hash must be readable"), + ); + } } diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/index_picker.rs b/packages/rs-drive/src/query/drive_document_ranked_query/index_picker.rs index cfaab9be252..5f5408fbf30 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/index_picker.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/index_picker.rs @@ -253,6 +253,17 @@ pub fn encode_equality_prefix_values( prefix encoder disagreed on the pins", )) })?; + // A null pin addresses the subtree the write walkers create + // for an **absent** value: they encode it as + // `get_raw_for_document_type(..).unwrap_or_default()` — an + // empty path segment — for user and system properties alike. + // Null must short-circuit here because the system-property + // encoders (`$updatedAt`, `$creatorId`, …) reject null before + // any encoding happens, which would make the stored + // empty-segment prefix unaddressable. + if value.is_null() { + return Ok(Vec::new()); + } document_type .serialize_value_for_key(&property.name, value, platform_version) .map_err(|e| { diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs b/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs index cacce78a1d5..4a477c98195 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs @@ -2381,6 +2381,166 @@ mod pinned_prefix { ); } + /// A **null** pin addresses the prefix subtree the write path + /// creates for an *absent* optional leading property: the walkers + /// encode a missing value as an empty path segment + /// (`get_raw_for_document_type(..).unwrap_or_default()`), so + /// `WHERE tag == null` must resolve to that empty segment — read, + /// proof, and client-side verification all reconstructing the same + /// stored path. Documents that *do* carry a tag live under their + /// own prefix and must not leak into the null prefix's ranking. + #[test] + fn a_null_pin_addresses_the_absent_value_prefix() { + const TAGGED_DOCTYPE: &str = "taggedGrade"; + let (drive, contract) = setup_grades_compound_ranked(); + let pv = platform_version(); + let document_type = contract + .document_type_for_name(TAGGED_DOCTYPE) + .expect("taggedGrade doctype exists"); + + // Tagless rows land under the empty-segment prefix; the + // "honors" row must stay in its own prefix. + for (i, (tag, class, grade)) in [ + (None, "math", 80i64), + (None, "math", 90), + (None, "science", 60), + (Some("honors"), "math", 100), + ] + .iter() + .enumerate() + { + let mut doc: Document = document_type + .random_document(Some(7000 + i as u64), pv) + .expect("random document"); + let mut props = BTreeMap::new(); + if let Some(tag) = tag { + props.insert("tag".to_string(), Value::Text(tag.to_string())); + } + props.insert(CLASS_PROPERTY.to_string(), Value::Text(class.to_string())); + props.insert("grade".to_string(), Value::I64(*grade)); + doc.set_properties(props); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo((&doc, None)), + owner_id: None, + }, + contract: &contract, + document_type, + }, + false, + BlockInfo::default(), + true, + None, + pv, + None, + ) + .expect("expected to insert a tagged grade document"); + } + + let null_pin = vec![WhereClause { + field: "tag".to_string(), + operator: WhereOperator::Equal, + value: Value::Null, + }]; + let group_by = vec![CLASS_PROPERTY.to_string()]; + let order_by = vec![OrderClause { + field: "grade".to_string(), + ascending: false, + }]; + let request = |prove: bool| DocumentRankedRequest { + contract: &contract, + document_type, + group_by: &group_by, + select: SelectProjection::avg("grade"), + having: &[], + order_by: &order_by, + where_clauses: &null_pin, + limit: Some(2), + offset: None, + has_start_at: false, + prove, + }; + + let page = match drive + .execute_document_ranked_request(request(false), None, pv) + .expect("the null-pinned read succeeds") + { + DocumentRankedResponse::Entries(page) => page, + DocumentRankedResponse::Proof(_) => panic!("expected entries, got a proof"), + }; + assert_eq!( + page.entries, + vec![ + RankedEntry { + key: b"math".to_vec(), + value: RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(170, 2)), + }, + RankedEntry { + key: b"science".to_vec(), + value: RankedEntryValue::AvgFixedPoint(compute_avg_fixed_point(60, 1)), + }, + ], + "the tagless ranking: math 85 then science 60 — the honors row \ + (math 100) must not leak into the null prefix" + ); + + // Proof round trip through the shared resolver, so the verifier + // reconstructs the same empty-segment path the prover walked. + let proof = match drive + .execute_document_ranked_request(request(true), None, pv) + .expect("the null-pinned prove succeeds") + { + DocumentRankedResponse::Proof(proof) => proof, + DocumentRankedResponse::Entries(_) => panic!("expected a proof, got entries"), + }; + let mode = detect_ranked_mode( + &SelectProjection::avg("grade"), + &group_by, + &[], + &order_by, + &null_pin, + RankedPaginationInputs { + limit: Some(2), + offset: None, + has_start_at: false, + }, + pv, + ) + .expect("the null-pinned case is well-formed"); + let query = resolve_ranked_query_for_mode( + contract.id_ref().to_buffer(), + document_type, + TAGGED_DOCTYPE.to_string(), + contract + .document_types() + .get(TAGGED_DOCTYPE) + .expect("taggedGrade doctype exists") + .indexes(), + &mode, + pv, + ) + .expect("the compound index covers the null-pinned request"); + assert_eq!( + query.equality_prefix_values, + vec![Vec::::new()], + "a null pin must encode as the write path's empty segment" + ); + let (root_hash, verified) = query + .verify_ranked_top_k_proof(&proof, pv) + .expect("the null-pinned proof must verify"); + assert_eq!(verified.entries, page.entries); + assert_eq!( + root_hash, + drive + .grove + .root_hash(None, &pv.drive.grove_version) + .unwrap() + .expect("root hash must be readable"), + ); + } + /// An unpinned request over the compound-only contract has no /// covering index — there is no global cross-prefix ordering to /// serve, so the rejection names the missing coverage. diff --git a/packages/rs-drive/tests/supporting_files/contract/grades/grades-compound-ranked-contract.json b/packages/rs-drive/tests/supporting_files/contract/grades/grades-compound-ranked-contract.json index 364d13703de..df4179096ce 100644 --- a/packages/rs-drive/tests/supporting_files/contract/grades/grades-compound-ranked-contract.json +++ b/packages/rs-drive/tests/supporting_files/contract/grades/grades-compound-ranked-contract.json @@ -12,8 +12,12 @@ { "name": "byIdentityAndClass", "properties": [ - { "identityId": "asc" }, - { "class": "asc" } + { + "identityId": "asc" + }, + { + "class": "asc" + } ], "averageable": "grade", "rangeAverageable": true, @@ -47,6 +51,50 @@ "grade" ], "additionalProperties": false + }, + "taggedGrade": { + "type": "object", + "documentsMutable": false, + "canBeDeleted": false, + "indices": [ + { + "name": "byTagAndClass", + "properties": [ + { + "tag": "asc" + }, + { + "class": "asc" + } + ], + "averageable": "grade", + "rangeAverageable": true, + "rankedAverageable": true + } + ], + "properties": { + "tag": { + "type": "string", + "maxLength": 32, + "position": 0 + }, + "class": { + "type": "string", + "maxLength": 32, + "position": 1 + }, + "grade": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "position": 2 + } + }, + "required": [ + "class", + "grade" + ], + "additionalProperties": false } } -} +} \ No newline at end of file diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/v3.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/v3.rs index 74c2e14564a..6b0098a97c4 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/v3.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_query_versions/v3.rs @@ -16,9 +16,11 @@ use crate::version::drive_abci_versions::drive_abci_query_versions::{ /// secondary. Everything else — including multi-clause `having` and /// `having` on a select with no ranked axis — keeps the v1 behavior. /// -/// Same mixed-network rationale as the v1 → v2 flip: earlier protocol -/// versions keep the v2 table and keep rejecting the shape, so nodes -/// agree until the upgrade carries. The wire surface is unchanged — +/// Mixed-network safety comes from the shipped tables: protocol +/// versions 1–11 select `DRIVE_ABCI_QUERY_VERSIONS_V0`, and versions +/// 12–13 select `DRIVE_ABCI_QUERY_VERSIONS_V1`. Both use helper +/// version 0 and reject ranked and `HAVING` shapes, so nodes agree +/// until the PV14 upgrade carries. The wire surface is unchanged — /// `GetDocumentsRequestV1.having` has been wire-stable since the v1 /// document query, and the response reuses the additive /// `ResultData.ranked` entries shape (with `skipped` unset, since a diff --git a/packages/rs-sdk/src/platform/documents/document_having_entries.rs b/packages/rs-sdk/src/platform/documents/document_having_entries.rs index 2d8785fd610..6b6522e00b0 100644 --- a/packages/rs-sdk/src/platform/documents/document_having_entries.rs +++ b/packages/rs-sdk/src/platform/documents/document_having_entries.rs @@ -310,9 +310,10 @@ mod tests { } } - /// The generic FromProof guard in drive-proof-verifier must not be - /// reachable from the SDK path: this impl (on `DocumentQuery`) is - /// the one `fetch` resolves, and it runs the real verification. + /// HAVING limits are a hard inclusive range, `1..=100`: `0` (the + /// unset sentinel) and anything above `MAX_HAVING_LIMIT` are + /// rejected client side rather than clamped, because the limit is + /// echoed in the proof envelope and re-checked by the verifier. #[test] fn limit_is_required_and_capped_client_side() { for limit in [0u32, 101] { From 650416f5e8197f01ffb682916e4b6495005f4ef7 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 21:29:41 +0700 Subject: [PATCH 12/12] refactor(dpp): give the ranked prefix-overlap rule its own file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validate_no_ranked_prefix_overlap moves from v3/mod.rs into v3/ranked_prefix_overlap.rs — one frozen unit of generation-3 grammar per file, same layout rationale as the mode-detection version modules. No behavior change. Co-Authored-By: Claude Fable 5 --- .../class_methods/try_from_schema/v3/mod.rs | 86 ++---------------- .../v3/ranked_prefix_overlap.rs | 90 +++++++++++++++++++ 2 files changed, 95 insertions(+), 81 deletions(-) create mode 100644 packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/ranked_prefix_overlap.rs diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs index 92a8b13fcde..1f449468155 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/mod.rs @@ -15,15 +15,13 @@ //! index-key length ceilings, and the constants they are derived from. use crate::data_contract::config::DataContractConfig; -use crate::data_contract::document_type::class_methods::consensus_or_protocol_data_contract_error; -// The prefix-overlap rule below runs on every parse path; only the -// key-length rule is validation-only. +// Only the ranked key-length rule below names these, and it is validation-only. +#[cfg(feature = "validation")] use crate::data_contract::document_type::index::Index; #[cfg(feature = "validation")] use crate::data_contract::document_type::property::DocumentPropertyType; use crate::data_contract::document_type::v2::DocumentTypeV2; use crate::data_contract::document_type::DocumentType; -use crate::data_contract::errors::DataContractError; use crate::data_contract::{TokenConfiguration, TokenContractPosition}; use crate::validation::operations::ProtocolValidationOperation; use crate::version::PlatformVersion; @@ -36,6 +34,9 @@ use crate::consensus::basic::data_contract::InvalidIndexedPropertyConstraintErro use super::common; +mod ranked_prefix_overlap; +use ranked_prefix_overlap::validate_no_ranked_prefix_overlap; + /// grovedb's ceiling on the key of an entry stored directly under an /// *indexed* tree's primary when the tree carries only the Count and/or Sum /// axes (`grovedb::operations::indexed_tree::MAX_CIDX_ITEM_KEY_LEN`). @@ -190,83 +191,6 @@ fn validate_ranked_index_property_key_length( ))) } -/// Rejects the one compound-ranked shape the storage layer cannot lay -/// out: a compound ranked index whose **full leading prefix** also -/// terminates a separate countable and/or summable index. -/// -/// A ranked flag on a compound index `[p1, …, pn]` puts an indexed tree -/// at each prefix's terminal `pn` property-name level — inside the value -/// trees of the `[p1, …, pn-1]` level. When another countable/summable -/// index terminates at exactly that prefix, those value trees are -/// aggregating (`CountTree` / `SumTree` / …), and every continuation -/// subtree inside them must be wrapped in a `NonCounted` / `NotSummed` -/// shell so its contents don't pollute the prefix index's aggregates. -/// grovedb structurally rejects that shell around an indexed tree — the -/// wrapper would neutralize the very aggregates the ranking indexes — -/// so the write path fails closed at document insert. Rejecting the -/// contract here surfaces the conflict at registration instead. -/// -/// Only the **exact** `n-1` prefix conflicts. An aggregating index -/// terminating at a shorter prefix wraps a plain intermediate -/// property-name tree (fine), and one extending *past* the ranked -/// terminal lives inside the indexed tree's value trees, which the -/// storage layer supports (see rs-drive's -/// `ranked_terminator_with_a_compound_continuation_gets_both_treatments`). -/// -/// Property comparison is by name, positionally: the merged index-level -/// tree keys sub-levels by property name in declaration order, so -/// `[a, b]` and `[b, a]` never share a level and cannot conflict. -/// -/// Unconditional (not gated on `full_validation`): the same structural -/// impossibility must reject the contract on every parse path — a -/// contract admitted through a non-validating parse would brick the -/// first document insert under the ranked index. -fn validate_no_ranked_prefix_overlap( - indices: &BTreeMap, -) -> Result<(), ProtocolError> { - for ranked in indices.values() { - let is_ranked = - ranked.ranked_countable || ranked.ranked_summable || ranked.ranked_averageable; - if !is_ranked || ranked.properties.len() < 2 { - continue; - } - let prefix = &ranked.properties[..ranked.properties.len() - 1]; - for other in indices.values() { - if other.name == ranked.name { - continue; - } - let terminates_at_prefix = other.properties.len() == prefix.len() - && other - .properties - .iter() - .zip(prefix.iter()) - .all(|(a, b)| a.name == b.name); - let aggregates = other.countable.is_countable() || other.summable.is_some(); - if terminates_at_prefix && aggregates { - return Err(consensus_or_protocol_data_contract_error( - DataContractError::InvalidContractStructure(format!( - "compound ranked index `{}` conflicts with index `{}`: the ranked \ - index's leading prefix [{}] also terminates a countable/summable \ - index, so the ranked terminal tree would sit inside aggregating \ - value trees and need a NonCounted/NotSummed shell — which the \ - storage layer rejects for indexed trees because the wrapper would \ - neutralize the aggregates the ranking indexes. Drop the ranked \ - flags, or drop the aggregate flags from the prefix index", - ranked.name, - other.name, - prefix - .iter() - .map(|p| p.name.as_str()) - .collect::>() - .join(", "), - )), - )); - } - } - } - Ok(()) -} - /// The [`common::RankedIndexKeyLengthCheck`] generation 3 runs on every indexed /// property, resolved at compile time. /// diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/ranked_prefix_overlap.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/ranked_prefix_overlap.rs new file mode 100644 index 00000000000..ff182ae52fa --- /dev/null +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/ranked_prefix_overlap.rs @@ -0,0 +1,90 @@ +//! The generation-3 [`super::common::RankedIndexStructureCheck`]: the cross-index +//! structural rule the ranked grammar adds on top of the shared parse core. +//! +//! Its own file for the same reason the mode-detection versions have their +//! own files: the rule is one frozen unit of generation-3 grammar, and a +//! later generation that needs a different structural rule supplies its own +//! callback instead of editing this one. + +use crate::data_contract::document_type::class_methods::consensus_or_protocol_data_contract_error; +use crate::data_contract::document_type::index::Index; +use crate::data_contract::errors::DataContractError; +use crate::ProtocolError; +use std::collections::BTreeMap; + +/// Rejects the one compound-ranked shape the storage layer cannot lay +/// out: a compound ranked index whose **full leading prefix** also +/// terminates a separate countable and/or summable index. +/// +/// A ranked flag on a compound index `[p1, …, pn]` puts an indexed tree +/// at each prefix's terminal `pn` property-name level — inside the value +/// trees of the `[p1, …, pn-1]` level. When another countable/summable +/// index terminates at exactly that prefix, those value trees are +/// aggregating (`CountTree` / `SumTree` / …), and every continuation +/// subtree inside them must be wrapped in a `NonCounted` / `NotSummed` +/// shell so its contents don't pollute the prefix index's aggregates. +/// grovedb structurally rejects that shell around an indexed tree — the +/// wrapper would neutralize the very aggregates the ranking indexes — +/// so the write path fails closed at document insert. Rejecting the +/// contract here surfaces the conflict at registration instead. +/// +/// Only the **exact** `n-1` prefix conflicts. An aggregating index +/// terminating at a shorter prefix wraps a plain intermediate +/// property-name tree (fine), and one extending *past* the ranked +/// terminal lives inside the indexed tree's value trees, which the +/// storage layer supports (see rs-drive's +/// `ranked_terminator_with_a_compound_continuation_gets_both_treatments`). +/// +/// Property comparison is by name, positionally: the merged index-level +/// tree keys sub-levels by property name in declaration order, so +/// `[a, b]` and `[b, a]` never share a level and cannot conflict. +/// +/// Unconditional (not gated on `full_validation`): the same structural +/// impossibility must reject the contract on every parse path — a +/// contract admitted through a non-validating parse would brick the +/// first document insert under the ranked index. +pub(super) fn validate_no_ranked_prefix_overlap( + indices: &BTreeMap, +) -> Result<(), ProtocolError> { + for ranked in indices.values() { + let is_ranked = + ranked.ranked_countable || ranked.ranked_summable || ranked.ranked_averageable; + if !is_ranked || ranked.properties.len() < 2 { + continue; + } + let prefix = &ranked.properties[..ranked.properties.len() - 1]; + for other in indices.values() { + if other.name == ranked.name { + continue; + } + let terminates_at_prefix = other.properties.len() == prefix.len() + && other + .properties + .iter() + .zip(prefix.iter()) + .all(|(a, b)| a.name == b.name); + let aggregates = other.countable.is_countable() || other.summable.is_some(); + if terminates_at_prefix && aggregates { + return Err(consensus_or_protocol_data_contract_error( + DataContractError::InvalidContractStructure(format!( + "compound ranked index `{}` conflicts with index `{}`: the ranked \ + index's leading prefix [{}] also terminates a countable/summable \ + index, so the ranked terminal tree would sit inside aggregating \ + value trees and need a NonCounted/NotSummed shell — which the \ + storage layer rejects for indexed trees because the wrapper would \ + neutralize the aggregates the ranking indexes. Drop the ranked \ + flags, or drop the aggregate flags from the prefix index", + ranked.name, + other.name, + prefix + .iter() + .map(|p| p.name.as_str()) + .collect::>() + .join(", "), + )), + )); + } + } + } + Ok(()) +}