From d2ba6d0d1493a5f82d05f1f357936063a9446e7b Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sat, 1 Aug 2026 23:35:52 -0700 Subject: [PATCH 1/6] DRILL-8552: Add Storage Plugin for Apache Accumulo This PR introduces a new storage plugin for Apache Accumulo, enabling Drill to query Accumulo tables using standard SQL. Features: - Full SQL query support for Accumulo tables - Dynamic schema discovery (column families as MAPs) - Filter pushdown (row key ranges to Accumulo Range scans) - Projection pushdown (column family/qualifier selection) - Limit pushdown (early scan termination) - Sort pushdown (ORDER BY row_key uses natural ordering) Authentication modes: - PASSWORD: Username/password authentication - KERBEROS + SHARED_USER: Service principal for all queries - KERBEROS + USER_IMPERSONATION: Delegation tokens for per-user identity - USER_TRANSLATION: Per-user Accumulo credentials from CredentialsProvider Key components: - AccumuloStoragePlugin/Config: Plugin configuration and lifecycle - AccumuloConnectionManager: Centralized auth and client management - AccumuloGroupScan/SubScan: Distributed scan planning - AccumuloRecordReader: Data reading and vector population - AccumuloPushFilterIntoScan: Filter pushdown optimizer rule - AccumuloPushSortIntoScan: Sort pushdown optimizer rule - DelegationTokenInfo: Serializable token wrapper for distributed execution Tested with Accumulo 2.1.4 LTS. Co-Authored-By: Claude Opus 4.5 --- contrib/pom.xml | 1 + contrib/storage-accumulo/DESIGN.md | 1018 +++++++++++++++++ contrib/storage-accumulo/README.md | 373 ++++++ contrib/storage-accumulo/pom.xml | 127 ++ .../exec/store/accumulo/AccumuloAuthType.java | 52 + .../AccumuloCompareFunctionsProcessor.java | 170 +++ .../accumulo/AccumuloConnectionManager.java | 518 +++++++++ .../store/accumulo/AccumuloFilterBuilder.java | 324 ++++++ .../store/accumulo/AccumuloGroupScan.java | 323 ++++++ .../accumulo/AccumuloPushFilterIntoScan.java | 177 +++ .../accumulo/AccumuloPushSortIntoScan.java | 142 +++ .../store/accumulo/AccumuloRecordReader.java | 473 ++++++++ .../accumulo/AccumuloScanBatchCreator.java | 132 +++ .../exec/store/accumulo/AccumuloScanSpec.java | 295 +++++ .../store/accumulo/AccumuloSchemaFactory.java | 103 ++ .../store/accumulo/AccumuloStoragePlugin.java | 200 ++++ .../accumulo/AccumuloStoragePluginConfig.java | 414 +++++++ .../exec/store/accumulo/AccumuloSubScan.java | 174 +++ .../store/accumulo/AccumuloTypeConverter.java | 341 ++++++ .../store/accumulo/DelegationTokenInfo.java | 187 +++ .../accumulo/DrillAccumuloConstants.java | 64 ++ .../store/accumulo/DrillAccumuloTable.java | 140 +++ .../accumulo/schema/AccumuloColumnType.java | 210 ++++ .../schema/AccumuloSchemaProvider.java | 89 ++ .../exec/store/accumulo/schema/ColumnDef.java | 146 +++ .../schema/MetadataTableSchemaProvider.java | 293 +++++ .../store/accumulo/schema/TableSchema.java | 193 ++++ .../resources/bootstrap-storage-plugins.json | 12 + .../src/main/resources/drill-module.conf | 36 + .../accumulo/AccumuloBasicQueryTest.java | 93 ++ .../accumulo/AccumuloFilterBuilderTest.java | 209 ++++ .../AccumuloIntegrationTestsSuite.java | 233 ++++ .../accumulo/AccumuloKerberosConfigTest.java | 263 +++++ .../accumulo/AccumuloLimitPushdownTest.java | 228 ++++ .../AccumuloProjectionPushdownTest.java | 228 ++++ .../AccumuloPushdownIntegrationTest.java | 238 ++++ .../store/accumulo/AccumuloScanSpecTest.java | 196 ++++ .../accumulo/AccumuloSortPushdownTest.java | 182 +++ .../AccumuloStoragePluginConfigTest.java | 350 ++++++ .../store/accumulo/AccumuloTestUtils.java | 220 ++++ .../store/accumulo/AccumuloTestsSuite.java | 56 + .../accumulo/AccumuloTypeConverterTest.java | 270 +++++ .../exec/store/accumulo/BaseAccumuloTest.java | 110 ++ .../accumulo/DelegationTokenInfoTest.java | 187 +++ .../accumulo/DrillAccumuloConstantsTest.java | 74 ++ .../schema/AccumuloColumnTypeTest.java | 123 ++ .../accumulo/schema/TableSchemaTest.java | 192 ++++ distribution/pom.xml | 5 + distribution/src/assemble/component.xml | 1 + 49 files changed, 10185 insertions(+) create mode 100644 contrib/storage-accumulo/DESIGN.md create mode 100644 contrib/storage-accumulo/README.md create mode 100644 contrib/storage-accumulo/pom.xml create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloAuthType.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloCompareFunctionsProcessor.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloConnectionManager.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilder.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloPushFilterIntoScan.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloPushSortIntoScan.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloScanBatchCreator.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloScanSpec.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloSchemaFactory.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePlugin.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePluginConfig.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloSubScan.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloTypeConverter.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DelegationTokenInfo.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloConstants.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloTable.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/AccumuloColumnType.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/AccumuloSchemaProvider.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/ColumnDef.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/MetadataTableSchemaProvider.java create mode 100644 contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/TableSchema.java create mode 100644 contrib/storage-accumulo/src/main/resources/bootstrap-storage-plugins.json create mode 100644 contrib/storage-accumulo/src/main/resources/drill-module.conf create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloBasicQueryTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilderTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloIntegrationTestsSuite.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloKerberosConfigTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloLimitPushdownTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloProjectionPushdownTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloPushdownIntegrationTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloScanSpecTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSortPushdownTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePluginConfigTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestUtils.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestsSuite.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTypeConverterTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/DelegationTokenInfoTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/DrillAccumuloConstantsTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/schema/AccumuloColumnTypeTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/schema/TableSchemaTest.java diff --git a/contrib/pom.xml b/contrib/pom.xml index d84f7c26850..81875426b52 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -59,6 +59,7 @@ format-spss format-syslog format-xml + storage-accumulo storage-cassandra storage-drill storage-druid diff --git a/contrib/storage-accumulo/DESIGN.md b/contrib/storage-accumulo/DESIGN.md new file mode 100644 index 00000000000..ab602bdb562 --- /dev/null +++ b/contrib/storage-accumulo/DESIGN.md @@ -0,0 +1,1018 @@ +# Apache Accumulo Storage Plugin for Drill - Design Document + +**Status**: Design Review +**Target Implementation**: Option A (High Abstraction) with extensibility for Option B +**Target Accumulo Version**: 2.1.x LTS +**Target Drill Version**: Latest (master branch) + +--- + +## 1. Executive Summary + +This document describes the architecture for a production-grade Accumulo storage plugin for Apache Drill with: + +- **Option A (Primary)**: High-level abstraction treating Accumulo as a SQL-queryable table store +- **Maximum pushdowns**: Filter, column projection, limit pushdown +- **Thorough testing**: Unit tests + integration tests using MiniAccumuloCluster +- **Future extensibility**: Clear design points for Option B (low abstraction / advanced iterators) + +--- + +## 2. Architecture Overview + +### 2.1 System Design Diagram + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Drill Query Engine │ +│ (Calcite Planner, etc.) │ +└────────────────────────┬────────────────────────────────────────┘ + │ + ┌───────────────┴────────────────┐ + │ AccumuloStoragePlugin │ + │ (Lifecycle, Schema Mgmt) │ + └───────────────┬────────────────┘ + │ + ┌───────────────┴───────────────────┐ + │ Optimizer Rules Layer │ + ├─────────────────────────────────┤ + │ • FilterPushDownRule │ + │ • ProjectionPushDownRule │ + │ • LimitPushDownRule │ + └──────────────┬────────────────────┘ + │ + ┌──────────────┴───────────────────┐ + │ Physical Plan Layer │ + ├──────────────────────────────────┤ + │ • AccumuloGroupScan │ + │ • AccumuloSubScan (per tablet) │ + │ • AccumuloRecordReader │ + └──────────────┬────────────────────┘ + │ + ┌──────────────┴───────────────────┐ + │ Accumulo Client Layer │ + ├──────────────────────────────────┤ + │ • AccumuloClient (singleton) │ + │ • Scanner/BatchScanner creation │ + │ • Iterator construction (internal)│ + └──────────────┬────────────────────┘ + │ + ┌──────────────┴───────────────────┐ + │ Accumulo Cluster │ + │ (ZooKeeper, TabletServers, etc.) │ + └───────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────┐ +│ SCHEMA MANAGEMENT SUBSYSTEM (Option A) │ +├─────────────────────────────────────────────────────────────────┤ +│ • AccumuloSchemaFactory: Table/schema discovery │ +│ • AccumuloSchemaProvider: (Interface for future Option B) │ +│ • DefaultSchemaProvider: Implements metadata table approach │ +│ • ScanSamplingProvider: Fallback schema inference (future) │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 3. Component Architecture + +### 3.1 Core Components (Required for Both Options A and B) + +#### **AccumuloStoragePluginConfig** +``` +Responsibilities: +├── Store connection parameters +│ ├── ZooKeeper quorum +│ ├── Accumulo instance name +│ ├── Username/password +│ └── Optional: Custom iterator library paths +├── Serialize/deserialize via Jackson +├── Provide equals/hashCode for caching +└── Extension point: Custom iterator configuration (Option B) +``` + +#### **AccumuloStoragePlugin** +``` +Responsibilities: +├── Lifecycle management +│ ├── Initialize AccumuloClient (singleton/pooled) +│ ├── Close client on shutdown +│ └── Thread-safe connection reuse +├── Schema registration +│ ├── Register Calcite schema with table discovery +│ ├── Use AccumuloSchemaProvider (pluggable) +│ └── Support multiple schema discovery strategies +├── Optimizer rule registration +│ ├── Return filter pushdown rule +│ ├── Return projection pushdown rule +│ ├── Return limit pushdown rule +│ └── Return sort rules (if applicable) +└── Extension point: Register custom iterator rules (Option B) +``` + +#### **AccumuloScanSpec** (Logical Scan Selection) +``` +Responsibilities: +├── Store scan parameters +│ ├── Table name +│ ├── Row key range (start, stop, inclusive flags) +│ ├── Pushed-down filter (as serializable expression) +│ ├── Projected column families/qualifiers +│ └── Limit value (if pushed down) +├── Serialize via Jackson for distributed execution +└── Extension point: Custom iterator spec (Option B) +``` + +#### **AccumuloGroupScan** (Logical→Physical Planning) +``` +Responsibilities: +├── Fragment scan across tablets +│ ├── Discover tablet ranges from Accumulo metadata +│ ├── Map tablets to endpoints (TabletServer affinity) +│ └── Create SubScan for each tablet +├── Modifiable by optimizer rules +│ ├── Support cloning for filter/projection modifications +│ ├── Track what's been pushed down (flags for idempotency) +│ └── Fall back to client-side filtering if needed +├── Provide scan statistics +│ ├── Row count estimates +│ └── Memory/data size estimates +└── Extension point: Custom iterator serialization (Option B) +``` + +#### **AccumuloSubScan** (Physical Scan on Single Tablet) +``` +Responsibilities: +├── Represent scan on one tablet +├── Carry scan parameters for that partition +├── Be non-executable (converted to RecordReader) +└── Support physical visitor pattern for Drill execution +``` + +#### **AccumuloRecordReader** (Data Streaming) +``` +Responsibilities: +├── Read data from Accumulo +│ ├── Create Scanner with range/filters/projections +│ ├── Iterate through results +│ └── Convert to Drill ValueVectors +├── Handle schema discovery +│ ├── Infer types from first batch (Option A) +│ └── Apply configured schema (if provided) +├── Manage memory and batching +│ ├── Target record count per batch +│ ├── Respect max memory per batch +│ └── Handle column mapping (Accumulo row → Drill columns) +└── Extension point: Custom iterator integration (Option B) +``` + +### 3.2 Schema Management Subsystem (Pluggable) + +#### **AccumuloSchemaProvider** (Interface) +```java +interface AccumuloSchemaProvider { + TableSchema getTableSchema(AccumuloClient client, String tableName); + Set discoverTableNames(AccumuloClient client); +} +``` + +#### **DefaultMetadataTableProvider** (Option A - Recommended) +``` +Responsibilities: +├── Maintain schema metadata in Accumulo table +│ ├── Table name: "_schema" (configurable) +│ ├── Row key format: {table_name} +│ ├── Column families: metadata +│ └── Columns: family:qualifiers, type_mapping, etc. +├── Discover and cache table schemas +├── Support schema updates via special API +└── Graceful fallback to scan sampling if metadata not found +``` + +Format example: +``` +Row: "users_table" + metadata:columns → "name,age,email" + metadata:families → "cf1:name,cf1:age,cf1:email" + metadata:types → "STRING,INT,STRING" +``` + +#### **ScanSamplingProvider** (Fallback) +``` +Responsibilities: +├── Sample first N rows from table +├── Infer column structure from sampled data +├── Infer type hints (basic - STRING for all) +└── Use only if metadata table unavailable +``` + +--- + +## 4. Pushdown Strategy (Option A - High Abstraction) + +### 4.1 Filter Pushdown + +**Architecture:** +``` +SQL Filter (e.g., "WHERE age > 30 AND city = 'NYC'") + ↓ +Calcite RexNode (optimizer expression) + ↓ +Drill LogicalExpression (via DrillOptiq.toDrill) + ↓ +FilterToPushdownConverter (Accumulo-specific) + ├── Converts to Accumulo Scanner filters (if possible) + └── OR marks for client-side filtering (partial pushdown) + ↓ +AccumuloScanSpec (includes serialized filter or NONE) + ↓ +AccumuloRecordReader (applies filters) +``` + +**Supported Predicates (Option A):** +- Comparison operators: =, !=, <, >, <=, >= +- Logical operators: AND, OR (with limits - see below) +- Null checks: IS NULL, IS NOT NULL +- String operations: LIKE (basic patterns) +- Numeric ranges: BETWEEN + +**Limitations & Strategy:** +- Complex predicates (nested OR/AND beyond 2 levels, complex LIKE patterns) → client-side +- Non-scalar expressions (function calls, computed columns) → client-side +- **Partial pushdown**: If only part of filter can be pushed, keep FilterPrel in plan + +**Implementation Class: `AccumuloFilterPushdownRule`** +``` +Pattern match: Filter → Scan +├── Extract filter conditions +├── Try converting each condition to Scanner filter range +├── Track which conditions were pushed +├── If all pushed: Remove Filter prel +└── If partial: Keep Filter prel for client-side processing +``` + +### 4.2 Column Projection Pushdown + +**Architecture:** +``` +Projected Columns (e.g., "SELECT name, age FROM users") + ↓ +SchemaPath list in GroupScan + ↓ +Map to Accumulo column families/qualifiers + ↓ +ProjectionToCFQualifierConverter + ├── Parse Drill column names + ├── Map to Accumulo CF:CQ pairs + └── Create column projection spec + ↓ +Scanner.fetchColumnFamilies()/fetchColumns() + ↓ +TabletServer applies system iterator + ↓ +Unnecessary columns never transferred over network +``` + +**Implementation Class: `AccumuloProjectionPushdownRule`** +``` +Pattern match: Project → Scan +├── Extract projected columns from ProjectPrel +├── Map to Accumulo families/qualifiers +├── Update AccumuloGroupScan with column spec +├── Remove ProjectPrel (handled by scanner) +``` + +### 4.3 Limit Pushdown + +**Strategy (Partial):** +- Accumulo iterators don't natively support LIMIT +- **Implementation**: + - Store limit in AccumuloScanSpec + - RecordReader stops after limit rows received + - Saves compute, not network bandwidth (network bound for limit) + - Still beneficial: stops TabletServer iteration early + +**Implementation Class: `AccumuloLimitPushdownRule`** +``` +Pattern match: Limit → Scan +├── Extract limit value +├── Add to AccumuloScanSpec +├── Remove Limit prel (handled by RecordReader) +``` + +### 4.4 Sort/Order Pushdown + +**Architecture:** +``` +ORDER BY clause (e.g., "ORDER BY row_key, name") + ↓ +Calcite SortRel (sort operator) + ↓ +SortToScannerConverter +├── Extract sort columns +├── Check if sort can be satisfied by Accumulo's natural order +│ └── Natural order: Row Key (primary), then CF, then CQ +├── If matches: Mark GroupScan to use Scanner (sorted, single-threaded) +└── If doesn't match: Keep SortRel (client-side sort) + ↓ +AccumuloGroupScan with sort mode flag + ├── sort_mode = SCANNER (single-threaded, sorted) + └── sort_mode = BATCH_SCANNER (multi-threaded, unsorted - default) + ↓ +AccumuloRecordReader uses appropriate scanner type +``` + +**Trade-offs & Cost Analysis:** + +| Aspect | Unsorted (BatchScanner) | Sorted (Scanner) | +|--------|------------------------|------------------| +| **Parallelism** | Multi-threaded tablets | Single-threaded tablet scan | +| **Network Throughput** | High (parallel) | Lower (sequential) | +| **Sorted Result** | NO | YES (by row key) | +| **Memory** | Constant (streaming) | Constant (streaming) | +| **Best For** | Large scans without sort | Small result sets or required sort | + +**Supported Sort Patterns (Option A):** +1. **No ORDER BY** → Use BatchScanner (default, maximum parallelism) +2. **ORDER BY row_key** → Use Scanner (leverages Accumulo sort) +3. **ORDER BY row_key, column_family** → Use Scanner (Accumulo's secondary sort) +4. **ORDER BY row_key ASC** → Use Scanner +5. **Complex sort** (e.g., by non-key columns) → Keep SortRel (client-side) + +**Implementation Class: `AccumuloSortPushdownRule`** +``` +Pattern match: Sort → Scan +├── Extract sort columns and directions +├── Check if sort matches Accumulo's natural order +│ ├── All sort keys must be row_key (primary) +│ ├── All must be ASC (Accumulo sorts ascending) +│ └── No computed columns or expressions +├── If matches: +│ ├── Set GroupScan.useSortedScanner = true +│ ├── Remove SortRel from plan +│ └── Track in AccumuloGroupScan for cost estimation +├── If doesn't match: +│ └── Keep SortRel (optimizer will do client-side sort) +└── Cost estimate: Parallelism loss vs. sort savings +``` + +**Cost Estimation Logic:** +``` +If using Scanner (sorted, single-threaded): + cost = row_count / tablet_count // No parallelism benefit + +If using BatchScanner (unsorted, multi-threaded): + cost = row_count / tablet_count / parallel_factor + +Sort cost (if client-side): + cost += row_count * log(row_count) + +Optimizer chooses Scanner only if: + scanner_cost < batchscanner_cost + sort_cost +``` + +**Configuration Flag in AccumuloGroupScan:** +```java +private boolean sortedScannerRequired = false; // Set by optimizer rule +private List sortColumns; // For cost estimation + +@Override +public ScanStats getScanStats() { + // Adjust cost based on scanner type + double baseCost = calculateBaseCost(); + if (sortedScannerRequired) { + baseCost *= 1.5; // Penalty for losing parallelism + } + return new ScanStats(...); +} +``` + +**Limitations of Sort Pushdown:** +1. Only sort by row_key supported (not by column values) +2. Ascending only (Accumulo's natural order) +3. No multi-column sorts (would need complex key ordering) +4. Batch results from different tablets may still need client-side sorting if using BatchScanner + +### 4.5 Potential Future Pushdowns + +- **Aggregation pushdown**: Requires custom iterators (Option B) + - COUNT, SUM, etc. via server-side combiners + - Out of scope for Option A + - Will be enabled by Option B iterator framework + +--- + +## 5. Schema Discovery & Type Mapping (Option A) + +### 5.1 Table Structure in Drill + +**Assumption**: Each Accumulo table maps to a Drill table with this logical structure: + +``` +Accumulo Row (key) +├── Row ID → Drill column: "row_key" (BYTES) +└── For each configured column family: + ├── Column family data → Drill column (structure TBD) + ├── Could be: Separate columns per qualifier (flat) + └── Could be: MAP (nested) +``` + +**Design Decision for Option A:** +``` +Flat mapping (recommended for OLAP): + Accumulo row with CF "user_data" and qualifiers "name", "age" + + Maps to Drill columns: + ├── row_key (BYTES) - actual Accumulo row key + ├── user_data_name (VARCHAR) - CF:qualifier → column + └── user_data_age (INTEGER) - CF:qualifier → column +``` + +### 5.2 Schema Discovery Workflow + +``` +Drill startup or table reference + ↓ +AccumuloSchemaFactory.registerSchemas() + ↓ +Iterate over configured tables / discover via Accumulo + ↓ +For each table: getTableSchema() + ├── Check metadata table (DefaultMetadataTableProvider) + ├── If found: Parse schema definition + ├── If not: Fall back to scan sampling + └── Cache schema + ↓ +DrillAccumuloTable created with schema + ↓ +getRowType() → RelDataType for Calcite +``` + +### 5.3 Metadata Table Format (Option A - Recommended) + +**System Metadata Table**: `_drill_schema` (or configured name) + +**Row Key**: Actual Accumulo table name (e.g., "users_table") + +**Schema Structure**: +``` +Column Family: "metadata" + +Row: "users_table" +├── metadata:qualified_name → "users_table" (redundant, for clarity) +├── metadata:row_key_type → "BYTES" +├── metadata:column_definitions → JSON array +│ [ +│ {"family":"cf1", "qualifier":"name", "column_name":"name", "type":"VARCHAR"}, +│ {"family":"cf1", "qualifier":"age", "column_name":"age", "type":"INT"}, +│ {"family":"cf2", "qualifier":"email", "column_name":"email", "type":"VARCHAR"} +│ ] +└── metadata:updated → timestamp +``` + +**Alternative (simpler)**: Plain text format +``` +metadata:columns_csv → "cf1:name:VARCHAR,cf1:age:INT,cf2:email:VARCHAR" +``` + +--- + +## 6. Configuration & Connection Management + +### 6.1 Plugin Configuration File + +**Location**: `conf.d/accumulo.conf` or via REST API + +```json +{ + "type": "accumulo", + "enabled": true, + "config": { + "zooKeeper.quorum": "localhost:2181", + "instanceName": "accumulo", + "userName": "root", + "password": "password", + "clientTimeout": "30s", + "schemaProvider": "default", + "metadataTable": "_drill_schema", + "maxConnectionPoolSize": 10 + } +} +``` + +### 6.2 Connection Management + +``` +Plugin initialization + ↓ +Create AccumuloClient singleton via builder + ├── client = Accumulo.newClient() + │ .to(instanceName, zooKeepers) + │ .as(userName, password) + │ .build() + └── Store in plugin instance (thread-safe, reusable) + ↓ +Maintain reference count / lifecycle + ├── Close on plugin shutdown + └── Reuse for all scans + ↓ +RecordReaders share same client + ├── Create Scanner/BatchScanner from client + └── Automatic resource cleanup (try-with-resources) +``` + +--- + +## 7. Testing Strategy + +### 7.1 Test Infrastructure + +**Primary**: MiniAccumuloCluster (embedded Accumulo for unit/integration tests) + +```java +@BeforeClass +public static void setupCluster() { + File tmpDir = new File(System.getProperty("java.io.tmpdir"), "accumulo-test"); + miniCluster = new MiniAccumuloCluster(tmpDir, "password"); + miniCluster.start(); + + client = miniCluster.getAccumuloClient("root", new PasswordToken("password")); + // Create test tables, populate data +} + +@AfterClass +public static void shutdownCluster() throws Exception { + client.close(); + miniCluster.stop(); +} +``` + +**Advantages**: +- Full Accumulo functionality (iterators, tablets, etc.) +- No external infrastructure needed +- Reproduces production scenarios + +**Disadvantages**: +- Slower startup/shutdown than unit tests +- ZooKeeper process overhead +- Best for integration tests, not unit-level + +### 7.2 Test Categories & Coverage + +#### **Unit Tests** (Fast, mocked dependencies) +- `AccumuloStoragePluginConfigTest` + - Jackson serialization/deserialization + - Config validation + - Equality/hashcode + +- `AccumuloScanSpecTest` + - Spec serialization (distributed execution) + - Filter/projection spec encoding + +- `FilterPushdownConverterTest` + - RexNode → Accumulo filter conversion + - Partial pushdown scenarios + - Unsupported filter detection + +- `SchemaProviderTest` + - Metadata table parsing + - Schema cache behavior + - Fallback to sampling + +#### **Integration Tests** (MiniAccumuloCluster-based) + +- `BaseAccumuloTest` + - Setup/teardown MiniAccumuloCluster + - Utility methods for test data creation + +- `AccumuloPluginInitializationTest` + - Plugin initialization + - Schema discovery + - Table listing + +- `AccumuloQueryTest` (Extends BaseTestQuery - full Drill integration) + - Basic SELECT queries + - Filter pushdown verification (query plan inspection) + - Projection pushdown verification + - Limit pushdown verification + - Sort pushdown verification (Scanner vs BatchScanner selection) + - Complex queries (joins, aggregates - without pushdown) + - Error handling (missing tables, invalid credentials) + +- `AccumuloRecordReaderTest` + - Data type mapping + - Batch size handling + - Column projection + - Schema inference (fallback mode) + +- `AccumuloGroupScanTest` + - Tablet fragmentation + - Endpoint affinity mapping + - Scan statistics accuracy + +#### **Test Data Scenarios** + +``` +Setup: +├── Empty tables (edge case) +├── Single-row tables +├── Large tables (1M+ rows) +├── Tables with multiple column families +├── Tables with null values +├── Tables with different data types +│ ├── Strings +│ ├── Numbers (INT, BIGINT) +│ ├── Floats/doubles +│ └── Bytes/blobs +└── Tables with special characters in keys/values + +Queries: +├── SELECT * → all columns +├── SELECT specific columns → projection +├── WHERE conditions → filters +├── WHERE + ORDER BY +├── LIMIT +├── Combinations: project + filter + limit +└── Aggregates (COUNT, SUM) → no pushdown, client-side only +``` + +### 7.3 Test Coverage Goals + +- **Unit tests**: 80%+ code coverage (configs, utilities, converters) +- **Integration tests**: Critical paths (query execution, data retrieval, pushdown verification) +- **Edge cases**: Empty results, null values, type mismatches, connection failures + +--- + +## 8. Future Extension: Option B (Advanced/Power User Mode) + +### 8.1 Design for Option B Extensibility + +**Option B Goals:** +- Expose Accumulo iterators to Drill users +- Allow custom iterator specification +- Support advanced server-side aggregation/computation + +**Design Points for Future Option B:** + +#### **1. Iterator Configuration in Accumulo Config** +```json +{ + "type": "accumulo", + "iterators": { + "custom": { + "jar_path": "/path/to/custom-iterators.jar", + "class_prefix": "com.example.accumulo.iterators" + } + } +} +``` + +#### **2. Extended AccumuloScanSpec for Iterators** +```java +// Current (Option A): +public class AccumuloScanSpec { + private HBaseScanSpec scanSpec; + private Filter filter; + // ... +} + +// Future (Option B): +public class AccumuloScanSpec { + // ... Option A fields ... + + // Option B extension: + @JsonProperty("custom_iterators") + private List customIterators; // NEW + + @JsonProperty("iterator_options") + private Map iteratorOptions; // NEW +} + +class IteratorConfig { + String name; + String className; + int priority; + Map options; +} +``` + +#### **3. Option B Optimizer Rule: CustomIteratorPushdownRule** +``` +Pattern: Aggregate → Scan +├── Detect if aggregation can map to custom iterator +├── Check if iterator class available +├── Create IteratorConfig +├── Add to AccumuloScanSpec.customIterators +└── Remove Aggregate prel (handled server-side) +``` + +#### **4. Option B Configuration in SQL (Future)** +```sql +-- Hypothetical future syntax: +SELECT * FROM accumulo.users_table +WITH (iterator_name = 'custom_agg', iterator_class = '...') +WHERE age > 30 +``` + +#### **5. Table Schema Hints for Option B** +``` +Metadata table addition: + metadata:custom_iterators → JSON list of available iterators for this table + +Used for: +├── Query plan optimization (which iterators can be used) +└── User hints (which iterators are recommended) +``` + +#### **6. AccumuloRecordReader Option B Extension** +```java +// Current (Option A): +public class AccumuloRecordReader { + private Scanner scanner; + // Uses default iterators +} + +// Future (Option B): +public class AccumuloRecordReader { + private Scanner scanner; + + // NEW: + private void addCustomIterators(List iterators) { + for (IteratorConfig cfg : iterators) { + IteratorSetting settings = new IteratorSetting( + cfg.priority, cfg.name, cfg.className); + cfg.options.forEach(settings::addOption); + scanner.addScanIterator(settings); + } + } +} +``` + +### 8.2 Migration Path from Option A to Option B + +1. **Phase 1 (Current - Option A)**: High abstraction, filter/projection/limit pushdowns +2. **Phase 2 (Future - Option B prep)**: Support custom iterator configuration in plugin config +3. **Phase 3 (Future - Option B full)**: Implement iterator optimizer rules +4. **Phase 4 (Future - Option B advanced)**: Support SQL hints for iterator selection + +**Backward Compatibility**: Option A queries remain valid in Option B; new iterator features are opt-in. + +--- + +## 9. Implementation Roadmap + +### Phase 1: Core Infrastructure +- [ ] Pom.xml and module setup +- [ ] AccumuloStoragePluginConfig +- [ ] AccumuloStoragePlugin +- [ ] Basic connection management +- [ ] Tests for config/connection + +### Phase 2: Schema & Table Discovery +- [ ] AccumuloSchemaProvider interface +- [ ] DefaultMetadataTableProvider implementation +- [ ] AccumuloSchemaFactory +- [ ] DrillAccumuloTable +- [ ] Tests for schema discovery + +### Phase 3: Basic Scan Capability +- [ ] AccumuloScanSpec +- [ ] AccumuloGroupScan (tablet fragmentation) +- [ ] AccumuloSubScan +- [ ] AccumuloRecordReader (basic data reading) +- [ ] Tests: RecordReader, basic queries + +### Phase 4: Filter Pushdown +- [ ] FilterToPushdownConverter (RexNode → filter spec) +- [ ] AccumuloFilterPushdownRule +- [ ] Update AccumuloGroupScan for filter tracking +- [ ] Tests: Filter pushdown verification in query plans + +### Phase 5: Projection Pushdown +- [ ] ProjectionToCFQualifierConverter +- [ ] AccumuloProjectionPushdownRule +- [ ] Update RecordReader for column projection +- [ ] Tests: Projection pushdown verification + +### Phase 6: Limit & Sort Pushdown +- [ ] LimitPushdownRule +- [ ] RecordReader limit enforcement +- [ ] AccumuloSortPushdownRule (Scanner vs BatchScanner selection) +- [ ] Cost estimation for sort vs parallelism trade-off +- [ ] Update AccumuloGroupScan for sort mode +- [ ] Tests: Limit pushdown verification +- [ ] Tests: Sort pushdown verification (Scanner vs BatchScanner in plan) + +### Phase 7: Scan Statistics & Optimization +- [ ] Implement ScanStats calculation +- [ ] Tablet fragmentation cost estimation +- [ ] Row count estimates +- [ ] Cost-based optimizer integration + +### Phase 8: Integration Testing & Polish +- [ ] Comprehensive integration tests (queries, edge cases) +- [ ] Error handling and recovery +- [ ] Documentation and examples +- [ ] Performance tuning + +### Phase 9: Option B Design Prep (Future) +- [ ] Document iterator configuration in config schema +- [ ] Add Option B extension fields to AccumuloScanSpec (with ignore markers) +- [ ] Leave hook points in RecordReader for custom iterators +- [ ] No functional changes, just structure + +--- + +## 10. Key Design Principles + +### 10.1 SOLID Principles Applied + +**S - Single Responsibility** +- AccumuloStoragePlugin: Lifecycle only +- AccumuloGroupScan: Planning only +- AccumuloRecordReader: Data reading only +- Each converter class: Single conversion type + +**O - Open/Closed** +- AccumuloSchemaProvider interface allows new schema discovery strategies +- FilterPushdownConverter extensible for new filter types +- RecordReader designed for custom iterator injection (Option B) + +**L - Liskov Substitution** +- All providers implement AccumuloSchemaProvider contract +- All rules follow StoragePluginOptimizerRule pattern +- Drill interfaces implemented correctly + +**I - Interface Segregation** +- AccumuloSchemaProvider: Only schema methods +- Separate interfaces for converters (filter, projection) +- Plugin config separated from runtime state + +**D - Dependency Injection** +- RecordReader receives dependencies (client, spec, columns) +- Rules receive context and relationships +- No global state except plugin singleton + +### 10.2 Robustness Principles + +- **Partial pushdown**: Always safer than all-or-nothing +- **Graceful degradation**: Fall back to client-side processing +- **Error handling**: Clear exceptions, proper resource cleanup +- **Caching**: Schema cache with refresh mechanism +- **Idempotency**: Optimizer rules safe to run multiple times + +### 10.3 Performance Principles + +- **Avoid over-fragmentation**: Merge small tablets if beneficial +- **Endpoint affinity**: Prioritize data locality +- **Batch sizing**: Respect Drill's memory budgets +- **Network efficiency**: Project columns, push filters to server +- **Connection reuse**: Single client instance, pooled scanners + +--- + +## 11. Configuration Reference (Option A) + +### bootstrap-storage-plugins.json +```json +{ + "storage": { + "accumulo": { + "type": "accumulo", + "enabled": false, + "config": { + "zooKeeper": { + "quorum": "localhost:2181" + }, + "instance": { + "name": "accumulo" + }, + "auth": { + "principal": "root", + "token_type": "password", + "token": "password" + }, + "schema": { + "provider": "metadata_table", + "metadata_table": "_drill_schema" + } + } + } + } +} +``` + +### Example table schema in metadata table +``` +Table: _drill_schema +Row: "users" + metadata:columns_json → + [ + {"family":"cf1", "qualifier":"name", "column":"name", "type":"VARCHAR", "nullable":false}, + {"family":"cf1", "qualifier":"age", "column":"age", "type":"INT", "nullable":true}, + {"family":"cf1", "qualifier":"email", "column":"email", "type":"VARCHAR", "nullable":true} + ] +``` + +--- + +## 12. Dependencies + +### Maven Coordinates + +```xml + + + org.apache.accumulo + accumulo-core + 2.1.4 + + + + + org.apache.accumulo + accumulo-core + 2.1.4 + + + + + org.apache.accumulo + accumulo-minicluster + 2.1.4 + test + + + + + org.apache.drill.exec + drill-java-exec + provided + + + + + org.apache.drill + drill-common + provided + +``` + +--- + +## 13. Known Limitations & Trade-offs + +### Option A Limitations +1. **No custom iterator access**: Advanced Accumulo features not exposed +2. **Type inference**: Relies on metadata or sampling; limited automatic type detection +3. **Schema management**: Manual metadata maintenance required +4. **Sort optimization**: Complex due to Accumulo's scan model trade-offs + +### Accumulo Limitations (Not Plugin-specific) +1. **No built-in aggregation operators**: COUNT, SUM require custom iterators or client-side computation +2. **Sort incompatible with parallelism**: Can't use BatchScanner if sort order needed +3. **Schema-less nature**: Type consistency not enforced by Accumulo + +### Future Mitigations +- Option B for advanced users needing iterators +- Improved schema management tools/UI +- Performance benchmarks for sort trade-offs + +--- + +## 14. Success Criteria + +### Option A Completion +- [ ] All core components implemented and tested +- [ ] Filter pushdown working (verified in query plans) +- [ ] Projection pushdown working +- [ ] Limit pushdown working +- [ ] Sort pushdown working (Scanner vs BatchScanner selection) +- [ ] 80%+ unit test coverage +- [ ] Comprehensive integration tests passing +- [ ] Production-ready error handling +- [ ] Documentation complete +- [ ] Example queries/use cases documented + +### Quality Metrics +- No critical bugs in testing +- Query performance within expected range (compared to HBase plugin) +- Scan statistics accurate (±20% for row count estimates) +- All pushdowns verified by query plan inspection +- Sort optimization correctly chooses between Scanner (sorted) and BatchScanner (parallel) + +--- + +## 15. References & Resources + +- **Accumulo Documentation**: https://accumulo.apache.org/docs/2.x/ +- **Accumulo Client API**: https://accumulo.apache.org/docs/2.x/apidocs/ +- **Drill Plugin Architecture**: [HBase/Kudu plugins in codebase] +- **Drill Schema/Type System**: Calcite integration points +- **Iterator Development**: https://accumulo.apache.org/docs/2.x/development/iterators + +--- + +**Document Status**: Ready for Review +**Next Step**: Design review and approval, then proceed to Phase 1 implementation diff --git a/contrib/storage-accumulo/README.md b/contrib/storage-accumulo/README.md new file mode 100644 index 00000000000..5bc261c2d2b --- /dev/null +++ b/contrib/storage-accumulo/README.md @@ -0,0 +1,373 @@ +# Apache Accumulo Storage Plugin for Apache Drill + +This storage plugin enables Apache Drill to query Apache Accumulo tables using SQL. + +## Features + +- **Full SQL Support**: Query Accumulo tables using standard SQL syntax +- **Schema Discovery**: Automatic schema inference from Accumulo data +- **Pushdown Optimization**: Efficient query execution with multiple pushdown strategies: + - **Filter Pushdown**: Row key range filters translated to Accumulo Range scans + - **Projection Pushdown**: Column family/qualifier projections pushed to Accumulo Scanner + - **Limit Pushdown**: LIMIT clauses pushed to reduce data scanning + - **Sort Pushdown**: ORDER BY row_key uses Accumulo's natural ordering +- **Authentication**: Support for password and Kerberos authentication +- **User Impersonation**: Per-user query execution with delegation tokens + +## Requirements + +- Apache Drill 1.23.0 or later +- Apache Accumulo 2.1.x (tested with 2.1.4 LTS) +- Java 11 or later + +## Installation + +The Accumulo storage plugin is included in the Drill distribution. No additional installation is required. + +## Configuration + +### Password Authentication (Basic) + +Configure the plugin through the Drill Web UI (http://localhost:8047/storage) or via REST API: + +```json +{ + "type": "accumulo", + "zookeeperQuorum": "localhost:2181", + "instanceName": "accumulo", + "username": "root", + "password": "secret", + "enabled": true +} +``` + +### Kerberos Authentication + +For enterprise environments with Kerberos, configure the plugin with Kerberos authentication: + +#### Shared User Mode (Service Principal Only) + +All queries run as the service principal. Simple setup, but no per-user authorization. + +```json +{ + "type": "accumulo", + "zookeeperQuorum": "zk1:2181,zk2:2181,zk3:2181", + "instanceName": "accumulo", + "authenticationType": "KERBEROS", + "principal": "drill/drillserver.example.com@EXAMPLE.COM", + "keytabPath": "/etc/security/keytabs/drill.keytab", + "saslQop": "auth", + "authMode": "SHARED_USER", + "enabled": true +} +``` + +#### User Translation Mode (Per-User Credentials) + +Each Drill user has their own Accumulo credentials stored in the credentials provider. The plugin looks up credentials based on the query user. + +```json +{ + "type": "accumulo", + "zookeeperQuorum": "zk1:2181,zk2:2181,zk3:2181", + "instanceName": "accumulo", + "authMode": "USER_TRANSLATION", + "credentialsProvider": { + "credentialsProviderType": "PlainCredentialsProvider", + "credentials": {}, + "userCredentials": { + "alice": {"username": "accumulo_alice", "password": "alice_pass"}, + "bob": {"username": "accumulo_bob", "password": "bob_pass"} + } + }, + "enabled": true +} +``` + +#### User Impersonation Mode (Delegation Tokens) + +The service authenticates with Kerberos, then impersonates the Drill query user via delegation tokens. This enables per-user authorization and audit trails. + +```json +{ + "type": "accumulo", + "zookeeperQuorum": "zk1:2181,zk2:2181,zk3:2181", + "instanceName": "accumulo", + "authenticationType": "KERBEROS", + "principal": "drill/drillserver.example.com@EXAMPLE.COM", + "keytabPath": "/etc/security/keytabs/drill.keytab", + "saslQop": "auth-conf", + "accumuloServicePrimary": "accumulo", + "useDelegationTokens": true, + "authMode": "USER_IMPERSONATION", + "enabled": true +} +``` + +### Configuration Properties + +#### Connection Properties + +| Property | Description | Default | +|----------|-------------|---------| +| `zookeeperQuorum` | Comma-separated list of ZooKeeper servers (host:port) | Required | +| `instanceName` | Accumulo instance name | Required | + +#### Password Authentication Properties + +| Property | Description | Default | +|----------|-------------|---------| +| `username` | Accumulo username | Required for PASSWORD auth | +| `password` | Accumulo password | Required for PASSWORD auth | +| `credentialsProvider` | Alternative credential provider | null | + +#### Kerberos Authentication Properties + +| Property | Description | Default | +|----------|-------------|---------| +| `authenticationType` | Authentication type: `PASSWORD` or `KERBEROS` | `PASSWORD` | +| `principal` | Kerberos principal (e.g., `drill/host@REALM`) | Required for KERBEROS | +| `keytabPath` | Path to the Kerberos keytab file | Required for KERBEROS | +| `saslQop` | SASL Quality of Protection: `auth`, `auth-int`, `auth-conf` | `auth` | +| `accumuloServicePrimary` | Accumulo service principal primary name | `accumulo` | + +#### User Impersonation Properties + +| Property | Description | Default | +|----------|-------------|---------| +| `authMode` | Authorization mode: `SHARED_USER` or `USER_IMPERSONATION` | `SHARED_USER` | +| `useDelegationTokens` | Enable delegation tokens for distributed execution | `false` | + +#### Optional Properties + +| Property | Description | Default | +|----------|-------------|---------| +| `schemaMetadataTable` | Table for schema metadata | `_drill_schema` | +| `clientTimeout` | Client operation timeout (ms) | `30000` | +| `batchScannerThreads` | Number of batch scanner threads | `10` | + +### SASL Quality of Protection (QoP) + +| QoP Value | Description | +|-----------|-------------| +| `auth` | Authentication only (default) | +| `auth-int` | Authentication + integrity protection | +| `auth-conf` | Authentication + integrity + confidentiality (encryption) | + +For production environments, `auth-conf` is recommended for full encryption of data in transit. + +## Usage + +### Basic Queries + +```sql +-- Select all columns from a table +SELECT * FROM accumulo.`my_table`; + +-- Select specific columns +SELECT row_key, cf.column1, cf.column2 FROM accumulo.`my_table`; + +-- Select entire column family +SELECT personal FROM accumulo.`users`; +``` + +### Filter Queries + +```sql +-- Row key equality +SELECT * FROM accumulo.`my_table` WHERE row_key = 'row_001'; + +-- Row key range +SELECT * FROM accumulo.`my_table` +WHERE row_key >= 'row_100' AND row_key < 'row_200'; +``` + +### Limit Queries + +```sql +-- Limit results (pushed down to Accumulo) +SELECT * FROM accumulo.`my_table` LIMIT 100; +``` + +### Sort Queries + +```sql +-- Order by row key (uses Accumulo's natural ordering) +SELECT * FROM accumulo.`my_table` ORDER BY row_key ASC; +``` + +### Combined Queries + +```sql +-- Filter, project, sort, and limit +SELECT row_key, cf.name, cf.value +FROM accumulo.`my_table` +WHERE row_key >= 'row_100' +ORDER BY row_key ASC +LIMIT 50; +``` + +## Data Model + +### Row Key + +The Accumulo row key is exposed as a special column named `row_key` of type `VARBINARY`. + +### Column Families + +Each Accumulo column family is exposed as a Drill MAP type. Column qualifiers within a family become fields in the map: + +``` +Accumulo: row_001 -> personal:first_name = "John", personal:last_name = "Doe" +Drill: row_key = 'row_001', personal = {first_name: "John", last_name: "Doe"} +``` + +### Data Types + +All values are stored as `VARBINARY` by default. Use Drill's CAST functions for type conversion: + +```sql +SELECT + row_key, + CAST(cf.age AS INT) as age, + CAST(cf.salary AS DOUBLE) as salary +FROM accumulo.`employees`; +``` + +## Schema Metadata (Optional) + +For better schema support, you can store table schemas in a metadata table. Create a table `_drill_schema` (configurable) with the following structure: + +- Row key: table name +- Column family: `schema` +- Qualifiers: column definitions in format `column_family:qualifier:type` + +Example: +``` +_drill_schema -> employees -> schema:cf.name = "VARCHAR" + -> schema:cf.age = "INT" + -> schema:cf.salary = "DOUBLE" +``` + +## Kerberos Setup Guide + +### Prerequisites + +1. **Kerberos KDC**: A running Kerberos KDC with principals configured +2. **Accumulo with Kerberos**: Accumulo configured for Kerberos authentication +3. **Service Principal**: Create a principal for Drill (e.g., `drill/hostname@REALM`) +4. **Keytab**: Export the keytab for the Drill principal + +### Steps + +1. **Create Drill Service Principal** + ```bash + kadmin -q "addprinc -randkey drill/drillserver.example.com@EXAMPLE.COM" + kadmin -q "xst -k /etc/security/keytabs/drill.keytab drill/drillserver.example.com@EXAMPLE.COM" + ``` + +2. **Configure Drill Impersonation** (if using USER_IMPERSONATION mode) + + Enable impersonation in `drill-override.conf`: + ``` + drill.exec.impersonation.enabled: true + ``` + +3. **Configure Accumulo for Delegation Tokens** (if using USER_IMPERSONATION mode) + + Ensure Accumulo is configured to support delegation tokens. This typically requires: + - `general.kerberos.keytab` and `general.kerberos.principal` set in `accumulo.properties` + - Accumulo master configured for token generation + +4. **Configure the Storage Plugin** + + Use the Drill Web UI or REST API to configure the plugin with your Kerberos settings. + +### Troubleshooting Kerberos + +- **Check keytab validity**: `klist -kt /path/to/drill.keytab` +- **Test kinit**: `kinit -kt /path/to/drill.keytab drill/hostname@REALM` +- **Verify Accumulo connection**: Use Accumulo shell with Kerberos to verify connectivity +- **Check Drill logs**: Enable debug logging for `org.apache.drill.exec.store.accumulo` + +## Performance Considerations + +1. **Row Key Filters**: Always filter on `row_key` when possible - these filters are pushed down to Accumulo as Range scans. + +2. **Projection**: Select only the columns you need - column projections are pushed to Accumulo's `fetchColumn()` API. + +3. **Limit**: Use LIMIT when you only need a subset of rows - limits are pushed down to stop scanning early. + +4. **Batch Size**: The default batch size of 4000 rows works well for most queries. Adjust `batchScannerThreads` for parallel scanning. + +5. **User Impersonation**: When using delegation tokens, tokens are cached per-user with a 1-hour TTL to minimize overhead. + +## Troubleshooting + +### Connection Issues + +If you see connection errors, verify: +1. ZooKeeper is running and accessible +2. Accumulo instance name is correct +3. Credentials are valid +4. Network connectivity to ZooKeeper and Accumulo tablet servers + +### Kerberos Authentication Issues + +1. Verify keytab file exists and is readable +2. Check principal format matches the keytab +3. Ensure KDC is accessible from Drill nodes +4. Verify Accumulo SASL settings match Drill configuration + +### Query Performance + +For slow queries: +1. Check if row key filters can be added +2. Use EXPLAIN to verify pushdowns are working +3. Consider reducing batch size for memory-constrained environments + +## Development + +### Building + +```bash +mvn clean install -pl contrib/storage-accumulo -DskipTests +``` + +### Running Tests + +```bash +# Unit tests +mvn test -pl contrib/storage-accumulo + +# Integration tests (requires MiniAccumuloCluster) +mvn test -Dtest=AccumuloIntegrationTestsSuite -pl contrib/storage-accumulo + +# Kerberos tests (requires Kerberos-enabled cluster) +mvn test -pl contrib/storage-accumulo \ + -Ddrill.accumulo.kerberos.enabled=true \ + -Ddrill.accumulo.principal=drill/host@REALM \ + -Ddrill.accumulo.keytab=/path/to/keytab +``` + +## Authentication Modes Summary + +| Mode | Auth Type | Description | Use Case | +|------|-----------|-------------|----------| +| PASSWORD + SHARED_USER | Password | Username/password for all queries | Development, simple deployments | +| PASSWORD + USER_TRANSLATION | Password | Per-user Accumulo credentials from CredentialsProvider | Multi-user with separate Accumulo accounts | +| KERBEROS + SHARED_USER | Kerberos | Service principal for all queries | Enterprise, single service account | +| KERBEROS + USER_IMPERSONATION | Kerberos + Delegation Tokens | Service authenticates, then impersonates Drill user | Enterprise, per-user audit/authorization | + +## Future Enhancements + +Planned features for future releases: +- Custom Accumulo iterators for server-side processing +- Visibility/Authorization support +- Write support (INSERT/UPDATE/DELETE) +- Statistics-based cost estimation + +## License + +Apache License 2.0 diff --git a/contrib/storage-accumulo/pom.xml b/contrib/storage-accumulo/pom.xml new file mode 100644 index 00000000000..09d3417b2ad --- /dev/null +++ b/contrib/storage-accumulo/pom.xml @@ -0,0 +1,127 @@ + + + + 4.0.0 + + drill-contrib-parent + org.apache.drill.contrib + 1.23.0-SNAPSHOT + + + drill-storage-accumulo + + Drill : Contrib : Storage : Accumulo + + + 2.1.4 + **/AccumuloTestsSuite.class + + + + + + org.apache.drill.exec + drill-java-exec + ${project.version} + + + + + org.apache.accumulo + accumulo-core + ${accumulo.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + commons-logging + commons-logging + + + + + + + org.apache.drill.exec + drill-java-exec + tests + ${project.version} + test + + + org.apache.drill + drill-common + tests + ${project.version} + test + + + + + org.apache.accumulo + accumulo-minicluster + ${accumulo.version} + test + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ${accumulo.TestSuite} + + + + accumulo.test.root + ${project.build.directory}/data + + + logback.log.dir + ${project.build.directory}/surefire-reports + + + + + + + + diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloAuthType.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloAuthType.java new file mode 100644 index 00000000000..490c8e03488 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloAuthType.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import com.google.common.base.Strings; + +/** + * Authentication types supported by the Accumulo storage plugin. + */ +public enum AccumuloAuthType { + /** + * Username/password authentication (default). + * Uses Accumulo's PasswordToken for authentication. + */ + PASSWORD, + + /** + * Kerberos authentication using SASL. + * Uses Accumulo's KerberosToken for authentication. + * Requires a principal and keytab path to be configured. + */ + KERBEROS; + + /** + * Parses the authentication type from a string, with a default fallback. + * + * @param authType the string representation of the auth type + * @param defaultType the default type to use if authType is null or empty + * @return the parsed AccumuloAuthType + */ + public static AccumuloAuthType parseOrDefault(String authType, AccumuloAuthType defaultType) { + if (Strings.isNullOrEmpty(authType)) { + return defaultType; + } + return AccumuloAuthType.valueOf(authType.toUpperCase()); + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloCompareFunctionsProcessor.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloCompareFunctionsProcessor.java new file mode 100644 index 00000000000..7de16f491ec --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloCompareFunctionsProcessor.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.nio.charset.StandardCharsets; + +import org.apache.drill.common.FunctionNames; +import org.apache.drill.common.expression.CastExpression; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.ValueExpressions.IntExpression; +import org.apache.drill.common.expression.ValueExpressions.LongExpression; +import org.apache.drill.common.expression.ValueExpressions.QuotedString; +import org.apache.drill.common.expression.visitors.AbstractExprVisitor; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** + * Processor for comparison functions in filter expressions. + * + *

Extracts the field path and comparison value from expressions like: + *

    + *
  • row_key = 'value'
    • + *
  • row_key > 'start'
    • + *
  • row_key < 'end'
    • + *
+ */ +public class AccumuloCompareFunctionsProcessor + extends AbstractExprVisitor { + + private byte[] value; + private boolean success; + private SchemaPath path; + private String functionName; + + public static boolean isCompareFunction(String functionName) { + return COMPARE_FUNCTIONS_TRANSPOSE_MAP.containsKey(functionName); + } + + public static AccumuloCompareFunctionsProcessor createFunctionsProcessorInstance( + FunctionCall call) { + String functionName = call.getName(); + AccumuloCompareFunctionsProcessor evaluator = + new AccumuloCompareFunctionsProcessor(functionName); + + LogicalExpression nameArg = call.arg(0); + LogicalExpression valueArg = call.argCount() >= 2 ? call.arg(1) : null; + + if (valueArg != null) { + // Binary function (e.g., row_key = 'value') + if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) { + // Value on left side, field on right - swap and transpose function + LogicalExpression swapArg = valueArg; + valueArg = nameArg; + nameArg = swapArg; + evaluator.functionName = COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(evaluator.functionName); + } + evaluator.success = nameArg.accept(evaluator, valueArg); + } else if (call.arg(0) instanceof SchemaPath) { + // Unary function (IS NULL, IS NOT NULL) + evaluator.success = true; + evaluator.path = (SchemaPath) nameArg; + } + + return evaluator; + } + + public AccumuloCompareFunctionsProcessor(String functionName) { + this.success = false; + this.functionName = functionName; + } + + public byte[] getValue() { + return value; + } + + public boolean isSuccess() { + return success; + } + + public SchemaPath getPath() { + return path; + } + + public String getFunctionName() { + return functionName; + } + + @Override + public Boolean visitCastExpression(CastExpression e, LogicalExpression valueArg) + throws RuntimeException { + if (e.getInput() instanceof CastExpression || e.getInput() instanceof SchemaPath) { + return e.getInput().accept(this, valueArg); + } + return false; + } + + @Override + public Boolean visitUnknown(LogicalExpression e, LogicalExpression valueArg) + throws RuntimeException { + return false; + } + + @Override + public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) + throws RuntimeException { + if (valueArg instanceof QuotedString) { + this.value = ((QuotedString) valueArg).value.getBytes(StandardCharsets.UTF_8); + this.path = path; + return true; + } + if (valueArg instanceof IntExpression) { + this.value = String.valueOf(((IntExpression) valueArg).getInt()) + .getBytes(StandardCharsets.UTF_8); + this.path = path; + return true; + } + if (valueArg instanceof LongExpression) { + this.value = String.valueOf(((LongExpression) valueArg).getLong()) + .getBytes(StandardCharsets.UTF_8); + this.path = path; + return true; + } + return false; + } + + private static final ImmutableSet> VALUE_EXPRESSION_CLASSES; + static { + ImmutableSet.Builder> builder = ImmutableSet.builder(); + VALUE_EXPRESSION_CLASSES = builder + .add(QuotedString.class) + .add(IntExpression.class) + .add(LongExpression.class) + .build(); + } + + static final ImmutableMap COMPARE_FUNCTIONS_TRANSPOSE_MAP; + static { + ImmutableMap.Builder builder = ImmutableMap.builder(); + COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder + // Unary functions + .put(FunctionNames.IS_NOT_NULL, FunctionNames.IS_NOT_NULL) + .put(FunctionNames.IS_NULL, FunctionNames.IS_NULL) + // Binary functions - transpose for when value is on left + .put(FunctionNames.EQ, FunctionNames.EQ) + .put(FunctionNames.NE, FunctionNames.NE) + .put(FunctionNames.GE, FunctionNames.LE) + .put(FunctionNames.GT, FunctionNames.LT) + .put(FunctionNames.LE, FunctionNames.GE) + .put(FunctionNames.LT, FunctionNames.GT) + .build(); + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloConnectionManager.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloConnectionManager.java new file mode 100644 index 00000000000..9ef83e66512 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloConnectionManager.java @@ -0,0 +1,518 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.admin.DelegationTokenConfig; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.DelegationToken; +import org.apache.accumulo.core.client.security.tokens.KerberosToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.common.logical.StoragePluginConfig.AuthMode; +import org.apache.drill.exec.proto.UserBitShared.UserCredentials; +import org.apache.drill.exec.store.security.UsernamePasswordCredentials; +import org.apache.drill.exec.util.ImpersonationUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Strings; + +/** + * Manages Accumulo client connections with support for multiple authentication modes. + * + *

This class centralizes all Accumulo client creation and authentication logic, + * supporting:

+ *
    + *
  • Password authentication (username/password)
    • + *
  • Kerberos authentication (principal/keytab)
    • + *
  • User translation (per-user credentials lookup)
    • + *
  • Delegation tokens for user impersonation in distributed execution
    • + *
+ * + *

Authentication Modes:

+ *
    + *
  • SHARED_USER: All queries use a single shared service client
    • + *
  • USER_TRANSLATION: Per-user Accumulo credentials are looked up from + * the CredentialsProvider based on the Drill query user
    • + *
  • USER_IMPERSONATION: Service authenticates with Kerberos, then + * impersonates the query user via delegation tokens
    • + *
+ */ +public class AccumuloConnectionManager implements Closeable { + private static final Logger logger = LoggerFactory.getLogger(AccumuloConnectionManager.class); + + /** + * Default TTL for cached delegation tokens (1 hour). + * Tokens older than this will be refreshed. + */ + private static final long DEFAULT_TOKEN_TTL_MILLIS = TimeUnit.HOURS.toMillis(1); + + /** + * Default delegation token lifetime when requesting from Accumulo. + */ + private static final long DEFAULT_TOKEN_LIFETIME_MILLIS = TimeUnit.HOURS.toMillis(24); + + private final AccumuloStoragePluginConfig config; + + /** + * Shared service client (for PASSWORD or KERBEROS + SHARED_USER mode). + */ + private volatile AccumuloClient serviceClient; + private final Object serviceClientLock = new Object(); + + /** + * Cache of per-user clients for USER_TRANSLATION mode. + * Key is the Accumulo username (from translated credentials). + */ + private final Map userClientCache = new ConcurrentHashMap<>(); + + /** + * Cache of delegation tokens per user for USER_IMPERSONATION mode. + */ + private final Map delegationTokenCache = new ConcurrentHashMap<>(); + + public AccumuloConnectionManager(AccumuloStoragePluginConfig config) { + this.config = config; + } + + /** + * Returns the shared service client. + * + *

For PASSWORD mode, this uses the configured username/password. + * For KERBEROS mode, this authenticates using the service principal and keytab.

+ * + *

The client is lazily initialized and cached for reuse.

+ * + * @return the service AccumuloClient + * @throws UserException if connection fails + */ + public AccumuloClient getServiceClient() { + if (serviceClient == null) { + synchronized (serviceClientLock) { + if (serviceClient == null) { + try { + serviceClient = createServiceClient(); + logger.info("Created Accumulo service client for instance: {}", config.getInstanceName()); + } catch (AccumuloException | AccumuloSecurityException | IOException e) { + throw UserException.connectionError(e) + .message("Failed to connect to Accumulo instance '%s' at '%s'", + config.getInstanceName(), config.getZookeeperQuorum()) + .addContext("AuthenticationType", config.getAuthenticationType()) + .build(logger); + } + } + } + } + return serviceClient; + } + + /** + * Returns an AccumuloClient for the specified user. + * + *

Behavior depends on the auth mode:

+ *
    + *
  • SHARED_USER: Returns the service client (same for all users)
    • + *
  • USER_TRANSLATION: Creates/returns a client using per-user credentials
    • + *
  • USER_IMPERSONATION: Returns a client using the user's delegation token
    • + *
+ * + * @param userName the Drill query user name + * @return an AccumuloClient for the user + * @throws UserException if client creation fails + */ + public AccumuloClient getClientForUser(String userName) { + return getClientForUser(userName, null); + } + + /** + * Returns an AccumuloClient for the specified user with optional user credentials. + * + * @param userName the Drill query user name + * @param userCredentials the user credentials (for USER_TRANSLATION mode) + * @return an AccumuloClient for the user + * @throws UserException if client creation fails + */ + public AccumuloClient getClientForUser(String userName, UserCredentials userCredentials) { + AuthMode authMode = config.getAuthMode(); + + switch (authMode) { + case SHARED_USER: + return getServiceClient(); + + case USER_TRANSLATION: + return getClientForUserTranslation(userName, userCredentials); + + case USER_IMPERSONATION: + return getClientForUserImpersonation(userName); + + default: + throw UserException.connectionError() + .message("Unsupported auth mode: %s", authMode) + .build(logger); + } + } + + /** + * Creates or returns a cached client for USER_TRANSLATION mode. + * + *

In this mode, per-user Accumulo credentials are looked up from the + * CredentialsProvider based on the Drill query user.

+ */ + private AccumuloClient getClientForUserTranslation(String userName, UserCredentials userCredentials) { + // Build UserCredentials if not provided + if (userCredentials == null && userName != null) { + userCredentials = UserCredentials.newBuilder() + .setUserName(userName) + .build(); + } + + // Look up per-user credentials + Optional creds = config.getUsernamePasswordCredentials(userCredentials); + + if (!creds.isPresent()) { + throw UserException.connectionError() + .message("No credentials found for user '%s' in USER_TRANSLATION mode. " + + "Please configure credentials for this user in the storage plugin.", userName) + .addContext("Plugin", "accumulo") + .addContext("AuthMode", "USER_TRANSLATION") + .build(logger); + } + + String accumuloUsername = creds.get().getUsername(); + String accumuloPassword = creds.get().getPassword(); + + // Check cache first (keyed by Accumulo username) + AccumuloClient cachedClient = userClientCache.get(accumuloUsername); + if (cachedClient != null) { + logger.debug("Using cached client for translated user: {} -> {}", userName, accumuloUsername); + return cachedClient; + } + + // Create new client + synchronized (userClientCache) { + // Double-check + cachedClient = userClientCache.get(accumuloUsername); + if (cachedClient != null) { + return cachedClient; + } + + try { + logger.info("Creating Accumulo client for translated user: {} -> {}", userName, accumuloUsername); + + Properties props = new Properties(); + props.setProperty("instance.name", config.getInstanceName()); + props.setProperty("instance.zookeepers", config.getZookeeperQuorum()); + + AccumuloClient client = Accumulo.newClient() + .from(props) + .as(accumuloUsername, new PasswordToken(accumuloPassword)) + .build(); + + userClientCache.put(accumuloUsername, client); + return client; + + } catch (Exception e) { + throw UserException.connectionError(e) + .message("Failed to create Accumulo client for translated user '%s' (Accumulo user: '%s')", + userName, accumuloUsername) + .build(logger); + } + } + } + + /** + * Creates a client for USER_IMPERSONATION mode using delegation tokens. + */ + private AccumuloClient getClientForUserImpersonation(String userName) { + if (config.getAuthenticationType() != AccumuloAuthType.KERBEROS) { + throw UserException.connectionError() + .message("User impersonation requires Kerberos authentication") + .build(logger); + } + + if (!config.isUseDelegationTokens()) { + throw UserException.connectionError() + .message("User impersonation requires delegation tokens to be enabled") + .build(logger); + } + + try { + DelegationTokenInfo tokenInfo = getDelegationToken(userName); + return createClientWithDelegationToken(tokenInfo); + } catch (Exception e) { + throw UserException.connectionError(e) + .message("Failed to create impersonated client for user '%s'", userName) + .build(logger); + } + } + + /** + * Gets or creates a delegation token for the specified user. + * + *

Tokens are cached with a TTL to avoid repeated token creation. + * This method is thread-safe.

+ * + * @param userName the user to get a delegation token for + * @return the delegation token info + * @throws AccumuloException if token creation fails + * @throws AccumuloSecurityException if authentication fails + * @throws IOException if token serialization fails + */ + public DelegationTokenInfo getDelegationToken(String userName) + throws AccumuloException, AccumuloSecurityException, IOException { + + // Check cache first + DelegationTokenInfo cached = delegationTokenCache.get(userName); + if (cached != null && !cached.isOlderThan(DEFAULT_TOKEN_TTL_MILLIS)) { + logger.debug("Using cached delegation token for user: {}", userName); + return cached; + } + + // Need to create/refresh token + synchronized (delegationTokenCache) { + // Double-check after acquiring lock + cached = delegationTokenCache.get(userName); + if (cached != null && !cached.isOlderThan(DEFAULT_TOKEN_TTL_MILLIS)) { + return cached; + } + + logger.info("Creating delegation token for user: {}", userName); + + // Use the service client to obtain a delegation token + AccumuloClient client = getServiceClient(); + + // Create a proxy user UGI for the query user + UserGroupInformation proxyUgi = ImpersonationUtil.createProxyUgi(userName); + + // Request delegation token for the proxy user + DelegationTokenConfig tokenConfig = new DelegationTokenConfig(); + tokenConfig.setTokenLifetime(DEFAULT_TOKEN_LIFETIME_MILLIS, TimeUnit.MILLISECONDS); + + DelegationToken token = client.securityOperations() + .getDelegationToken(tokenConfig); + + DelegationTokenInfo tokenInfo = DelegationTokenInfo.fromDelegationToken(userName, token); + delegationTokenCache.put(userName, tokenInfo); + + logger.info("Created delegation token for user: {} (expires in {} ms)", + userName, DEFAULT_TOKEN_LIFETIME_MILLIS); + + return tokenInfo; + } + } + + /** + * Creates an AccumuloClient using a delegation token. + * + *

This method is used by distributed fragments to create a client + * with the delegated user identity.

+ * + * @param tokenInfo the delegation token info + * @return a new AccumuloClient authenticated with the delegation token + * @throws AccumuloException if client creation fails + * @throws AccumuloSecurityException if authentication fails + */ + public AccumuloClient createClientWithDelegationToken(DelegationTokenInfo tokenInfo) + throws AccumuloException, AccumuloSecurityException { + + AuthenticationToken token = tokenInfo.toAuthenticationToken(); + + Properties props = new Properties(); + props.setProperty("instance.name", config.getInstanceName()); + props.setProperty("instance.zookeepers", config.getZookeeperQuorum()); + + // Configure SASL for delegation token authentication + if (!Strings.isNullOrEmpty(config.getSaslQop())) { + props.setProperty("rpc.sasl.qop", config.getSaslQop()); + } + + return Accumulo.newClient() + .from(props) + .as(tokenInfo.getUserName(), token) + .build(); + } + + /** + * Creates an AccumuloClient using username/password credentials. + * + *

This method is used for USER_TRANSLATION mode where per-user + * credentials are stored in the CredentialsProvider.

+ * + * @param credentials the username/password credentials + * @return a new AccumuloClient + * @throws AccumuloException if client creation fails + * @throws AccumuloSecurityException if authentication fails + */ + public AccumuloClient createClientWithCredentials(UsernamePasswordCredentials credentials) + throws AccumuloException, AccumuloSecurityException { + + Properties props = new Properties(); + props.setProperty("instance.name", config.getInstanceName()); + props.setProperty("instance.zookeepers", config.getZookeeperQuorum()); + + return Accumulo.newClient() + .from(props) + .as(credentials.getUsername(), new PasswordToken(credentials.getPassword())) + .build(); + } + + /** + * Creates the service client based on the configured authentication type. + */ + private AccumuloClient createServiceClient() + throws AccumuloException, AccumuloSecurityException, IOException { + + AccumuloAuthType authType = config.getAuthenticationType(); + + Properties props = new Properties(); + props.setProperty("instance.name", config.getInstanceName()); + props.setProperty("instance.zookeepers", config.getZookeeperQuorum()); + + String principal; + AuthenticationToken token; + + if (authType == AccumuloAuthType.KERBEROS) { + principal = config.getPrincipal(); + token = createKerberosToken(); + + // Configure SASL properties + if (!Strings.isNullOrEmpty(config.getSaslQop())) { + props.setProperty("rpc.sasl.qop", config.getSaslQop()); + } + if (!Strings.isNullOrEmpty(config.getAccumuloServicePrimary())) { + props.setProperty("sasl.kerberos.server.primary", config.getAccumuloServicePrimary()); + } + } else { + // PASSWORD authentication + Optional creds = config.getUsernamePasswordCredentials(null); + if (creds.isPresent()) { + principal = creds.get().getUsername(); + token = new PasswordToken(creds.get().getPassword()); + } else { + principal = config.getUsername(); + token = new PasswordToken(config.getPassword()); + } + } + + logger.debug("Creating Accumulo client with auth type: {}, principal: {}", authType, principal); + + return Accumulo.newClient() + .from(props) + .as(principal, token) + .build(); + } + + /** + * Creates a KerberosToken for service authentication. + */ + private KerberosToken createKerberosToken() throws IOException { + String keytabPath = config.getKeytabPath(); + String principal = config.getPrincipal(); + + if (Strings.isNullOrEmpty(keytabPath)) { + throw new IOException("Keytab path is required for Kerberos authentication"); + } + if (Strings.isNullOrEmpty(principal)) { + throw new IOException("Principal is required for Kerberos authentication"); + } + + File keytabFile = new File(keytabPath); + if (!keytabFile.exists()) { + throw new IOException("Keytab file does not exist: " + keytabPath); + } + if (!keytabFile.canRead()) { + throw new IOException("Cannot read keytab file: " + keytabPath); + } + + logger.info("Logging in with Kerberos principal: {} using keytab: {}", principal, keytabPath); + + // Login using the keytab + UserGroupInformation.loginUserFromKeytab(principal, keytabPath); + + return new KerberosToken(); + } + + /** + * Clears the delegation token cache. + * This forces new tokens to be obtained on next request. + */ + public void clearDelegationTokenCache() { + delegationTokenCache.clear(); + logger.debug("Cleared delegation token cache"); + } + + /** + * Returns the number of cached delegation tokens. + * Primarily for testing/monitoring purposes. + */ + public int getDelegationTokenCacheSize() { + return delegationTokenCache.size(); + } + + /** + * Returns the number of cached user clients (for USER_TRANSLATION mode). + * Primarily for testing/monitoring purposes. + */ + public int getUserClientCacheSize() { + return userClientCache.size(); + } + + @Override + public void close() { + // Close service client + synchronized (serviceClientLock) { + if (serviceClient != null) { + try { + logger.debug("Closing Accumulo service client for instance: {}", config.getInstanceName()); + serviceClient.close(); + } catch (Exception e) { + logger.warn("Error closing Accumulo service client", e); + } + serviceClient = null; + } + } + + // Close all cached user clients + for (Map.Entry entry : userClientCache.entrySet()) { + try { + logger.debug("Closing cached client for user: {}", entry.getKey()); + entry.getValue().close(); + } catch (Exception e) { + logger.warn("Error closing cached client for user: {}", entry.getKey(), e); + } + } + userClientCache.clear(); + + // Clear delegation token cache + clearDelegationTokenCache(); + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilder.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilder.java new file mode 100644 index 00000000000..0b38d5b1fc8 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilder.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.util.Arrays; +import java.util.List; + +import org.apache.drill.common.FunctionNames; +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.visitors.AbstractExprVisitor; + +/** + * Builds Accumulo scan specifications from Drill filter expressions. + * + *

This class converts Drill's LogicalExpression filter representation into + * Accumulo scan parameters (start row, stop row). It focuses on row key + * predicates since those can be efficiently pushed down to Accumulo's scan range.

+ * + *

Supported predicates on row_key:

+ *
    + *
  • row_key = 'value' → exact range
    • + *
  • row_key > 'value' → start row (exclusive)
    • + *
  • row_key >= 'value' → start row (inclusive)
    • + *
  • row_key < 'value' → stop row (exclusive)
    • + *
  • row_key <= 'value' → stop row (inclusive)
    • + *
  • AND combinations → intersect ranges
    • + *
  • OR combinations → union ranges (if contiguous)
    • + *
+ */ +public class AccumuloFilterBuilder + extends AbstractExprVisitor + implements DrillAccumuloConstants { + + private final AccumuloGroupScan groupScan; + private final LogicalExpression filterExpression; + private boolean allExpressionsConverted = true; + + public AccumuloFilterBuilder(AccumuloGroupScan groupScan, LogicalExpression filterExpression) { + this.groupScan = groupScan; + this.filterExpression = filterExpression; + } + + /** + * Parses the filter expression and returns an updated scan specification. + * + * @return the scan spec with row key ranges, or null if no filters can be pushed + */ + public AccumuloScanSpec parseTree() { + AccumuloScanSpec parsedSpec = filterExpression.accept(this, null); + if (parsedSpec != null) { + // Merge with existing scan spec + parsedSpec = mergeScanSpecs(FunctionNames.AND, groupScan.getScanSpec(), parsedSpec); + } + return parsedSpec; + } + + /** + * Returns true if all filter expressions were converted to Accumulo scan parameters. + * If false, the filter operator should remain in the plan for client-side filtering. + */ + public boolean isAllExpressionsConverted() { + return allExpressionsConverted; + } + + @Override + public AccumuloScanSpec visitUnknown(LogicalExpression e, Void value) throws RuntimeException { + allExpressionsConverted = false; + return null; + } + + @Override + public AccumuloScanSpec visitBooleanOperator(BooleanOperator op, Void value) + throws RuntimeException { + return visitFunctionCall(op, value); + } + + @Override + public AccumuloScanSpec visitFunctionCall(FunctionCall call, Void value) + throws RuntimeException { + AccumuloScanSpec nodeScanSpec = null; + String functionName = call.getName(); + List args = call.args(); + + if (AccumuloCompareFunctionsProcessor.isCompareFunction(functionName)) { + AccumuloCompareFunctionsProcessor processor = + AccumuloCompareFunctionsProcessor.createFunctionsProcessorInstance(call); + if (processor.isSuccess()) { + nodeScanSpec = createScanSpecFromComparison(processor); + } + } else { + switch (functionName) { + case FunctionNames.AND: + case FunctionNames.OR: + AccumuloScanSpec firstScanSpec = args.get(0).accept(this, null); + for (int i = 1; i < args.size(); ++i) { + AccumuloScanSpec nextScanSpec = args.get(i).accept(this, null); + if (firstScanSpec != null && nextScanSpec != null) { + nodeScanSpec = mergeScanSpecs(functionName, firstScanSpec, nextScanSpec); + } else { + allExpressionsConverted = false; + if (FunctionNames.AND.equals(functionName)) { + // For AND, keep whichever spec we have + nodeScanSpec = firstScanSpec == null ? nextScanSpec : firstScanSpec; + } + // For OR, if either is null we can't push down the whole OR + } + firstScanSpec = nodeScanSpec; + } + break; + default: + // Unknown function + break; + } + } + + if (nodeScanSpec == null) { + allExpressionsConverted = false; + } + + return nodeScanSpec; + } + + /** + * Creates a scan spec from a comparison processor result. + */ + private AccumuloScanSpec createScanSpecFromComparison( + AccumuloCompareFunctionsProcessor processor) { + + String functionName = processor.getFunctionName(); + SchemaPath field = processor.getPath(); + byte[] fieldValue = processor.getValue(); + + // Only handle row_key predicates for now + boolean isRowKey = field.getRootSegmentPath().equalsIgnoreCase(ROW_KEY); + if (!isRowKey) { + // Column predicates require iterators - not supported in Option A + return null; + } + + byte[] startRow = null; + byte[] stopRow = null; + boolean startRowInclusive = true; + boolean stopRowInclusive = false; + + switch (functionName) { + case FunctionNames.EQ: + // row_key = 'value' → scan exactly that row + startRow = fieldValue; + // Stop row should be just after the value + stopRow = Arrays.copyOf(fieldValue, fieldValue.length + 1); + startRowInclusive = true; + stopRowInclusive = false; + break; + + case FunctionNames.NE: + // row_key != 'value' → can't efficiently push down (would need full scan minus one row) + return null; + + case FunctionNames.GE: + // row_key >= 'value' → start at value (inclusive) + startRow = fieldValue; + startRowInclusive = true; + break; + + case FunctionNames.GT: + // row_key > 'value' → start just after value + startRow = Arrays.copyOf(fieldValue, fieldValue.length + 1); + startRowInclusive = true; + break; + + case FunctionNames.LE: + // row_key <= 'value' → stop just after value + stopRow = Arrays.copyOf(fieldValue, fieldValue.length + 1); + stopRowInclusive = false; + break; + + case FunctionNames.LT: + // row_key < 'value' → stop at value (exclusive) + stopRow = fieldValue; + stopRowInclusive = false; + break; + + default: + return null; + } + + return new AccumuloScanSpec( + groupScan.getTableName(), + startRow, + stopRow, + startRowInclusive, + stopRowInclusive, + groupScan.getScanSpec().getColumns(), + null, // No filter expression needed when using row ranges + groupScan.getScanSpec().getLimit(), + groupScan.getScanSpec().isUseSortedScanner()); + } + + /** + * Merges two scan specs using AND or OR logic. + */ + private AccumuloScanSpec mergeScanSpecs( + String functionName, + AccumuloScanSpec leftSpec, + AccumuloScanSpec rightSpec) { + + byte[] startRow = null; + byte[] stopRow = null; + boolean startRowInclusive = true; + boolean stopRowInclusive = false; + + switch (functionName) { + case FunctionNames.AND: + // AND: Take the intersection (max of starts, min of stops) + startRow = maxOfStartRows(leftSpec.getStartRow(), rightSpec.getStartRow()); + stopRow = minOfStopRows(leftSpec.getStopRow(), rightSpec.getStopRow()); + break; + + case FunctionNames.OR: + // OR: Take the union (min of starts, max of stops) + startRow = minOfStartRows(leftSpec.getStartRow(), rightSpec.getStartRow()); + stopRow = maxOfStopRows(leftSpec.getStopRow(), rightSpec.getStopRow()); + break; + + default: + return leftSpec; + } + + return new AccumuloScanSpec( + leftSpec.getTableName(), + startRow, + stopRow, + startRowInclusive, + stopRowInclusive, + leftSpec.getColumns(), + leftSpec.getFilterExpression(), + leftSpec.getLimit(), + leftSpec.isUseSortedScanner()); + } + + /** + * Returns the maximum of two start rows (later in sort order). + */ + private byte[] maxOfStartRows(byte[] left, byte[] right) { + if (left == null) { + return right; + } + if (right == null) { + return left; + } + return compareBytes(left, right) >= 0 ? left : right; + } + + /** + * Returns the minimum of two start rows (earlier in sort order). + */ + private byte[] minOfStartRows(byte[] left, byte[] right) { + if (left == null) { + return right; + } + if (right == null) { + return left; + } + return compareBytes(left, right) <= 0 ? left : right; + } + + /** + * Returns the minimum of two stop rows (earlier in sort order). + */ + private byte[] minOfStopRows(byte[] left, byte[] right) { + if (left == null) { + return right; + } + if (right == null) { + return left; + } + return compareBytes(left, right) <= 0 ? left : right; + } + + /** + * Returns the maximum of two stop rows (later in sort order). + */ + private byte[] maxOfStopRows(byte[] left, byte[] right) { + if (left == null) { + return right; + } + if (right == null) { + return left; + } + return compareBytes(left, right) >= 0 ? left : right; + } + + /** + * Compares two byte arrays lexicographically. + */ + private int compareBytes(byte[] left, byte[] right) { + int minLen = Math.min(left.length, right.length); + for (int i = 0; i < minLen; i++) { + int cmp = (left[i] & 0xFF) - (right[i] & 0xFF); + if (cmp != 0) { + return cmp; + } + } + return left.length - right.length; + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java new file mode 100644 index 00000000000..14e2196d019 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import org.apache.drill.common.PlanStringBuilder; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.physical.EndpointAffinity; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; +import org.apache.drill.exec.store.StoragePluginRegistry; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; + +/** + * Group scan for Accumulo tables. + * + *

This class handles scan planning and fragmentation across Accumulo tablets. + * It can be modified by optimizer rules to apply filter, projection, limit, and sort pushdowns.

+ * + *

For user impersonation mode, this class carries a delegation token that is + * serialized to JSON for distributed planning and passed to SubScans for execution.

+ */ +@JsonTypeName("accumulo-scan") +public class AccumuloGroupScan extends AbstractGroupScan { + + private AccumuloStoragePluginConfig storagePluginConfig; + private AccumuloStoragePlugin storagePlugin; + private AccumuloScanSpec scanSpec; + private List columns; + private int maxRecords; + + /** + * Delegation token for user impersonation in distributed execution. + * When present, SubScans will use this token to create user-impersonated clients. + */ + private DelegationTokenInfo delegationTokenInfo; + + private boolean filterPushedDown = false; + private boolean projectionPushedDown = false; + private boolean limitPushedDown = false; + private boolean sortPushedDown = false; + + @JsonCreator + public AccumuloGroupScan( + @JsonProperty("userName") String userName, + @JsonProperty("scanSpec") AccumuloScanSpec scanSpec, + @JsonProperty("storage") AccumuloStoragePluginConfig storagePluginConfig, + @JsonProperty("columns") List columns, + @JsonProperty("maxRecords") int maxRecords, + @JsonProperty("delegationTokenInfo") DelegationTokenInfo delegationTokenInfo, + @JacksonInject StoragePluginRegistry pluginRegistry) throws IOException, ExecutionSetupException { + this(userName, pluginRegistry.resolve(storagePluginConfig, AccumuloStoragePlugin.class), + scanSpec, columns, maxRecords, delegationTokenInfo); + } + + public AccumuloGroupScan( + String userName, + AccumuloStoragePlugin storagePlugin, + AccumuloScanSpec scanSpec, + List columns, + int maxRecords) { + this(userName, storagePlugin, scanSpec, columns, maxRecords, null); + } + + public AccumuloGroupScan( + String userName, + AccumuloStoragePlugin storagePlugin, + AccumuloScanSpec scanSpec, + List columns, + int maxRecords, + DelegationTokenInfo delegationTokenInfo) { + super(userName); + this.storagePlugin = storagePlugin; + this.storagePluginConfig = storagePlugin.getConfig(); + this.scanSpec = scanSpec; + this.columns = columns == null ? ALL_COLUMNS : columns; + this.maxRecords = maxRecords; + this.delegationTokenInfo = delegationTokenInfo; + } + + /** + * Copy constructor for cloning. + */ + private AccumuloGroupScan(AccumuloGroupScan that) { + super(that); + this.storagePlugin = that.storagePlugin; + this.storagePluginConfig = that.storagePluginConfig; + this.scanSpec = that.scanSpec; + this.columns = that.columns == null ? ALL_COLUMNS : that.columns; + this.maxRecords = that.maxRecords; + this.delegationTokenInfo = that.delegationTokenInfo; + this.filterPushedDown = that.filterPushedDown; + this.projectionPushedDown = that.projectionPushedDown; + this.limitPushedDown = that.limitPushedDown; + this.sortPushedDown = that.sortPushedDown; + } + + @Override + public GroupScan clone(List columns) { + AccumuloGroupScan cloned = new AccumuloGroupScan(this); + cloned.columns = columns; + // Mark projection as pushed down if we're projecting specific columns + if (columns != null && !columns.equals(ALL_COLUMNS)) { + cloned.projectionPushedDown = true; + } + return cloned; + } + + @Override + public PhysicalOperator getNewWithChildren(List children) { + return new AccumuloGroupScan(this); + } + + @Override + public void applyAssignments(List endpoints) { + // TODO: Implement tablet-to-endpoint assignment for data locality + } + + @Override + public AccumuloSubScan getSpecificScan(int minorFragmentId) { + // Pass delegation token to SubScan for distributed execution + return new AccumuloSubScan(getUserName(), storagePlugin, scanSpec, columns, maxRecords, delegationTokenInfo); + } + + @Override + public int getMaxParallelizationWidth() { + // TODO: Return actual number of tablets; for now return 1 + return 1; + } + + @Override + public List getOperatorAffinity() { + // TODO: Return endpoint affinities based on tablet locations + return Collections.emptyList(); + } + + @Override + public ScanStats getScanStats() { + // TODO: Calculate actual scan statistics from Accumulo metadata + long rowCount = 100000; // Estimate + int columnCount = columns != null && !columns.equals(ALL_COLUMNS) ? columns.size() : 10; + double cpuCost = rowCount * columnCount; + + // Adjust cost for pushdowns + if (filterPushedDown) { + cpuCost *= 0.5; + rowCount *= 0.5; + } + if (projectionPushedDown) { + // Projection reduces network I/O significantly + cpuCost *= 0.7; + } + if (sortPushedDown) { + cpuCost *= 1.2; // Slight penalty for using Scanner instead of BatchScanner + } + if (limitPushedDown && maxRecords > 0) { + // Limit pushdown significantly reduces work + rowCount = Math.min(rowCount, maxRecords); + cpuCost = rowCount * columnCount; + } + + return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, cpuCost, rowCount * columnCount * 8); + } + + @Override + @JsonIgnore + public boolean supportsLimitPushdown() { + return true; + } + + @Override + public GroupScan applyLimit(int maxRecords) { + // If limit is already set and is more restrictive, keep the current one + if (this.maxRecords > 0 && this.maxRecords <= maxRecords) { + return null; + } + + AccumuloGroupScan newScan = new AccumuloGroupScan(this); + newScan.maxRecords = maxRecords; + newScan.limitPushedDown = true; + return newScan; + } + + @Override + public String getDigest() { + return toString(); + } + + @Override + public String toString() { + return new PlanStringBuilder(this) + .field("scanSpec", scanSpec) + .field("columns", columns) + .field("maxRecords", maxRecords) + .field("filterPushedDown", filterPushedDown) + .field("projectionPushedDown", projectionPushedDown) + .field("limitPushedDown", limitPushedDown) + .field("sortPushedDown", sortPushedDown) + .field("hasDelegationToken", delegationTokenInfo != null) + .toString(); + } + + // Getters for Jackson serialization + + @JsonProperty("scanSpec") + public AccumuloScanSpec getScanSpec() { + return scanSpec; + } + + @JsonProperty("storage") + public AccumuloStoragePluginConfig getStoragePluginConfig() { + return storagePluginConfig; + } + + @JsonProperty("columns") + public List getColumns() { + return columns; + } + + @JsonProperty("maxRecords") + public int getMaxRecords() { + return maxRecords; + } + + @JsonProperty("delegationTokenInfo") + public DelegationTokenInfo getDelegationTokenInfo() { + return delegationTokenInfo; + } + + @JsonIgnore + public AccumuloStoragePlugin getStoragePlugin() { + return storagePlugin; + } + + @JsonIgnore + public String getTableName() { + return scanSpec != null ? scanSpec.getTableName() : null; + } + + /** + * Returns true if this scan has a delegation token for user impersonation. + */ + @JsonIgnore + public boolean hasDelegationToken() { + return delegationTokenInfo != null; + } + + // Pushdown tracking methods + + @JsonIgnore + public boolean isFilterPushedDown() { + return filterPushedDown; + } + + public void setFilterPushedDown(boolean filterPushedDown) { + this.filterPushedDown = filterPushedDown; + } + + @JsonIgnore + public boolean isProjectionPushedDown() { + return projectionPushedDown; + } + + public void setProjectionPushedDown(boolean projectionPushedDown) { + this.projectionPushedDown = projectionPushedDown; + } + + @JsonIgnore + public boolean isLimitPushedDown() { + return limitPushedDown; + } + + public void setLimitPushedDown(boolean limitPushedDown) { + this.limitPushedDown = limitPushedDown; + } + + @JsonIgnore + public boolean isSortPushedDown() { + return sortPushedDown; + } + + public void setSortPushedDown(boolean sortPushedDown) { + this.sortPushedDown = sortPushedDown; + } + + /** + * Returns a new AccumuloGroupScan with the given scan spec. + * Used by optimizer rules to create modified scans. + */ + public AccumuloGroupScan cloneWithNewScanSpec(AccumuloScanSpec newScanSpec) { + AccumuloGroupScan cloned = new AccumuloGroupScan(this); + cloned.scanSpec = newScanSpec; + return cloned; + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloPushFilterIntoScan.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloPushFilterIntoScan.java new file mode 100644 index 00000000000..90320c54f0e --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloPushFilterIntoScan.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.exec.planner.logical.DrillOptiq; +import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.planner.physical.FilterPrel; +import org.apache.drill.exec.planner.physical.PrelUtil; +import org.apache.drill.exec.planner.physical.ProjectPrel; +import org.apache.drill.exec.planner.physical.ScanPrel; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; + +import com.google.common.collect.ImmutableList; + +/** + * Optimizer rule to push filter predicates into Accumulo scans. + * + *

This rule matches Filter → Scan patterns (optionally with a Project in between) + * and pushes row_key predicates down to the Accumulo scan as row ranges.

+ * + *

Supported patterns:

+ *
    + *
  • Filter → ScanPrel (AccumuloGroupScan)
    • + *
  • Filter → Project → ScanPrel (AccumuloGroupScan)
    • + *
+ */ +public abstract class AccumuloPushFilterIntoScan extends StoragePluginOptimizerRule { + + private AccumuloPushFilterIntoScan(RelOptRuleOperand operand, String description) { + super(operand, description); + } + + /** + * Rule for Filter directly on Scan. + */ + public static final StoragePluginOptimizerRule FILTER_ON_SCAN = + new AccumuloPushFilterIntoScan( + RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), + "AccumuloPushFilterIntoScan:Filter_On_Scan") { + + @Override + public void onMatch(RelOptRuleCall call) { + final ScanPrel scan = call.rel(1); + final FilterPrel filter = call.rel(0); + final RexNode condition = filter.getCondition(); + + AccumuloGroupScan groupScan = (AccumuloGroupScan) scan.getGroupScan(); + if (groupScan.isFilterPushedDown()) { + // Already processed - don't re-process + return; + } + + doPushFilterToScan(call, filter, null, scan, groupScan, condition); + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = call.rel(1); + if (scan.getGroupScan() instanceof AccumuloGroupScan) { + return super.matches(call); + } + return false; + } + }; + + /** + * Rule for Filter on Project on Scan. + */ + public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = + new AccumuloPushFilterIntoScan( + RelOptHelper.some(FilterPrel.class, + RelOptHelper.some(ProjectPrel.class, RelOptHelper.any(ScanPrel.class))), + "AccumuloPushFilterIntoScan:Filter_On_Project") { + + @Override + public void onMatch(RelOptRuleCall call) { + final ScanPrel scan = call.rel(2); + final ProjectPrel project = call.rel(1); + final FilterPrel filter = call.rel(0); + + AccumuloGroupScan groupScan = (AccumuloGroupScan) scan.getGroupScan(); + if (groupScan.isFilterPushedDown()) { + // Already processed - don't re-process + return; + } + + // Push filter through project + final RexNode condition = RelOptUtil.pushPastProject(filter.getCondition(), project); + + doPushFilterToScan(call, filter, project, scan, groupScan, condition); + } + + @Override + public boolean matches(RelOptRuleCall call) { + final ScanPrel scan = call.rel(2); + if (scan.getGroupScan() instanceof AccumuloGroupScan) { + return super.matches(call); + } + return false; + } + }; + + /** + * Pushes filter conditions to the Accumulo scan. + */ + protected void doPushFilterToScan( + final RelOptRuleCall call, + final FilterPrel filter, + final ProjectPrel project, + final ScanPrel scan, + final AccumuloGroupScan groupScan, + final RexNode condition) { + + // Convert RexNode to Drill LogicalExpression + final LogicalExpression conditionExp = DrillOptiq.toDrill( + new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), + scan, + condition); + + // Build Accumulo scan spec from filter + final AccumuloFilterBuilder filterBuilder = + new AccumuloFilterBuilder(groupScan, conditionExp); + final AccumuloScanSpec newScanSpec = filterBuilder.parseTree(); + + if (newScanSpec == null) { + // No filter could be pushed down + return; + } + + // Create new group scan with pushed filter + final AccumuloGroupScan newGroupScan = groupScan.cloneWithNewScanSpec(newScanSpec); + newGroupScan.setFilterPushedDown(true); + + // Create new scan prel + final ScanPrel newScanPrel = new ScanPrel( + scan.getCluster(), + filter.getTraitSet(), + newGroupScan, + scan.getRowType(), + scan.getTable()); + + // If there's a project, keep it + final RelNode childRel = project == null + ? newScanPrel + : project.copy(project.getTraitSet(), ImmutableList.of(newScanPrel)); + + if (filterBuilder.isAllExpressionsConverted()) { + // All filter conditions were pushed - remove the filter operator + call.transformTo(childRel); + } else { + // Partial pushdown - keep filter for remaining conditions + call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel))); + } + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloPushSortIntoScan.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloPushSortIntoScan.java new file mode 100644 index 00000000000..d12b16a7617 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloPushSortIntoScan.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.util.List; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.rel.RelFieldCollation.Direction; +import org.apache.drill.exec.planner.logical.DrillScanRel; +import org.apache.drill.exec.planner.logical.DrillSortRel; +import org.apache.drill.exec.planner.logical.RelOptHelper; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Optimizer rule that pushes sort operations into Accumulo scans. + * + *

Accumulo naturally returns rows sorted by row key in ascending order. + * This rule detects when a sort is on the row_key column and can be satisfied + * by Accumulo's natural ordering:

+ * + *
    + *
  • ASC order on row_key: Use Scanner (maintains order) - data is naturally sorted
    • + *
  • DESC order on row_key: Configure scanner for reverse iteration
    • + *
+ * + *

When sort is pushed down, the sort operator may be eliminated by + * subsequent optimization passes since the data is already sorted.

+ */ +public abstract class AccumuloPushSortIntoScan extends StoragePluginOptimizerRule { + private static final Logger logger = LoggerFactory.getLogger(AccumuloPushSortIntoScan.class); + + private AccumuloPushSortIntoScan(RelOptRuleOperand operand, String description) { + super(operand, description); + } + + /** + * Rule for Sort directly on Scan. + */ + public static final StoragePluginOptimizerRule SORT_ON_SCAN = + new AccumuloPushSortIntoScan( + RelOptHelper.some(DrillSortRel.class, RelOptHelper.any(DrillScanRel.class)), + "AccumuloPushSortIntoScan:Sort_On_Scan") { + + @Override + public void onMatch(RelOptRuleCall call) { + DrillSortRel sort = call.rel(0); + DrillScanRel scan = call.rel(1); + doPushSortIntoScan(call, sort, scan); + } + + @Override + public boolean matches(RelOptRuleCall call) { + DrillScanRel scan = call.rel(1); + if (!(scan.getGroupScan() instanceof AccumuloGroupScan)) { + return false; + } + AccumuloGroupScan groupScan = (AccumuloGroupScan) scan.getGroupScan(); + // Don't push sort if already pushed + return !groupScan.isSortPushedDown(); + } + }; + + /** + * Pushes sort into Accumulo scan if the sort is on row_key. + */ + protected void doPushSortIntoScan(RelOptRuleCall call, DrillSortRel sort, DrillScanRel scan) { + AccumuloGroupScan groupScan = (AccumuloGroupScan) scan.getGroupScan(); + + // Check if sort is on row_key + RelCollation collation = sort.getCollation(); + List fieldCollations = collation.getFieldCollations(); + + // We only support single-column sort on row_key for now + if (fieldCollations.size() != 1) { + logger.debug("Sort has {} fields, only single-column sort on row_key is supported", + fieldCollations.size()); + return; + } + + RelFieldCollation fieldCollation = fieldCollations.get(0); + int fieldIndex = fieldCollation.getFieldIndex(); + + // row_key is always at index 0 in our schema + if (fieldIndex != 0) { + logger.debug("Sort field index {} is not row_key (index 0)", fieldIndex); + return; + } + + // Check the sort direction + Direction direction = fieldCollation.getDirection(); + boolean isDescending = (direction == Direction.DESCENDING || direction == Direction.STRICTLY_DESCENDING); + + // Create new scan spec with sort direction + AccumuloScanSpec newScanSpec = groupScan.getScanSpec().withSortOrder(isDescending); + AccumuloGroupScan newGroupScan = groupScan.cloneWithNewScanSpec(newScanSpec); + newGroupScan.setSortPushedDown(true); + + // Create new scan with the updated group scan + DrillScanRel newScan = new DrillScanRel( + scan.getCluster(), + scan.getTraitSet(), + scan.getTable(), + newGroupScan, + scan.getRowType(), + scan.getColumns(), + scan.partitionFilterPushdown()); + + // Keep the sort but with the underlying scan now using sorted iteration + // The sort may be removed by later optimization passes if the data is already sorted + DrillSortRel newSort = new DrillSortRel( + sort.getCluster(), + sort.getTraitSet(), + newScan, + sort.getCollation(), + sort.offset, + sort.fetch); + + call.transformTo(newSort); + logger.debug("Pushed {} sort into Accumulo scan for table {}", + isDescending ? "DESC" : "ASC", groupScan.getTableName()); + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java new file mode 100644 index 00000000000..4fd11af4dee --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java @@ -0,0 +1,473 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.PathSegment; +import org.apache.drill.common.expression.PathSegment.NameSegment; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.ops.OperatorContext; +import org.apache.drill.exec.ops.OperatorStats; +import org.apache.drill.exec.physical.impl.OutputMutator; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.store.AbstractRecordReader; +import org.apache.drill.exec.vector.NullableVarBinaryVector; +import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.VarBinaryVector; +import org.apache.drill.exec.vector.complex.MapVector; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.collect.Sets; + +/** + * RecordReader for Accumulo storage plugin. + * + *

This reader scans Accumulo tables and populates Drill value vectors. + * It uses the dynamic schema approach similar to HBase, where column families + * are represented as maps containing their qualifiers as fields.

+ * + *

Row structure:

+ *
    + *
  • row_key: VARBINARY - the Accumulo row key
    • + *
  • Each column family becomes a MAP with qualifier names as keys
    • + *
+ * + *

For user impersonation mode, the reader may own the AccumuloClient + * (created from a delegation token) and is responsible for closing it. + * For shared user mode, the reader uses a shared client and should not close it.

+ */ +public class AccumuloRecordReader extends AbstractRecordReader implements DrillAccumuloConstants { + private static final Logger logger = LoggerFactory.getLogger(AccumuloRecordReader.class); + + // Batch constraints to avoid OOM + private static final int MAX_ALLOCATED_MEMORY_PER_BATCH = 64 * 1024 * 1024; // 64 MB + private static final int TARGET_RECORD_COUNT = 4000; + + private final AccumuloClient client; + private final AccumuloScanSpec scanSpec; + private final int maxRecords; + + /** + * Whether this reader owns the client and should close it. + * True for user impersonation mode (client created from delegation token), + * false for shared user mode (client is shared/pooled). + */ + private final boolean ownsClient; + + private OutputMutator outputMutator; + private OperatorContext operatorContext; + + private Scanner scanner; + private Iterator> scanIterator; + + private Map familyVectorMap; + private VarBinaryVector rowKeyVector; + + private Set requestedFamilies; + private Map> requestedColumns; // family -> set of qualifiers + private boolean rowKeyOnly; + private int recordsRead; + + /** + * Creates an AccumuloRecordReader with a shared client (does not own the client). + */ + public AccumuloRecordReader( + AccumuloClient client, + AccumuloScanSpec scanSpec, + List projectedColumns, + int maxRecords) { + this(client, scanSpec, projectedColumns, maxRecords, false); + } + + /** + * Creates an AccumuloRecordReader with explicit client ownership. + * + * @param client the Accumulo client to use + * @param scanSpec the scan specification + * @param projectedColumns columns to project + * @param maxRecords maximum records to read (-1 for unlimited) + * @param ownsClient true if this reader owns the client and should close it + */ + public AccumuloRecordReader( + AccumuloClient client, + AccumuloScanSpec scanSpec, + List projectedColumns, + int maxRecords, + boolean ownsClient) { + this.client = Preconditions.checkNotNull(client, "AccumuloClient required"); + this.scanSpec = Preconditions.checkNotNull(scanSpec, "AccumuloScanSpec required"); + this.maxRecords = maxRecords > 0 ? maxRecords : Integer.MAX_VALUE; + this.recordsRead = 0; + this.ownsClient = ownsClient; + + setColumns(projectedColumns); + + if (ownsClient) { + logger.debug("RecordReader owns the AccumuloClient and will close it when done"); + } + } + + /** + * Transforms projected columns and determines which Accumulo columns to fetch. + */ + @Override + protected Collection transformColumns(Collection columns) { + Set transformed = Sets.newLinkedHashSet(); + requestedFamilies = Sets.newHashSet(); + requestedColumns = new HashMap<>(); + + rowKeyOnly = true; + + if (!isStarQuery()) { + for (SchemaPath column : columns) { + if (column.getRootSegment().getPath().equalsIgnoreCase(ROW_KEY)) { + transformed.add(ROW_KEY_PATH); + continue; + } + + rowKeyOnly = false; + NameSegment root = column.getRootSegment(); + String family = root.getPath(); + transformed.add(SchemaPath.getSimplePath(family)); + + PathSegment child = root.getChild(); + if (child != null && child.isNamed()) { + // Specific column within family: cf.qualifier + String qualifier = child.getNameSegment().getPath(); + requestedColumns.computeIfAbsent(family, k -> Sets.newHashSet()).add(qualifier); + } else { + // Entire column family requested + requestedFamilies.add(family); + } + } + } else { + rowKeyOnly = false; + transformed.add(ROW_KEY_PATH); + } + + return transformed; + } + + @Override + public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { + this.operatorContext = context; + this.outputMutator = output; + familyVectorMap = new HashMap<>(); + + try { + // Create scanner + scanner = client.createScanner(scanSpec.getTableName(), Authorizations.EMPTY); + + // Configure scan range + configureRange(); + + // Configure which columns to fetch + configureColumns(); + + // Set batch size + scanner.setBatchSize(TARGET_RECORD_COUNT); + + // Setup output vectors + setupOutputVectors(); + + // Get iterator + scanIterator = scanner.iterator(); + + } catch (TableNotFoundException e) { + throw new ExecutionSetupException("Accumulo table not found: " + scanSpec.getTableName(), e); + } catch (SchemaChangeException e) { + throw new ExecutionSetupException("Schema setup failed", e); + } + } + + /** + * Configures the scan range based on start/stop rows. + */ + private void configureRange() { + byte[] startRow = scanSpec.getStartRow(); + byte[] stopRow = scanSpec.getStopRow(); + + if (startRow != null && stopRow != null) { + scanner.setRange(new Range( + new Text(startRow), true, + new Text(stopRow), false)); + } else if (startRow != null) { + scanner.setRange(new Range(new Text(startRow), null)); + } else if (stopRow != null) { + scanner.setRange(new Range(null, new Text(stopRow))); + } + // else: full table scan (default) + } + + /** + * Configures which column families/qualifiers to fetch. + */ + private void configureColumns() { + // If specific columns requested from the scan spec, use those + List specColumns = scanSpec.getColumns(); + if (specColumns != null && !specColumns.isEmpty()) { + for (AccumuloScanSpec.AccumuloColumnSpec col : specColumns) { + String family = col.getColumnFamily(); + String qualifier = col.getColumnQualifier(); + if (qualifier != null && !qualifier.isEmpty()) { + scanner.fetchColumn(new Text(family), new Text(qualifier)); + } else { + scanner.fetchColumnFamily(new Text(family)); + } + } + return; + } + + // Otherwise use the projected columns + if (rowKeyOnly || isStarQuery()) { + // Fetch all columns + return; + } + + // Fetch entire requested families + for (String family : requestedFamilies) { + scanner.fetchColumnFamily(new Text(family)); + } + + // Fetch specific columns (but only if their family isn't already fully requested) + for (Map.Entry> entry : requestedColumns.entrySet()) { + String family = entry.getKey(); + if (!requestedFamilies.contains(family)) { + for (String qualifier : entry.getValue()) { + scanner.fetchColumn(new Text(family), new Text(qualifier)); + } + } + } + } + + /** + * Sets up output vectors based on requested columns. + */ + private void setupOutputVectors() throws SchemaChangeException { + // Add row_key vector + for (SchemaPath column : getColumns()) { + if (column.equals(ROW_KEY_PATH)) { + MaterializedField field = MaterializedField.create(ROW_KEY, ROW_KEY_TYPE); + rowKeyVector = outputMutator.addField(field, VarBinaryVector.class); + } else { + getOrCreateFamilyVector(column.getRootSegment().getPath(), false); + } + } + } + + @Override + public int next() { + Stopwatch watch = Stopwatch.createStarted(); + + // Clear and allocate vectors + if (rowKeyVector != null) { + rowKeyVector.clear(); + rowKeyVector.allocateNew(); + } + for (ValueVector v : familyVectorMap.values()) { + v.clear(); + v.allocateNew(); + } + + int rowCount = 0; + String currentRowKey = null; + int currentRowIndex = -1; + + OperatorStats operatorStats = operatorContext == null ? null : operatorContext.getStats(); + + while (canAddNewRow(rowCount) && recordsRead < maxRecords) { + Map.Entry entry = null; + + try { + if (operatorStats != null) { + operatorStats.startWait(); + } + try { + if (!scanIterator.hasNext()) { + break; + } + entry = scanIterator.next(); + } finally { + if (operatorStats != null) { + operatorStats.stopWait(); + } + } + } catch (Exception e) { + throw new DrillRuntimeException("Error reading from Accumulo", e); + } + + Key key = entry.getKey(); + byte[] rowKeyBytes = key.getRow().getBytes(); + String rowKeyStr = new String(rowKeyBytes, StandardCharsets.UTF_8); + + // Check if this is a new row + if (!rowKeyStr.equals(currentRowKey)) { + if (currentRowKey != null) { + // Finished previous row, increment row count + rowCount++; + recordsRead++; + } + + // Check limits again after incrementing + if (!canAddNewRow(rowCount) || recordsRead >= maxRecords) { + // Can't add more rows, but we consumed this entry + // We need to handle this edge case - for now we'll include it + if (rowCount >= TARGET_RECORD_COUNT || recordsRead >= maxRecords) { + break; + } + } + + currentRowKey = rowKeyStr; + currentRowIndex = rowCount; + + // Set row key + if (rowKeyVector != null) { + rowKeyVector.getMutator().setSafe(currentRowIndex, rowKeyBytes, 0, rowKeyBytes.length); + } + } + + // Skip value population if row_key only query + if (!rowKeyOnly) { + String family = key.getColumnFamily().toString(); + String qualifier = key.getColumnQualifier().toString(); + byte[] valueBytes = entry.getValue().get(); + + MapVector familyVector = getOrCreateFamilyVector(family, true); + NullableVarBinaryVector qualifierVector = getOrCreateColumnVector(familyVector, qualifier); + qualifierVector.getMutator().setSafe(currentRowIndex, valueBytes, 0, valueBytes.length); + } + } + + // Don't forget the last row + if (currentRowKey != null && currentRowIndex == rowCount) { + rowCount++; + recordsRead++; + } + + setOutputRowCount(rowCount); + + logger.debug("Read {} records from {} in {} ms", + rowCount, scanSpec.getTableName(), watch.elapsed(TimeUnit.MILLISECONDS)); + + return rowCount; + } + + /** + * Gets or creates a MapVector for the given column family. + */ + private MapVector getOrCreateFamilyVector(String familyName, boolean allocateOnCreate) { + try { + MapVector v = familyVectorMap.get(familyName); + if (v == null) { + SchemaPath column = SchemaPath.getSimplePath(familyName); + MaterializedField field = MaterializedField.create(column.getAsNamePart().getName(), COLUMN_FAMILY_TYPE); + v = outputMutator.addField(field, MapVector.class); + if (allocateOnCreate) { + v.allocateNew(); + } + getColumns().add(column); + familyVectorMap.put(familyName, v); + } + return v; + } catch (SchemaChangeException e) { + throw new DrillRuntimeException(e); + } + } + + /** + * Gets or creates a column vector within a family MapVector. + */ + private NullableVarBinaryVector getOrCreateColumnVector(MapVector mv, String qualifier) { + int oldSize = mv.size(); + NullableVarBinaryVector v = mv.addOrGet(qualifier, COLUMN_TYPE, NullableVarBinaryVector.class); + if (oldSize != mv.size()) { + v.allocateNew(); + } + return v; + } + + /** + * Sets the value count on all output vectors. + */ + private void setOutputRowCount(int count) { + for (ValueVector vv : familyVectorMap.values()) { + vv.getMutator().setValueCount(count); + } + if (rowKeyVector != null) { + rowKeyVector.getMutator().setValueCount(count); + } + } + + /** + * Checks if a new row can be added to the current batch. + */ + private boolean canAddNewRow(int rowCount) { + return rowCount < TARGET_RECORD_COUNT && + operatorContext.getAllocator().getAllocatedMemory() < MAX_ALLOCATED_MEMORY_PER_BATCH; + } + + @Override + public void close() throws Exception { + // Close the scanner + if (scanner != null) { + try { + scanner.close(); + } catch (Exception e) { + logger.warn("Error closing Accumulo scanner for table {}", scanSpec.getTableName(), e); + } + } + + // Close the client only if we own it (user impersonation mode) + if (ownsClient && client != null) { + try { + logger.debug("Closing owned AccumuloClient for table {}", scanSpec.getTableName()); + client.close(); + } catch (Exception e) { + logger.warn("Error closing Accumulo client for table {}", scanSpec.getTableName(), e); + } + } + } + + @Override + public String toString() { + return "AccumuloRecordReader[table=" + scanSpec.getTableName() + ", ownsClient=" + ownsClient + "]"; + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloScanBatchCreator.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloScanBatchCreator.java new file mode 100644 index 00000000000..944657bcee7 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloScanBatchCreator.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.util.LinkedList; +import java.util.List; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.ops.ExecutorFragmentContext; +import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.impl.BatchCreator; +import org.apache.drill.exec.physical.impl.ScanBatch; +import org.apache.drill.exec.record.RecordBatch; +import org.apache.drill.exec.store.RecordReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; + +/** + * BatchCreator for Accumulo scan operations. + * + *

This class creates the execution pipeline for Accumulo scans by wiring + * together the AccumuloSubScan with AccumuloRecordReaders.

+ * + *

For user impersonation mode, this class creates clients using the + * delegation token passed from the SubScan. When a delegation token is + * present, the reader owns the client and is responsible for closing it.

+ */ +public class AccumuloScanBatchCreator implements BatchCreator { + private static final Logger logger = LoggerFactory.getLogger(AccumuloScanBatchCreator.class); + + @Override + public ScanBatch getBatch( + ExecutorFragmentContext context, + AccumuloSubScan subScan, + List children) throws ExecutionSetupException { + + Preconditions.checkArgument(children.isEmpty(), "AccumuloSubScan should have no children"); + + List readers = new LinkedList<>(); + List columns = subScan.getColumns(); + + if (columns == null) { + columns = GroupScan.ALL_COLUMNS; + } + + try { + // Determine if we need to create a new client from delegation token + // or use the shared service client + AccumuloClient client; + boolean ownsClient; + + if (subScan.hasDelegationToken()) { + // User impersonation mode: create a new client from the delegation token + // The reader will own this client and close it when done + DelegationTokenInfo tokenInfo = subScan.getDelegationTokenInfo(); + logger.debug("Creating Accumulo client from delegation token for user: {}", + tokenInfo.getUserName()); + + client = subScan.getStoragePlugin().getConnectionManager() + .createClientWithDelegationToken(tokenInfo); + ownsClient = true; + + logger.info("Created impersonated Accumulo client for user '{}' to scan table '{}'", + tokenInfo.getUserName(), subScan.getScanSpec().getTableName()); + } else { + // Shared user mode: use the service client + // The reader does not own this client and should not close it + client = subScan.getStoragePlugin().getClient(); + ownsClient = false; + } + + // Create a record reader for this sub-scan + // In the future, we may have multiple readers for different tablet ranges + AccumuloRecordReader reader = new AccumuloRecordReader( + client, + subScan.getScanSpec(), + columns, + getMaxRecords(subScan), + ownsClient); + + readers.add(reader); + + } catch (Exception e) { + throw new ExecutionSetupException( + "Failed to create Accumulo record reader for table: " + subScan.getScanSpec().getTableName(), e); + } + + return new ScanBatch(subScan, context, readers); + } + + /** + * Returns the maximum number of records to read, or -1 for unlimited. + * Uses the more restrictive limit from either the SubScan's maxRecords + * (set by limit pushdown) or the ScanSpec's limit. + */ + private int getMaxRecords(AccumuloSubScan subScan) { + int subScanLimit = subScan.getMaxRecords(); + Integer specLimit = subScan.getScanSpec().getLimit(); + + // If both are set, use the smaller one + if (subScanLimit > 0 && specLimit != null && specLimit > 0) { + return Math.min(subScanLimit, specLimit); + } + // Otherwise, use whichever is set + if (subScanLimit > 0) { + return subScanLimit; + } + if (specLimit != null && specLimit > 0) { + return specLimit; + } + return -1; + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloScanSpec.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloScanSpec.java new file mode 100644 index 00000000000..972c18f4750 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloScanSpec.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.util.List; +import java.util.Objects; + +import org.apache.drill.exec.planner.logical.DrillTableSelection; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Specification for an Accumulo table scan. + * + *

This class captures all scan parameters that may be pushed down to Accumulo, + * including table name, row key ranges, column projections, filters, and limits.

+ */ +public class AccumuloScanSpec implements DrillTableSelection { + + private final String tableName; + private final byte[] startRow; + private final byte[] stopRow; + private final boolean startRowInclusive; + private final boolean stopRowInclusive; + private final List columns; + private final String filterExpression; + private final Integer limit; + private final boolean useSortedScanner; + private final boolean sortDescending; + + @JsonCreator + public AccumuloScanSpec( + @JsonProperty("tableName") String tableName, + @JsonProperty("startRow") byte[] startRow, + @JsonProperty("stopRow") byte[] stopRow, + @JsonProperty("startRowInclusive") Boolean startRowInclusive, + @JsonProperty("stopRowInclusive") Boolean stopRowInclusive, + @JsonProperty("columns") List columns, + @JsonProperty("filterExpression") String filterExpression, + @JsonProperty("limit") Integer limit, + @JsonProperty("useSortedScanner") Boolean useSortedScanner, + @JsonProperty("sortDescending") Boolean sortDescending) { + this.tableName = tableName; + this.startRow = startRow; + this.stopRow = stopRow; + this.startRowInclusive = startRowInclusive != null ? startRowInclusive : true; + this.stopRowInclusive = stopRowInclusive != null ? stopRowInclusive : false; + this.columns = columns; + this.filterExpression = filterExpression; + this.limit = limit; + this.useSortedScanner = useSortedScanner != null ? useSortedScanner : false; + this.sortDescending = sortDescending != null ? sortDescending : false; + } + + /** + * Simple constructor for basic table scan. + */ + public AccumuloScanSpec(String tableName) { + this(tableName, null, null, true, false, null, null, null, false, false); + } + + @JsonProperty("tableName") + public String getTableName() { + return tableName; + } + + @JsonProperty("startRow") + public byte[] getStartRow() { + return startRow; + } + + @JsonProperty("stopRow") + public byte[] getStopRow() { + return stopRow; + } + + @JsonProperty("startRowInclusive") + public boolean isStartRowInclusive() { + return startRowInclusive; + } + + @JsonProperty("stopRowInclusive") + public boolean isStopRowInclusive() { + return stopRowInclusive; + } + + @JsonProperty("columns") + public List getColumns() { + return columns; + } + + @JsonProperty("filterExpression") + public String getFilterExpression() { + return filterExpression; + } + + @JsonProperty("limit") + public Integer getLimit() { + return limit; + } + + @JsonProperty("useSortedScanner") + public boolean isUseSortedScanner() { + return useSortedScanner; + } + + @JsonProperty("sortDescending") + public boolean isSortDescending() { + return sortDescending; + } + + @JsonIgnore + public boolean hasFilter() { + return filterExpression != null && !filterExpression.isEmpty(); + } + + @JsonIgnore + public boolean hasLimit() { + return limit != null && limit > 0; + } + + @JsonIgnore + public boolean hasRowRange() { + return startRow != null || stopRow != null; + } + + @JsonIgnore + @Override + public String digest() { + return toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AccumuloScanSpec that = (AccumuloScanSpec) o; + return Objects.equals(tableName, that.tableName) + && java.util.Arrays.equals(startRow, that.startRow) + && java.util.Arrays.equals(stopRow, that.stopRow) + && startRowInclusive == that.startRowInclusive + && stopRowInclusive == that.stopRowInclusive + && Objects.equals(columns, that.columns) + && Objects.equals(filterExpression, that.filterExpression) + && Objects.equals(limit, that.limit) + && useSortedScanner == that.useSortedScanner + && sortDescending == that.sortDescending; + } + + @Override + public int hashCode() { + int result = Objects.hash(tableName, startRowInclusive, stopRowInclusive, + columns, filterExpression, limit, useSortedScanner, sortDescending); + result = 31 * result + java.util.Arrays.hashCode(startRow); + result = 31 * result + java.util.Arrays.hashCode(stopRow); + return result; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("AccumuloScanSpec{"); + sb.append("tableName='").append(tableName).append('\''); + if (hasRowRange()) { + sb.append(", hasRowRange=true"); + } + if (hasFilter()) { + sb.append(", filterExpression='").append(filterExpression).append('\''); + } + if (hasLimit()) { + sb.append(", limit=").append(limit); + } + if (useSortedScanner) { + sb.append(", useSortedScanner=true"); + } + if (sortDescending) { + sb.append(", sortDescending=true"); + } + sb.append('}'); + return sb.toString(); + } + + /** + * Returns a new AccumuloScanSpec with the filter expression set. + */ + public AccumuloScanSpec withFilter(String filterExpression) { + return new AccumuloScanSpec(tableName, startRow, stopRow, startRowInclusive, + stopRowInclusive, columns, filterExpression, limit, useSortedScanner, sortDescending); + } + + /** + * Returns a new AccumuloScanSpec with the limit set. + */ + public AccumuloScanSpec withLimit(Integer limit) { + return new AccumuloScanSpec(tableName, startRow, stopRow, startRowInclusive, + stopRowInclusive, columns, filterExpression, limit, useSortedScanner, sortDescending); + } + + /** + * Returns a new AccumuloScanSpec with sorted scanner mode. + */ + public AccumuloScanSpec withSortedScanner(boolean useSortedScanner) { + return new AccumuloScanSpec(tableName, startRow, stopRow, startRowInclusive, + stopRowInclusive, columns, filterExpression, limit, useSortedScanner, sortDescending); + } + + /** + * Returns a new AccumuloScanSpec with sort order (ascending or descending). + */ + public AccumuloScanSpec withSortOrder(boolean descending) { + return new AccumuloScanSpec(tableName, startRow, stopRow, startRowInclusive, + stopRowInclusive, columns, filterExpression, limit, true, descending); + } + + /** + * Returns a new AccumuloScanSpec with column projection. + */ + public AccumuloScanSpec withColumns(List columns) { + return new AccumuloScanSpec(tableName, startRow, stopRow, startRowInclusive, + stopRowInclusive, columns, filterExpression, limit, useSortedScanner, sortDescending); + } + + /** + * Specification for a column to scan from Accumulo. + */ + public static class AccumuloColumnSpec { + private final String columnFamily; + private final String columnQualifier; + private final String drillColumnName; + + @JsonCreator + public AccumuloColumnSpec( + @JsonProperty("columnFamily") String columnFamily, + @JsonProperty("columnQualifier") String columnQualifier, + @JsonProperty("drillColumnName") String drillColumnName) { + this.columnFamily = columnFamily; + this.columnQualifier = columnQualifier; + this.drillColumnName = drillColumnName; + } + + @JsonProperty("columnFamily") + public String getColumnFamily() { + return columnFamily; + } + + @JsonProperty("columnQualifier") + public String getColumnQualifier() { + return columnQualifier; + } + + @JsonProperty("drillColumnName") + public String getDrillColumnName() { + return drillColumnName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AccumuloColumnSpec that = (AccumuloColumnSpec) o; + return Objects.equals(columnFamily, that.columnFamily) + && Objects.equals(columnQualifier, that.columnQualifier) + && Objects.equals(drillColumnName, that.drillColumnName); + } + + @Override + public int hashCode() { + return Objects.hash(columnFamily, columnQualifier, drillColumnName); + } + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloSchemaFactory.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloSchemaFactory.java new file mode 100644 index 00000000000..71f3780d40d --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloSchemaFactory.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.io.IOException; +import java.util.Collections; +import java.util.Set; + +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Table; +import org.apache.drill.exec.store.AbstractSchema; +import org.apache.drill.exec.store.AbstractSchemaFactory; +import org.apache.drill.exec.store.SchemaConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Schema factory for Accumulo storage plugin. + * + *

Responsible for registering the Accumulo schema and discovering tables.

+ */ +public class AccumuloSchemaFactory extends AbstractSchemaFactory { + private static final Logger logger = LoggerFactory.getLogger(AccumuloSchemaFactory.class); + + private final AccumuloStoragePlugin plugin; + + public AccumuloSchemaFactory(AccumuloStoragePlugin plugin) { + super(plugin.getName()); + this.plugin = plugin; + } + + @Override + public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException { + AccumuloSchema schema = new AccumuloSchema(getName()); + SchemaPlus schemaPlus = parent.add(getName(), schema); + schema.setHolder(schemaPlus); + } + + /** + * Accumulo schema implementation. + */ + class AccumuloSchema extends AbstractSchema { + + AccumuloSchema(String name) { + super(Collections.emptyList(), name); + } + + public void setHolder(SchemaPlus plusOfThis) { + // No-op for now + } + + @Override + public AbstractSchema getSubSchema(String name) { + return null; + } + + @Override + public Set getSubSchemaNames() { + return Collections.emptySet(); + } + + @Override + public Table getTable(String name) { + AccumuloScanSpec scanSpec = new AccumuloScanSpec(name); + try { + return new DrillAccumuloTable(plugin, getName(), scanSpec); + } catch (Exception e) { + logger.warn("Failure while loading table '{}' for schema '{}'.", name, getName(), e); + return null; + } + } + + @Override + public Set getTableNames() { + try { + return plugin.getSchemaProvider().discoverTableNames(plugin.getClient()); + } catch (Exception e) { + logger.warn("Failure while loading table names for schema '{}'.", getName(), e); + return Collections.emptySet(); + } + } + + @Override + public String getTypeName() { + return AccumuloStoragePluginConfig.NAME; + } + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePlugin.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePlugin.java new file mode 100644 index 00000000000..acbf80dc3b4 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePlugin.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.io.IOException; +import java.util.Set; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.common.logical.StoragePluginConfig.AuthMode; +import org.apache.drill.exec.ops.OptimizerRulesContext; +import org.apache.drill.exec.store.accumulo.schema.AccumuloSchemaProvider; +import org.apache.drill.exec.store.accumulo.schema.MetadataTableSchemaProvider; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.planner.PlannerPhase; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.AbstractStoragePlugin; +import org.apache.drill.exec.store.SchemaConfig; +import org.apache.drill.exec.store.StoragePluginOptimizerRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.ImmutableSet; + +/** + * Accumulo storage plugin for Apache Drill. + * + *

This plugin provides read access to Apache Accumulo tables, + * with support for filter, projection, limit, and sort pushdowns.

+ * + *

Authentication modes:

+ *
    + *
  • PASSWORD: Username/password authentication
    • + *
  • KERBEROS + SHARED_USER: Service principal for all queries
    • + *
  • KERBEROS + USER_IMPERSONATION: Service authenticates, then impersonates + * Drill user via delegation token
    • + *
+ */ +public class AccumuloStoragePlugin extends AbstractStoragePlugin { + private static final Logger logger = LoggerFactory.getLogger(AccumuloStoragePlugin.class); + + private final AccumuloStoragePluginConfig config; + private final AccumuloSchemaFactory schemaFactory; + private final AccumuloSchemaProvider schemaProvider; + private final AccumuloConnectionManager connectionManager; + + public AccumuloStoragePlugin( + AccumuloStoragePluginConfig config, + DrillbitContext context, + String name) { + super(context, name); + this.config = config; + this.schemaProvider = new MetadataTableSchemaProvider(config.getSchemaMetadataTable()); + this.schemaFactory = new AccumuloSchemaFactory(this); + this.connectionManager = new AccumuloConnectionManager(config); + + logger.info("Initialized Accumulo storage plugin '{}' with ZooKeeper quorum: {}, authType: {}, authMode: {}", + name, config.getZookeeperQuorum(), config.getAuthenticationType(), config.getAuthMode()); + } + + /** + * Returns the schema provider for this plugin. + */ + public AccumuloSchemaProvider getSchemaProvider() { + return schemaProvider; + } + + /** + * Returns the connection manager for this plugin. + */ + public AccumuloConnectionManager getConnectionManager() { + return connectionManager; + } + + @Override + public boolean supportsRead() { + return true; + } + + @Override + public AccumuloStoragePluginConfig getConfig() { + return config; + } + + /** + * Returns a shared AccumuloClient for service-level operations. + * + *

This is the service client that authenticates using the configured + * credentials (password or Kerberos). For user impersonation, use + * {@link #getClientForUser(String)} instead.

+ * + * @return the service Accumulo client + * @throws UserException if connection fails + */ + public AccumuloClient getClient() { + return connectionManager.getServiceClient(); + } + + /** + * Returns an AccumuloClient for the specified user. + * + *

Behavior depends on the auth mode:

+ *
    + *
  • SHARED_USER: Returns the service client (same for all users)
    • + *
  • USER_IMPERSONATION: Returns a client using the user's delegation token
    • + *
+ * + * @param userName the Drill query user name + * @return an AccumuloClient for the user + */ + public AccumuloClient getClientForUser(String userName) { + return connectionManager.getClientForUser(userName); + } + + /** + * Generates a delegation token for the specified user. + * + *

This is used in distributed execution to pass the user's credentials + * to executor fragments.

+ * + * @param userName the user to generate a token for + * @return the delegation token info, or null if impersonation is not enabled + */ + public DelegationTokenInfo generateDelegationToken(String userName) { + if (!config.isUserImpersonationEnabled() || !config.isUseDelegationTokens()) { + return null; + } + + try { + return connectionManager.getDelegationToken(userName); + } catch (Exception e) { + throw UserException.connectionError(e) + .message("Failed to generate delegation token for user '%s'", userName) + .addContext("Plugin", getName()) + .build(logger); + } + } + + @Override + public AbstractGroupScan getPhysicalScan(String userName, JSONOptions selection) throws IOException { + AccumuloScanSpec scanSpec = selection.getListWith(new TypeReference() {}); + + // Generate delegation token if user impersonation is enabled + DelegationTokenInfo delegationToken = null; + if (config.isUserImpersonationEnabled()) { + delegationToken = generateDelegationToken(userName); + logger.debug("Generated delegation token for user '{}' in physical scan", userName); + } + + return new AccumuloGroupScan(userName, this, scanSpec, null, -1, delegationToken); + } + + @Override + public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException { + schemaFactory.registerSchemas(schemaConfig, parent); + } + + @Override + public Set getOptimizerRules( + OptimizerRulesContext optimizerRulesContext, + PlannerPhase phase) { + switch (phase) { + case LOGICAL: + return ImmutableSet.of( + AccumuloPushSortIntoScan.SORT_ON_SCAN + ); + case PHYSICAL: + return ImmutableSet.of( + AccumuloPushFilterIntoScan.FILTER_ON_SCAN, + AccumuloPushFilterIntoScan.FILTER_ON_PROJECT + ); + default: + return ImmutableSet.of(); + } + } + + @Override + public void close() throws Exception { + logger.debug("Closing Accumulo storage plugin: {}", getName()); + connectionManager.close(); + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePluginConfig.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePluginConfig.java new file mode 100644 index 00000000000..7a838feb838 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePluginConfig.java @@ -0,0 +1,414 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.util.Objects; +import java.util.Optional; + +import org.apache.drill.common.PlanStringBuilder; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.common.logical.security.CredentialsProvider; +import org.apache.drill.exec.store.security.CredentialProviderUtils; +import org.apache.drill.exec.proto.UserBitShared.UserCredentials; +import org.apache.drill.exec.store.security.UsernamePasswordCredentials; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; + +/** + * Configuration for the Accumulo storage plugin. + * + *

This configuration supports connecting to an Accumulo cluster via ZooKeeper + * and includes settings for authentication, Kerberos, user impersonation, + * and optional schema metadata table.

+ * + *

Password Authentication Example:

+ * 
+ * {
+ *   "type": "accumulo",
+ *   "zookeeperQuorum": "localhost:2181",
+ *   "instanceName": "accumulo",
+ *   "username": "root",
+ *   "password": "secret",
+ *   "enabled": true
+ * }
+ *
+ * + *

Kerberos Authentication Example:

+ * 
+ * {
+ *   "type": "accumulo",
+ *   "zookeeperQuorum": "zk1:2181,zk2:2181",
+ *   "instanceName": "accumulo",
+ *   "authenticationType": "KERBEROS",
+ *   "principal": "drill/hostname@REALM",
+ *   "keytabPath": "/etc/security/keytabs/drill.keytab",
+ *   "saslQop": "auth",
+ *   "useDelegationTokens": true,
+ *   "authMode": "USER_IMPERSONATION",
+ *   "enabled": true
+ * }
+ *
+ */ +@JsonTypeName(AccumuloStoragePluginConfig.NAME) +public class AccumuloStoragePluginConfig extends StoragePluginConfig { + + public static final String NAME = "accumulo"; + + /** + * Default SASL QoP (Quality of Protection) value. + */ + public static final String DEFAULT_SASL_QOP = "auth"; + + /** + * Default Accumulo service name for SASL authentication. + */ + public static final String DEFAULT_ACCUMULO_SERVICE_PRIMARY = "accumulo"; + + // ===== Connection settings ===== + + /** + * Comma-separated list of ZooKeeper servers (host:port format). + * Example: "zk1:2181,zk2:2181,zk3:2181" + */ + private final String zookeeperQuorum; + + /** + * The Accumulo instance name. + */ + private final String instanceName; + + // ===== Password authentication settings (for backward compatibility) ===== + + /** + * The username for password authentication. + * Deprecated: prefer using credentialsProvider. + */ + private final String username; + + /** + * The password for password authentication. + * Deprecated: prefer using credentialsProvider. + */ + private final String password; + + // ===== Kerberos authentication settings ===== + + /** + * The authentication type: PASSWORD or KERBEROS. + */ + private final AccumuloAuthType authenticationType; + + /** + * Kerberos principal for service authentication. + * Format: primary/instance@REALM (e.g., "drill/hostname@EXAMPLE.COM") + */ + private final String principal; + + /** + * Path to the Kerberos keytab file. + */ + private final String keytabPath; + + /** + * SASL Quality of Protection: "auth", "auth-int", or "auth-conf". + * - auth: authentication only + * - auth-int: authentication + integrity protection + * - auth-conf: authentication + integrity + confidentiality (encryption) + */ + private final String saslQop; + + /** + * Accumulo service primary name for SASL authentication. + * Default is "accumulo". + */ + private final String accumuloServicePrimary; + + /** + * Whether to use delegation tokens for distributed execution. + * When true, the service will obtain delegation tokens for query users. + */ + private final boolean useDelegationTokens; + + // ===== Optional settings ===== + + /** + * Optional name of the schema metadata table. + * If set, the plugin will look for table schema definitions in this Accumulo table. + * Default is "_drill_schema". + */ + private final String schemaMetadataTable; + + /** + * Timeout in milliseconds for Accumulo client operations. + * Default is 30000 (30 seconds). + */ + private final Integer clientTimeout; + + /** + * Number of threads for BatchScanner operations. + * Default is 10. + */ + private final Integer batchScannerThreads; + + @JsonCreator + public AccumuloStoragePluginConfig( + @JsonProperty("zookeeperQuorum") String zookeeperQuorum, + @JsonProperty("instanceName") String instanceName, + @JsonProperty("username") String username, + @JsonProperty("password") String password, + @JsonProperty("authenticationType") String authenticationType, + @JsonProperty("principal") String principal, + @JsonProperty("keytabPath") String keytabPath, + @JsonProperty("saslQop") String saslQop, + @JsonProperty("accumuloServicePrimary") String accumuloServicePrimary, + @JsonProperty("useDelegationTokens") Boolean useDelegationTokens, + @JsonProperty("authMode") String authMode, + @JsonProperty("credentialsProvider") CredentialsProvider credentialsProvider, + @JsonProperty("schemaMetadataTable") String schemaMetadataTable, + @JsonProperty("clientTimeout") Integer clientTimeout, + @JsonProperty("batchScannerThreads") Integer batchScannerThreads) { + + super( + CredentialProviderUtils.getCredentialsProvider(username, password, credentialsProvider), + credentialsProvider == null, + AuthMode.parseOrDefault(authMode, AuthMode.SHARED_USER) + ); + + this.zookeeperQuorum = zookeeperQuorum; + this.instanceName = instanceName; + this.username = username; + this.password = password; + + this.authenticationType = AccumuloAuthType.parseOrDefault(authenticationType, AccumuloAuthType.PASSWORD); + this.principal = principal; + this.keytabPath = keytabPath; + this.saslQop = saslQop != null ? saslQop : DEFAULT_SASL_QOP; + this.accumuloServicePrimary = accumuloServicePrimary != null ? accumuloServicePrimary : DEFAULT_ACCUMULO_SERVICE_PRIMARY; + this.useDelegationTokens = useDelegationTokens != null ? useDelegationTokens : false; + + this.schemaMetadataTable = schemaMetadataTable != null ? schemaMetadataTable : "_drill_schema"; + this.clientTimeout = clientTimeout != null ? clientTimeout : 30000; + this.batchScannerThreads = batchScannerThreads != null ? batchScannerThreads : 10; + } + + /** + * Simplified constructor for password authentication (backward compatible). + */ + public AccumuloStoragePluginConfig( + String zookeeperQuorum, + String instanceName, + String username, + String password) { + this(zookeeperQuorum, instanceName, username, password, + null, null, null, null, null, null, null, null, null, null, null); + } + + // ===== Connection Getters ===== + + @JsonProperty("zookeeperQuorum") + public String getZookeeperQuorum() { + return zookeeperQuorum; + } + + @JsonProperty("instanceName") + public String getInstanceName() { + return instanceName; + } + + // ===== Password Auth Getters ===== + + @JsonProperty("username") + public String getUsername() { + return username; + } + + @JsonProperty("password") + public String getPassword() { + return password; + } + + // ===== Kerberos Auth Getters ===== + + @JsonProperty("authenticationType") + public AccumuloAuthType getAuthenticationType() { + return authenticationType; + } + + @JsonProperty("principal") + public String getPrincipal() { + return principal; + } + + @JsonProperty("keytabPath") + public String getKeytabPath() { + return keytabPath; + } + + @JsonProperty("saslQop") + public String getSaslQop() { + return saslQop; + } + + @JsonProperty("accumuloServicePrimary") + public String getAccumuloServicePrimary() { + return accumuloServicePrimary; + } + + @JsonProperty("useDelegationTokens") + public boolean isUseDelegationTokens() { + return useDelegationTokens; + } + + // ===== Optional Settings Getters ===== + + @JsonProperty("schemaMetadataTable") + public String getSchemaMetadataTable() { + return schemaMetadataTable; + } + + @JsonProperty("clientTimeout") + public Integer getClientTimeout() { + return clientTimeout; + } + + @JsonProperty("batchScannerThreads") + public Integer getBatchScannerThreads() { + return batchScannerThreads; + } + + // ===== Credential Helper Methods ===== + + /** + * Returns username/password credentials for the specified user context. + * + *

For SHARED_USER mode, returns the configured credentials. + * For USER_TRANSLATION mode, returns per-user credentials from the provider.

+ * + * @param userCredentials the query user credentials (may be null for SHARED_USER) + * @return Optional containing credentials if available + */ + @JsonIgnore + public Optional getUsernamePasswordCredentials( + UserCredentials userCredentials) { + + switch (authMode) { + case SHARED_USER: + return new UsernamePasswordCredentials.Builder() + .setCredentialsProvider(credentialsProvider) + .build(); + + case USER_TRANSLATION: + if (userCredentials == null) { + return Optional.empty(); + } + return new UsernamePasswordCredentials.Builder() + .setCredentialsProvider(credentialsProvider) + .setQueryUser(userCredentials.getUserName()) + .build(); + + case USER_IMPERSONATION: + // For impersonation, service credentials are used for initial auth + return new UsernamePasswordCredentials.Builder() + .setCredentialsProvider(credentialsProvider) + .build(); + + default: + return Optional.empty(); + } + } + + /** + * Returns whether user impersonation is enabled. + */ + @JsonIgnore + public boolean isUserImpersonationEnabled() { + return authMode == AuthMode.USER_IMPERSONATION; + } + + /** + * Returns whether user translation is enabled. + */ + @JsonIgnore + public boolean isUserTranslationEnabled() { + return authMode == AuthMode.USER_TRANSLATION; + } + + /** + * Returns whether Kerberos authentication is configured. + */ + @JsonIgnore + public boolean isKerberosEnabled() { + return authenticationType == AccumuloAuthType.KERBEROS; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AccumuloStoragePluginConfig that = (AccumuloStoragePluginConfig) o; + return useDelegationTokens == that.useDelegationTokens + && Objects.equals(zookeeperQuorum, that.zookeeperQuorum) + && Objects.equals(instanceName, that.instanceName) + && Objects.equals(username, that.username) + && Objects.equals(password, that.password) + && authenticationType == that.authenticationType + && Objects.equals(principal, that.principal) + && Objects.equals(keytabPath, that.keytabPath) + && Objects.equals(saslQop, that.saslQop) + && Objects.equals(accumuloServicePrimary, that.accumuloServicePrimary) + && Objects.equals(schemaMetadataTable, that.schemaMetadataTable) + && Objects.equals(clientTimeout, that.clientTimeout) + && Objects.equals(batchScannerThreads, that.batchScannerThreads) + && Objects.equals(authMode, that.authMode); + } + + @Override + public int hashCode() { + return Objects.hash(zookeeperQuorum, instanceName, username, password, + authenticationType, principal, keytabPath, saslQop, accumuloServicePrimary, + useDelegationTokens, schemaMetadataTable, clientTimeout, batchScannerThreads, + authMode); + } + + @Override + public String toString() { + return new PlanStringBuilder(this) + .field("zookeeperQuorum", zookeeperQuorum) + .field("instanceName", instanceName) + .field("authenticationType", authenticationType) + .field("authMode", authMode) + .field("username", username) + .maskedField("password", password) + .field("principal", principal) + .maskedField("keytabPath", keytabPath) + .field("saslQop", saslQop) + .field("accumuloServicePrimary", accumuloServicePrimary) + .field("useDelegationTokens", useDelegationTokens) + .field("schemaMetadataTable", schemaMetadataTable) + .field("clientTimeout", clientTimeout) + .field("batchScannerThreads", batchScannerThreads) + .toString(); + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloSubScan.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloSubScan.java new file mode 100644 index 00000000000..f95bef0eee6 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloSubScan.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.physical.base.AbstractBase; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.base.PhysicalVisitor; +import org.apache.drill.exec.physical.base.SubScan; +import org.apache.drill.exec.store.StoragePluginRegistry; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.base.Preconditions; + +/** + * Accumulo sub-scan for a specific portion of an Accumulo table. + * + *

In the future, this will represent a scan on specific tablets. + * For now, it represents a full table scan.

+ * + *

For user impersonation mode, this class carries a delegation token + * that was generated during planning and is used at execution time to + * create an Accumulo client with the user's identity.

+ */ +@JsonTypeName("accumulo-sub-scan") +public class AccumuloSubScan extends AbstractBase implements SubScan { + + public static final String OPERATOR_TYPE = "ACCUMULO_SUB_SCAN"; + + private final AccumuloStoragePlugin storagePlugin; + private final AccumuloScanSpec scanSpec; + private final List columns; + private final int maxRecords; + + /** + * Delegation token for user impersonation in distributed execution. + * When present, the record reader will use this token to create a + * user-impersonated Accumulo client. + */ + private final DelegationTokenInfo delegationTokenInfo; + + @JsonCreator + public AccumuloSubScan( + @JacksonInject StoragePluginRegistry registry, + @JsonProperty("userName") String userName, + @JsonProperty("storagePluginConfig") AccumuloStoragePluginConfig storagePluginConfig, + @JsonProperty("scanSpec") AccumuloScanSpec scanSpec, + @JsonProperty("columns") List columns, + @JsonProperty("maxRecords") int maxRecords, + @JsonProperty("delegationTokenInfo") DelegationTokenInfo delegationTokenInfo) throws ExecutionSetupException { + this(userName, registry.resolve(storagePluginConfig, AccumuloStoragePlugin.class), + scanSpec, columns, maxRecords, delegationTokenInfo); + } + + public AccumuloSubScan( + String userName, + AccumuloStoragePlugin storagePlugin, + AccumuloScanSpec scanSpec, + List columns) { + this(userName, storagePlugin, scanSpec, columns, -1, null); + } + + public AccumuloSubScan( + String userName, + AccumuloStoragePlugin storagePlugin, + AccumuloScanSpec scanSpec, + List columns, + int maxRecords) { + this(userName, storagePlugin, scanSpec, columns, maxRecords, null); + } + + public AccumuloSubScan( + String userName, + AccumuloStoragePlugin storagePlugin, + AccumuloScanSpec scanSpec, + List columns, + int maxRecords, + DelegationTokenInfo delegationTokenInfo) { + super(userName); + this.storagePlugin = storagePlugin; + this.scanSpec = scanSpec; + this.columns = columns; + this.maxRecords = maxRecords; + this.delegationTokenInfo = delegationTokenInfo; + } + + @JsonProperty("storagePluginConfig") + public AccumuloStoragePluginConfig getStoragePluginConfig() { + return storagePlugin.getConfig(); + } + + @JsonProperty("scanSpec") + public AccumuloScanSpec getScanSpec() { + return scanSpec; + } + + @JsonProperty("columns") + public List getColumns() { + return columns; + } + + @JsonProperty("maxRecords") + public int getMaxRecords() { + return maxRecords; + } + + @JsonProperty("delegationTokenInfo") + public DelegationTokenInfo getDelegationTokenInfo() { + return delegationTokenInfo; + } + + @JsonIgnore + public AccumuloStoragePlugin getStoragePlugin() { + return storagePlugin; + } + + /** + * Returns true if this sub-scan has a delegation token for user impersonation. + */ + @JsonIgnore + public boolean hasDelegationToken() { + return delegationTokenInfo != null; + } + + @Override + public boolean isExecutable() { + return false; + } + + @Override + public T accept(PhysicalVisitor physicalVisitor, X value) throws E { + return physicalVisitor.visitSubScan(this, value); + } + + @Override + public PhysicalOperator getNewWithChildren(List children) { + Preconditions.checkArgument(children.isEmpty()); + return new AccumuloSubScan(getUserName(), storagePlugin, scanSpec, columns, maxRecords, delegationTokenInfo); + } + + @Override + public Iterator iterator() { + return Collections.emptyIterator(); + } + + @Override + public String getOperatorType() { + return OPERATOR_TYPE; + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloTypeConverter.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloTypeConverter.java new file mode 100644 index 00000000000..50f70050442 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloTypeConverter.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; + +import org.apache.drill.exec.store.accumulo.schema.AccumuloColumnType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for converting Accumulo byte arrays to Java types. + * + *

Accumulo stores all data as byte arrays. This class provides methods to + * convert those byte arrays to appropriate Java types based on the configured + * column type in the schema.

+ * + *

Conversion strategies:

+ *
    + *
  • String types: UTF-8 decode the bytes
    • + *
  • Numeric types: Try parsing string representation first, fall back to binary
    • + *
  • Boolean: Check for "true"/"false" strings or binary 0/1
    • + *
  • Temporal types: Parse ISO-8601 string or epoch milliseconds
    • + *
+ */ +public final class AccumuloTypeConverter { + private static final Logger logger = LoggerFactory.getLogger(AccumuloTypeConverter.class); + + private AccumuloTypeConverter() { + // Utility class + } + + /** + * Converts Accumulo byte array to the appropriate Java type based on column type. + * + * @param value the byte array from Accumulo + * @param columnType the target type for conversion + * @return the converted Java object, or null if conversion fails + */ + public static Object convert(byte[] value, AccumuloColumnType columnType) { + if (value == null || value.length == 0) { + return null; + } + + switch (columnType) { + case VARCHAR: + return toVarchar(value); + case INT: + case INTEGER: + return toInteger(value); + case BIGINT: + case LONG: + return toLong(value); + case SMALLINT: + return toShort(value); + case TINYINT: + return toByte(value); + case FLOAT: + return toFloat(value); + case DOUBLE: + return toDouble(value); + case DECIMAL: + return toDecimal(value); + case BOOLEAN: + return toBoolean(value); + case DATE: + return toDate(value); + case TIME: + return toTime(value); + case TIMESTAMP: + return toTimestamp(value); + case VARBINARY: + return value; + case ANY: + default: + // For ANY type, return as string + return toVarchar(value); + } + } + + /** + * Converts byte array to String using UTF-8 encoding. + */ + public static String toVarchar(byte[] value) { + return new String(value, StandardCharsets.UTF_8); + } + + /** + * Converts byte array to Integer. + * Tries string parsing first, then binary interpretation. + */ + public static Integer toInteger(byte[] value) { + // Try parsing as string first (more common) + try { + String strValue = new String(value, StandardCharsets.UTF_8).trim(); + return Integer.parseInt(strValue); + } catch (NumberFormatException e) { + // Fall back to binary interpretation + if (value.length == 4) { + return ByteBuffer.wrap(value).getInt(); + } + logger.debug("Failed to convert byte array to Integer"); + return null; + } + } + + /** + * Converts byte array to Long. + * Tries string parsing first, then binary interpretation. + */ + public static Long toLong(byte[] value) { + try { + String strValue = new String(value, StandardCharsets.UTF_8).trim(); + return Long.parseLong(strValue); + } catch (NumberFormatException e) { + if (value.length == 8) { + return ByteBuffer.wrap(value).getLong(); + } + logger.debug("Failed to convert byte array to Long"); + return null; + } + } + + /** + * Converts byte array to Short. + */ + public static Short toShort(byte[] value) { + try { + String strValue = new String(value, StandardCharsets.UTF_8).trim(); + return Short.parseShort(strValue); + } catch (NumberFormatException e) { + if (value.length == 2) { + return ByteBuffer.wrap(value).getShort(); + } + logger.debug("Failed to convert byte array to Short"); + return null; + } + } + + /** + * Converts byte array to Byte. + */ + public static Byte toByte(byte[] value) { + try { + String strValue = new String(value, StandardCharsets.UTF_8).trim(); + return Byte.parseByte(strValue); + } catch (NumberFormatException e) { + if (value.length == 1) { + return value[0]; + } + logger.debug("Failed to convert byte array to Byte"); + return null; + } + } + + /** + * Converts byte array to Float. + */ + public static Float toFloat(byte[] value) { + try { + String strValue = new String(value, StandardCharsets.UTF_8).trim(); + return Float.parseFloat(strValue); + } catch (NumberFormatException e) { + if (value.length == 4) { + return ByteBuffer.wrap(value).getFloat(); + } + logger.debug("Failed to convert byte array to Float"); + return null; + } + } + + /** + * Converts byte array to Double. + */ + public static Double toDouble(byte[] value) { + try { + String strValue = new String(value, StandardCharsets.UTF_8).trim(); + return Double.parseDouble(strValue); + } catch (NumberFormatException e) { + if (value.length == 8) { + return ByteBuffer.wrap(value).getDouble(); + } + logger.debug("Failed to convert byte array to Double"); + return null; + } + } + + /** + * Converts byte array to BigDecimal. + */ + public static BigDecimal toDecimal(byte[] value) { + try { + String strValue = new String(value, StandardCharsets.UTF_8).trim(); + return new BigDecimal(strValue); + } catch (NumberFormatException e) { + logger.debug("Failed to convert byte array to BigDecimal"); + return null; + } + } + + /** + * Converts byte array to Boolean. + * Accepts "true"/"false" strings (case insensitive), "1"/"0", or binary 0/1. + */ + public static Boolean toBoolean(byte[] value) { + // First try string parsing (handles "true", "false", "1", "0", etc.) + String strValue = new String(value, StandardCharsets.UTF_8).trim().toLowerCase(); + if ("true".equals(strValue) || "1".equals(strValue) || "yes".equals(strValue)) { + return true; + } else if ("false".equals(strValue) || "0".equals(strValue) || "no".equals(strValue)) { + return false; + } + + // Fall back to binary interpretation for single byte + if (value.length == 1) { + return value[0] != 0; + } + + logger.debug("Failed to convert byte array to Boolean: {}", strValue); + return null; + } + + /** + * Converts byte array to LocalDate. + * Tries ISO-8601 date format first, then epoch days as long. + * + * @return epoch milliseconds at start of day UTC, or null if conversion fails + */ + public static Long toDate(byte[] value) { + String strValue = new String(value, StandardCharsets.UTF_8).trim(); + try { + LocalDate date = LocalDate.parse(strValue); + return date.atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli(); + } catch (DateTimeParseException e) { + // Try as epoch days + try { + long epochDays = Long.parseLong(strValue); + return LocalDate.ofEpochDay(epochDays).atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli(); + } catch (NumberFormatException e2) { + logger.debug("Failed to convert byte array to Date: {}", strValue); + return null; + } + } + } + + /** + * Converts byte array to LocalTime. + * Tries ISO-8601 time format first. + * + * @return milliseconds since midnight, or null if conversion fails + */ + public static Integer toTime(byte[] value) { + String strValue = new String(value, StandardCharsets.UTF_8).trim(); + try { + LocalTime time = LocalTime.parse(strValue); + return (int) (time.toNanoOfDay() / 1_000_000); + } catch (DateTimeParseException e) { + // Try as milliseconds since midnight + try { + return Integer.parseInt(strValue); + } catch (NumberFormatException e2) { + logger.debug("Failed to convert byte array to Time: {}", strValue); + return null; + } + } + } + + /** + * Converts byte array to Instant. + * Tries ISO-8601 timestamp format first, then epoch milliseconds. + * + * @return epoch milliseconds, or null if conversion fails + */ + public static Long toTimestamp(byte[] value) { + String strValue = new String(value, StandardCharsets.UTF_8).trim(); + try { + Instant instant = Instant.parse(strValue); + return instant.toEpochMilli(); + } catch (DateTimeParseException e) { + // Try as epoch milliseconds + try { + return Long.parseLong(strValue); + } catch (NumberFormatException e2) { + logger.debug("Failed to convert byte array to Timestamp: {}", strValue); + return null; + } + } + } + + /** + * Returns the string representation of the byte array for display/debugging. + */ + public static String toDisplayString(byte[] value) { + if (value == null) { + return "null"; + } + if (value.length == 0) { + return ""; + } + + // Try to interpret as UTF-8 string + String strValue = new String(value, StandardCharsets.UTF_8); + + // Check if it looks like valid text (printable characters) + boolean isPrintable = strValue.chars().allMatch(c -> + !Character.isISOControl(c) || Character.isWhitespace(c)); + + if (isPrintable) { + return strValue; + } + + // Return hex representation for binary data + StringBuilder hex = new StringBuilder("0x"); + for (byte b : value) { + hex.append(String.format("%02X", b)); + } + return hex.toString(); + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DelegationTokenInfo.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DelegationTokenInfo.java new file mode 100644 index 00000000000..48cc2261527 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DelegationTokenInfo.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.util.Base64; +import java.util.Objects; + +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.AuthenticationTokenSerializer; +import org.apache.accumulo.core.client.security.tokens.DelegationToken; +import org.apache.drill.common.PlanStringBuilder; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Serializable wrapper for Accumulo delegation tokens. + * + *

This class enables delegation tokens to be passed across Drill's distributed + * execution pipeline via JSON serialization. The token is stored as a Base64-encoded + * string for safe transport through JSON.

+ * + *

Usage flow:

+ *
    + *
  1. Service client obtains a delegation token for a user
    2. + *
  2. Token is wrapped in DelegationTokenInfo and attached to AccumuloGroupScan
    3. + *
  3. Token is serialized to JSON for distributed planning
    4. + *
  4. At execution time, token is deserialized and used to create a client
    5. + *
+ */ +public class DelegationTokenInfo { + + /** + * The username this delegation token was created for. + */ + private final String userName; + + /** + * Base64-encoded serialized delegation token. + * Using Base64 string instead of raw byte[] for safer JSON serialization. + */ + private final String serializedToken; + + /** + * The fully qualified class name of the token implementation. + * Needed for deserialization. + */ + private final String tokenClassName; + + /** + * Time when this delegation token was created (epoch millis). + * Used for cache eviction and token refresh decisions. + */ + private final long creationTime; + + @JsonCreator + public DelegationTokenInfo( + @JsonProperty("userName") String userName, + @JsonProperty("serializedToken") String serializedToken, + @JsonProperty("tokenClassName") String tokenClassName, + @JsonProperty("creationTime") long creationTime) { + this.userName = userName; + this.serializedToken = serializedToken; + this.tokenClassName = tokenClassName; + this.creationTime = creationTime; + } + + /** + * Creates a DelegationTokenInfo from an Accumulo DelegationToken. + * + * @param userName the user this token is for + * @param token the Accumulo delegation token + * @return a new DelegationTokenInfo + */ + public static DelegationTokenInfo fromDelegationToken(String userName, DelegationToken token) { + byte[] tokenBytes = AuthenticationTokenSerializer.serialize(token); + String serialized = Base64.getEncoder().encodeToString(tokenBytes); + String className = token.getClass().getName(); + return new DelegationTokenInfo(userName, serialized, className, System.currentTimeMillis()); + } + + /** + * Converts this wrapper back to an Accumulo AuthenticationToken. + * + *

Note: This returns an AuthenticationToken (the parent interface) rather than + * DelegationToken because the deserialization uses the stored class name.

+ * + * @return the deserialized AuthenticationToken + */ + @SuppressWarnings("unchecked") + @JsonIgnore + public AuthenticationToken toAuthenticationToken() { + byte[] tokenBytes = Base64.getDecoder().decode(serializedToken); + try { + Class tokenClass = + (Class) Class.forName(tokenClassName); + return AuthenticationTokenSerializer.deserialize(tokenClass, tokenBytes); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Failed to load token class: " + tokenClassName, e); + } + } + + @JsonProperty("userName") + public String getUserName() { + return userName; + } + + @JsonProperty("serializedToken") + public String getSerializedToken() { + return serializedToken; + } + + @JsonProperty("tokenClassName") + public String getTokenClassName() { + return tokenClassName; + } + + @JsonProperty("creationTime") + public long getCreationTime() { + return creationTime; + } + + /** + * Returns the age of this token in milliseconds. + */ + @JsonIgnore + public long getAgeMillis() { + return System.currentTimeMillis() - creationTime; + } + + /** + * Checks if this token is older than the specified age. + * + * @param maxAgeMillis maximum acceptable age in milliseconds + * @return true if the token is older than maxAgeMillis + */ + @JsonIgnore + public boolean isOlderThan(long maxAgeMillis) { + return getAgeMillis() > maxAgeMillis; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DelegationTokenInfo that = (DelegationTokenInfo) o; + return creationTime == that.creationTime + && Objects.equals(userName, that.userName) + && Objects.equals(serializedToken, that.serializedToken) + && Objects.equals(tokenClassName, that.tokenClassName); + } + + @Override + public int hashCode() { + return Objects.hash(userName, serializedToken, tokenClassName, creationTime); + } + + @Override + public String toString() { + return new PlanStringBuilder(this) + .field("userName", userName) + .field("tokenClassName", tokenClassName) + .field("creationTime", creationTime) + .field("tokenLength", serializedToken != null ? serializedToken.length() : 0) + .toString(); + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloConstants.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloConstants.java new file mode 100644 index 00000000000..bfde8b6a2b5 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloConstants.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; + +/** + * Constants used by the Accumulo storage plugin. + */ +public interface DrillAccumuloConstants { + + /** + * Name of the row key column in Drill queries. + */ + String ROW_KEY = "row_key"; + + /** + * Schema path for the row key. + */ + SchemaPath ROW_KEY_PATH = SchemaPath.getSimplePath(ROW_KEY); + + /** + * Type for the row key column (required VARBINARY). + */ + MajorType ROW_KEY_TYPE = Types.required(MinorType.VARBINARY); + + /** + * Type for column family maps (required MAP). + */ + MajorType COLUMN_FAMILY_TYPE = Types.required(MinorType.MAP); + + /** + * Type for individual columns within a family (optional VARBINARY). + */ + MajorType COLUMN_TYPE = Types.optional(MinorType.VARBINARY); + + /** + * Separator between column family and qualifier in Accumulo keys. + */ + String COLUMN_SEPARATOR = ":"; + + /** + * Default batch size for scanner caching. + */ + int DEFAULT_BATCH_SIZE = 4000; +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloTable.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloTable.java new file mode 100644 index 00000000000..f7d5699588c --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloTable.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.util.ArrayList; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.store.accumulo.schema.ColumnDef; +import org.apache.drill.exec.store.accumulo.schema.TableSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Represents an Accumulo table in Drill's query planner. + * + *

This class provides the row type (schema) for Accumulo tables to Drill's + * Calcite-based query planner. It uses the configured schema provider to + * discover column definitions.

+ * + *

Schema resolution follows this order:

+ *
    + *
  1. If explicit schema is defined in the metadata table, use that
    2. + *
  2. Otherwise, use a dynamic schema with row_key and a columns map
    3. + *
+ */ +public class DrillAccumuloTable extends DrillTable { + private static final Logger logger = LoggerFactory.getLogger(DrillAccumuloTable.class); + + public static final String ROW_KEY_COLUMN = "row_key"; + public static final String COLUMNS_MAP_COLUMN = "columns"; + + private final AccumuloStoragePlugin plugin; + private final AccumuloScanSpec scanSpec; + private TableSchema tableSchema; + + public DrillAccumuloTable( + AccumuloStoragePlugin plugin, + String storageEngineName, + AccumuloScanSpec scanSpec) { + super(storageEngineName, plugin, scanSpec); + this.plugin = plugin; + this.scanSpec = scanSpec; + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + TableSchema schema = getTableSchema(); + + ArrayList typeList = new ArrayList<>(); + ArrayList fieldNameList = new ArrayList<>(); + + // Always include row_key as first column + fieldNameList.add(ROW_KEY_COLUMN); + typeList.add(typeFactory.createTypeWithNullability( + typeFactory.createSqlType(schema.getRowKeyType().getSqlTypeName()), + false)); + + if (schema.hasExplicitColumns()) { + // Use explicit schema from metadata table + for (ColumnDef column : schema.getColumns()) { + fieldNameList.add(column.getName()); + RelDataType columnType = createColumnType(typeFactory, column); + typeList.add(typeFactory.createTypeWithNullability(columnType, column.isNullable())); + } + logger.debug("Using explicit schema for table '{}' with {} columns", + scanSpec.getTableName(), schema.getColumnCount()); + } else { + // Use dynamic schema with columns map + fieldNameList.add(COLUMNS_MAP_COLUMN); + typeList.add(typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY))); + logger.debug("Using dynamic schema for table '{}'", scanSpec.getTableName()); + } + + return typeFactory.createStructType(typeList, fieldNameList); + } + + /** + * Creates a RelDataType for the given column definition. + */ + private RelDataType createColumnType(RelDataTypeFactory typeFactory, ColumnDef column) { + SqlTypeName sqlType = column.getSqlTypeName(); + + switch (sqlType) { + case VARCHAR: + case CHAR: + // Use default precision for string types + return typeFactory.createSqlType(sqlType, 65535); + case DECIMAL: + // Use default precision and scale for decimal + return typeFactory.createSqlType(sqlType, 38, 10); + default: + return typeFactory.createSqlType(sqlType); + } + } + + /** + * Returns the table schema, loading it from the schema provider if needed. + */ + public TableSchema getTableSchema() { + if (tableSchema == null) { + try { + tableSchema = plugin.getSchemaProvider() + .getTableSchema(plugin.getClient(), scanSpec.getTableName()); + } catch (Exception e) { + logger.warn("Failed to load schema for table '{}', using dynamic schema", + scanSpec.getTableName(), e); + tableSchema = TableSchema.dynamic(scanSpec.getTableName()); + } + } + return tableSchema; + } + + public AccumuloScanSpec getScanSpec() { + return scanSpec; + } + + public AccumuloStoragePlugin getPlugin() { + return plugin; + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/AccumuloColumnType.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/AccumuloColumnType.java new file mode 100644 index 00000000000..d317c0980a8 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/AccumuloColumnType.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo.schema; + +import org.apache.calcite.sql.type.SqlTypeName; + +/** + * Supported column types for Accumulo tables in Drill. + * + *

Accumulo is schema-less and stores all data as byte arrays. + * This enum defines the logical types that Drill will use to interpret + * the byte data when reading from Accumulo.

+ */ +public enum AccumuloColumnType { + + /** + * Variable-length string (default type). + */ + VARCHAR(SqlTypeName.VARCHAR), + + /** + * Fixed-length string. + */ + CHAR(SqlTypeName.CHAR), + + /** + * 32-bit signed integer. + */ + INT(SqlTypeName.INTEGER), + + /** + * Alias for INT. + */ + INTEGER(SqlTypeName.INTEGER), + + /** + * 64-bit signed integer. + */ + BIGINT(SqlTypeName.BIGINT), + + /** + * Alias for BIGINT. + */ + LONG(SqlTypeName.BIGINT), + + /** + * 16-bit signed integer. + */ + SMALLINT(SqlTypeName.SMALLINT), + + /** + * 8-bit signed integer. + */ + TINYINT(SqlTypeName.TINYINT), + + /** + * Single-precision floating point. + */ + FLOAT(SqlTypeName.FLOAT), + + /** + * Double-precision floating point. + */ + DOUBLE(SqlTypeName.DOUBLE), + + /** + * Exact numeric with configurable precision and scale. + */ + DECIMAL(SqlTypeName.DECIMAL), + + /** + * Boolean value. + */ + BOOLEAN(SqlTypeName.BOOLEAN), + + /** + * Date without time component. + */ + DATE(SqlTypeName.DATE), + + /** + * Time without date component. + */ + TIME(SqlTypeName.TIME), + + /** + * Date and time. + */ + TIMESTAMP(SqlTypeName.TIMESTAMP), + + /** + * Binary data (raw bytes). + */ + VARBINARY(SqlTypeName.VARBINARY), + + /** + * Dynamic type (determined at runtime). + */ + ANY(SqlTypeName.ANY); + + private final SqlTypeName sqlTypeName; + + AccumuloColumnType(SqlTypeName sqlTypeName) { + this.sqlTypeName = sqlTypeName; + } + + /** + * Returns the corresponding Calcite SQL type name. + */ + public SqlTypeName getSqlTypeName() { + return sqlTypeName; + } + + /** + * Parses a type string to an AccumuloColumnType. + * + *

Case-insensitive matching. Returns VARCHAR if the type is not recognized.

+ * + * @param typeString the type string to parse + * @return the corresponding AccumuloColumnType + */ + public static AccumuloColumnType fromString(String typeString) { + if (typeString == null || typeString.trim().isEmpty()) { + return VARCHAR; + } + + String normalized = typeString.trim().toUpperCase(); + + // Handle common aliases + switch (normalized) { + case "STRING": + case "TEXT": + return VARCHAR; + case "INT": + case "INTEGER": + return INTEGER; + case "LONG": + case "BIGINT": + return BIGINT; + case "FLOAT": + case "REAL": + return FLOAT; + case "DOUBLE": + case "DOUBLE PRECISION": + return DOUBLE; + case "BOOL": + case "BOOLEAN": + return BOOLEAN; + case "BYTES": + case "BINARY": + case "VARBINARY": + return VARBINARY; + default: + try { + return valueOf(normalized); + } catch (IllegalArgumentException e) { + return VARCHAR; + } + } + } + + /** + * Returns true if this type is a numeric type. + */ + public boolean isNumeric() { + switch (this) { + case INT: + case INTEGER: + case BIGINT: + case LONG: + case SMALLINT: + case TINYINT: + case FLOAT: + case DOUBLE: + case DECIMAL: + return true; + default: + return false; + } + } + + /** + * Returns true if this type is a temporal type. + */ + public boolean isTemporal() { + switch (this) { + case DATE: + case TIME: + case TIMESTAMP: + return true; + default: + return false; + } + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/AccumuloSchemaProvider.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/AccumuloSchemaProvider.java new file mode 100644 index 00000000000..d775723eaf5 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/AccumuloSchemaProvider.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo.schema; + +import java.util.Set; + +import org.apache.accumulo.core.client.AccumuloClient; + +/** + * Interface for schema discovery strategies in the Accumulo storage plugin. + * + *

This interface abstracts how table schemas are discovered from Accumulo. + * Different implementations can provide schema information from different sources:

+ *
    + *
  • {@code MetadataTableSchemaProvider} - Reads schema from a dedicated Accumulo metadata table
    • + *
  • {@code ScanSamplingSchemaProvider} - Infers schema by sampling table data (future)
    • + *
  • {@code ConfigFileSchemaProvider} - Reads schema from external configuration files (future)
    • + *
+ * + *

This is the extension point for Option B (advanced mode) where custom schema + * providers could expose Accumulo-specific features like iterators.

+ */ +public interface AccumuloSchemaProvider { + + /** + * Returns the schema for the specified Accumulo table. + * + *

If the schema is not found or cannot be determined, implementations should + * return a dynamic schema (via {@link TableSchema#dynamic(String)}) rather than + * throwing an exception.

+ * + * @param client the Accumulo client + * @param tableName the name of the table + * @return the table schema, never null + */ + TableSchema getTableSchema(AccumuloClient client, String tableName); + + /** + * Discovers all table names available in the Accumulo instance. + * + *

Implementations should filter out system tables (e.g., tables starting with "accumulo.") + * unless specifically configured to include them.

+ * + * @param client the Accumulo client + * @return set of table names, never null (may be empty) + */ + Set discoverTableNames(AccumuloClient client); + + /** + * Returns true if this provider has schema information for the specified table. + * + *

This can be used to check if explicit schema metadata exists before + * falling back to dynamic schema discovery.

+ * + * @param client the Accumulo client + * @param tableName the name of the table + * @return true if schema information is available + */ + boolean hasSchema(AccumuloClient client, String tableName); + + /** + * Clears any cached schema information. + * + *

Called when schema metadata may have changed and needs to be refreshed.

+ */ + void clearCache(); + + /** + * Clears cached schema information for a specific table. + * + * @param tableName the table to clear from cache + */ + void clearCache(String tableName); +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/ColumnDef.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/ColumnDef.java new file mode 100644 index 00000000000..ab6b1c238d6 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/ColumnDef.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo.schema; + +import java.util.Objects; + +import org.apache.calcite.sql.type.SqlTypeName; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Represents a column definition for an Accumulo table in Drill. + * + *

Maps an Accumulo column family:qualifier pair to a Drill column with + * a specific SQL type.

+ */ +public class ColumnDef { + + private final String name; + private final String columnFamily; + private final String columnQualifier; + private final AccumuloColumnType type; + private final boolean nullable; + + @JsonCreator + public ColumnDef( + @JsonProperty("name") String name, + @JsonProperty("columnFamily") String columnFamily, + @JsonProperty("columnQualifier") String columnQualifier, + @JsonProperty("type") AccumuloColumnType type, + @JsonProperty("nullable") Boolean nullable) { + this.name = name; + this.columnFamily = columnFamily; + this.columnQualifier = columnQualifier; + this.type = type != null ? type : AccumuloColumnType.VARCHAR; + this.nullable = nullable != null ? nullable : true; + } + + /** + * Convenience constructor for creating a column definition. + */ + public static ColumnDef create(String name, String columnFamily, String columnQualifier, + AccumuloColumnType type) { + return new ColumnDef(name, columnFamily, columnQualifier, type, true); + } + + /** + * Convenience constructor for VARCHAR columns. + */ + public static ColumnDef varchar(String name, String columnFamily, String columnQualifier) { + return new ColumnDef(name, columnFamily, columnQualifier, AccumuloColumnType.VARCHAR, true); + } + + @JsonProperty("name") + public String getName() { + return name; + } + + @JsonProperty("columnFamily") + public String getColumnFamily() { + return columnFamily; + } + + @JsonProperty("columnQualifier") + public String getColumnQualifier() { + return columnQualifier; + } + + @JsonProperty("type") + public AccumuloColumnType getType() { + return type; + } + + @JsonProperty("nullable") + public boolean isNullable() { + return nullable; + } + + /** + * Returns the SQL type name for this column. + */ + @JsonIgnore + public SqlTypeName getSqlTypeName() { + return type.getSqlTypeName(); + } + + /** + * Returns the full Accumulo column identifier (family:qualifier). + */ + @JsonIgnore + public String getFullColumnName() { + if (columnQualifier == null || columnQualifier.isEmpty()) { + return columnFamily; + } + return columnFamily + ":" + columnQualifier; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ColumnDef columnDef = (ColumnDef) o; + return nullable == columnDef.nullable + && Objects.equals(name, columnDef.name) + && Objects.equals(columnFamily, columnDef.columnFamily) + && Objects.equals(columnQualifier, columnDef.columnQualifier) + && type == columnDef.type; + } + + @Override + public int hashCode() { + return Objects.hash(name, columnFamily, columnQualifier, type, nullable); + } + + @Override + public String toString() { + return "ColumnDef{" + + "name='" + name + '\'' + + ", columnFamily='" + columnFamily + '\'' + + ", columnQualifier='" + columnQualifier + '\'' + + ", type=" + type + + ", nullable=" + nullable + + '}'; + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/MetadataTableSchemaProvider.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/MetadataTableSchemaProvider.java new file mode 100644 index 00000000000..b6b555f7d19 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/MetadataTableSchemaProvider.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo.schema; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Schema provider that reads table schema from an Accumulo metadata table. + * + *

The metadata table stores schema information in the following format:

+ * 
+ * Row Key: {table_name}
+ * Column Family: "schema"
+ * Column Qualifiers:
+ *   - "row_key_type": The type of the row key (e.g., "VARCHAR", "VARBINARY")
+ *   - "columns": JSON array of column definitions
+ *
+ * Example:
+ *   Row: "users"
+ *   schema:row_key_type = "VARCHAR"
+ *   schema:columns = [
+ *     {"name":"name","columnFamily":"cf1","columnQualifier":"name","type":"VARCHAR","nullable":true},
+ *     {"name":"age","columnFamily":"cf1","columnQualifier":"age","type":"INT","nullable":true}
+ *   ]
+ *
+ * + *

This provider includes a configurable cache to reduce Accumulo metadata table lookups.

+ */ +public class MetadataTableSchemaProvider implements AccumuloSchemaProvider { + private static final Logger logger = LoggerFactory.getLogger(MetadataTableSchemaProvider.class); + + private static final String SCHEMA_COLUMN_FAMILY = "schema"; + private static final String ROW_KEY_TYPE_QUALIFIER = "row_key_type"; + private static final String COLUMNS_QUALIFIER = "columns"; + + private static final long DEFAULT_CACHE_TTL_MS = TimeUnit.MINUTES.toMillis(5); + + private final String metadataTableName; + private final ObjectMapper objectMapper; + private final Map schemaCache; + private final long cacheTtlMs; + + /** + * Creates a new MetadataTableSchemaProvider. + * + * @param metadataTableName the name of the Accumulo table storing schema metadata + */ + public MetadataTableSchemaProvider(String metadataTableName) { + this(metadataTableName, DEFAULT_CACHE_TTL_MS); + } + + /** + * Creates a new MetadataTableSchemaProvider with a custom cache TTL. + * + * @param metadataTableName the name of the Accumulo table storing schema metadata + * @param cacheTtlMs cache time-to-live in milliseconds + */ + public MetadataTableSchemaProvider(String metadataTableName, long cacheTtlMs) { + this.metadataTableName = metadataTableName; + this.objectMapper = new ObjectMapper(); + this.schemaCache = new ConcurrentHashMap<>(); + this.cacheTtlMs = cacheTtlMs; + } + + @Override + public TableSchema getTableSchema(AccumuloClient client, String tableName) { + // Check cache first + CachedSchema cached = schemaCache.get(tableName); + if (cached != null && !cached.isExpired()) { + logger.debug("Returning cached schema for table: {}", tableName); + return cached.schema; + } + + // Try to load from metadata table + TableSchema schema = loadSchemaFromMetadata(client, tableName); + + // Cache the result (even if dynamic) + schemaCache.put(tableName, new CachedSchema(schema)); + + return schema; + } + + @Override + public Set discoverTableNames(AccumuloClient client) { + Set tableNames = new HashSet<>(); + try { + for (String name : client.tableOperations().list()) { + // Filter out system tables and the metadata table itself + if (!name.startsWith("accumulo.") && !name.equals(metadataTableName)) { + tableNames.add(name); + } + } + } catch (Exception e) { + logger.warn("Failed to discover table names", e); + } + return tableNames; + } + + @Override + public boolean hasSchema(AccumuloClient client, String tableName) { + // Check cache first + CachedSchema cached = schemaCache.get(tableName); + if (cached != null && !cached.isExpired()) { + return cached.schema.hasExplicitColumns(); + } + + // Check metadata table + if (!metadataTableExists(client)) { + return false; + } + + try (Scanner scanner = client.createScanner(metadataTableName, Authorizations.EMPTY)) { + scanner.setRange(Range.exact(tableName)); + scanner.fetchColumn(new Text(SCHEMA_COLUMN_FAMILY), new Text(COLUMNS_QUALIFIER)); + return scanner.iterator().hasNext(); + } catch (TableNotFoundException e) { + return false; + } + } + + @Override + public void clearCache() { + schemaCache.clear(); + logger.debug("Cleared all cached schemas"); + } + + @Override + public void clearCache(String tableName) { + schemaCache.remove(tableName); + logger.debug("Cleared cached schema for table: {}", tableName); + } + + /** + * Loads schema from the metadata table. + */ + private TableSchema loadSchemaFromMetadata(AccumuloClient client, String tableName) { + if (!metadataTableExists(client)) { + logger.debug("Metadata table '{}' does not exist, using dynamic schema for: {}", + metadataTableName, tableName); + return TableSchema.dynamic(tableName); + } + + try (Scanner scanner = client.createScanner(metadataTableName, Authorizations.EMPTY)) { + scanner.setRange(Range.exact(tableName)); + scanner.fetchColumnFamily(new Text(SCHEMA_COLUMN_FAMILY)); + + AccumuloColumnType rowKeyType = AccumuloColumnType.VARBINARY; + List columns = null; + + for (Map.Entry entry : scanner) { + String qualifier = entry.getKey().getColumnQualifier().toString(); + String value = entry.getValue().toString(); + + if (ROW_KEY_TYPE_QUALIFIER.equals(qualifier)) { + rowKeyType = AccumuloColumnType.fromString(value); + } else if (COLUMNS_QUALIFIER.equals(qualifier)) { + columns = parseColumnsJson(value); + } + } + + if (columns == null || columns.isEmpty()) { + logger.debug("No explicit schema found for table: {}, using dynamic schema", tableName); + return TableSchema.dynamic(tableName); + } + + logger.debug("Loaded schema for table '{}' with {} columns", tableName, columns.size()); + return new TableSchema(tableName, rowKeyType, columns); + + } catch (TableNotFoundException e) { + logger.debug("Metadata table not found, using dynamic schema for: {}", tableName); + return TableSchema.dynamic(tableName); + } catch (Exception e) { + logger.warn("Error loading schema for table '{}', using dynamic schema", tableName, e); + return TableSchema.dynamic(tableName); + } + } + + /** + * Parses the JSON column definitions. + */ + private List parseColumnsJson(String json) { + try { + return objectMapper.readValue(json, new TypeReference>() {}); + } catch (Exception e) { + logger.warn("Failed to parse column definitions JSON: {}", e.getMessage()); + return new ArrayList<>(); + } + } + + /** + * Checks if the metadata table exists. + */ + private boolean metadataTableExists(AccumuloClient client) { + return client.tableOperations().exists(metadataTableName); + } + + /** + * Returns the name of the metadata table. + */ + public String getMetadataTableName() { + return metadataTableName; + } + + /** + * Cached schema entry with expiration tracking. + */ + private class CachedSchema { + final TableSchema schema; + final long timestamp; + + CachedSchema(TableSchema schema) { + this.schema = schema; + this.timestamp = System.currentTimeMillis(); + } + + boolean isExpired() { + return System.currentTimeMillis() - timestamp > cacheTtlMs; + } + } + + /** + * Writes schema metadata for a table to the metadata table. + * + *

This is a utility method for setting up schema metadata. + * It creates the metadata table if it doesn't exist.

+ * + * @param client the Accumulo client + * @param schema the table schema to write + * @throws Exception if the write fails + */ + public void writeSchema(AccumuloClient client, TableSchema schema) throws Exception { + // Create metadata table if it doesn't exist + if (!client.tableOperations().exists(metadataTableName)) { + client.tableOperations().create(metadataTableName); + logger.info("Created metadata table: {}", metadataTableName); + } + + // Write schema to metadata table + try (var writer = client.createBatchWriter(metadataTableName)) { + org.apache.accumulo.core.data.Mutation mutation = + new org.apache.accumulo.core.data.Mutation(schema.getTableName()); + + // Write row key type + mutation.put(SCHEMA_COLUMN_FAMILY, ROW_KEY_TYPE_QUALIFIER, + schema.getRowKeyType().name()); + + // Write columns as JSON + String columnsJson = objectMapper.writeValueAsString(schema.getColumns()); + mutation.put(SCHEMA_COLUMN_FAMILY, COLUMNS_QUALIFIER, columnsJson); + + writer.addMutation(mutation); + } + + // Clear cache for this table + clearCache(schema.getTableName()); + logger.info("Wrote schema for table: {}", schema.getTableName()); + } +} diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/TableSchema.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/TableSchema.java new file mode 100644 index 00000000000..a2503b4f867 --- /dev/null +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/schema/TableSchema.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo.schema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Represents the schema of an Accumulo table for Drill. + * + *

Contains the table name, row key type, and column definitions. + * The schema is used by Drill's query planner to understand the structure + * of Accumulo tables.

+ */ +public class TableSchema { + + public static final String ROW_KEY_COLUMN = "row_key"; + + private final String tableName; + private final AccumuloColumnType rowKeyType; + private final List columns; + private final Map columnsByName; + private final Map columnsByAccumuloKey; + + @JsonCreator + public TableSchema( + @JsonProperty("tableName") String tableName, + @JsonProperty("rowKeyType") AccumuloColumnType rowKeyType, + @JsonProperty("columns") List columns) { + this.tableName = tableName; + this.rowKeyType = rowKeyType != null ? rowKeyType : AccumuloColumnType.VARBINARY; + this.columns = columns != null ? new ArrayList<>(columns) : new ArrayList<>(); + + // Build lookup maps + this.columnsByName = new LinkedHashMap<>(); + this.columnsByAccumuloKey = new LinkedHashMap<>(); + for (ColumnDef col : this.columns) { + columnsByName.put(col.getName().toLowerCase(), col); + columnsByAccumuloKey.put(col.getFullColumnName(), col); + } + } + + /** + * Creates a builder for constructing a TableSchema. + */ + public static Builder builder(String tableName) { + return new Builder(tableName); + } + + /** + * Creates a dynamic schema with just the row key and a wildcard columns map. + * Used when no explicit schema is defined. + */ + public static TableSchema dynamic(String tableName) { + return new TableSchema(tableName, AccumuloColumnType.VARBINARY, Collections.emptyList()); + } + + @JsonProperty("tableName") + public String getTableName() { + return tableName; + } + + @JsonProperty("rowKeyType") + public AccumuloColumnType getRowKeyType() { + return rowKeyType; + } + + @JsonProperty("columns") + public List getColumns() { + return Collections.unmodifiableList(columns); + } + + /** + * Returns the column definition by Drill column name. + */ + @JsonIgnore + public ColumnDef getColumnByName(String name) { + return columnsByName.get(name.toLowerCase()); + } + + /** + * Returns the column definition by Accumulo column key (family:qualifier). + */ + @JsonIgnore + public ColumnDef getColumnByAccumuloKey(String accumuloKey) { + return columnsByAccumuloKey.get(accumuloKey); + } + + /** + * Returns true if this schema has explicit column definitions. + * If false, the schema is dynamic and will discover columns at runtime. + */ + @JsonIgnore + public boolean hasExplicitColumns() { + return !columns.isEmpty(); + } + + /** + * Returns the number of columns (excluding row_key). + */ + @JsonIgnore + public int getColumnCount() { + return columns.size(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TableSchema that = (TableSchema) o; + return Objects.equals(tableName, that.tableName) + && rowKeyType == that.rowKeyType + && Objects.equals(columns, that.columns); + } + + @Override + public int hashCode() { + return Objects.hash(tableName, rowKeyType, columns); + } + + @Override + public String toString() { + return "TableSchema{" + + "tableName='" + tableName + '\'' + + ", rowKeyType=" + rowKeyType + + ", columnCount=" + columns.size() + + '}'; + } + + /** + * Builder for constructing TableSchema instances. + */ + public static class Builder { + private final String tableName; + private AccumuloColumnType rowKeyType = AccumuloColumnType.VARBINARY; + private final List columns = new ArrayList<>(); + + private Builder(String tableName) { + this.tableName = tableName; + } + + public Builder rowKeyType(AccumuloColumnType type) { + this.rowKeyType = type; + return this; + } + + public Builder addColumn(ColumnDef column) { + this.columns.add(column); + return this; + } + + public Builder addColumn(String name, String family, String qualifier, AccumuloColumnType type) { + this.columns.add(ColumnDef.create(name, family, qualifier, type)); + return this; + } + + public Builder addVarcharColumn(String name, String family, String qualifier) { + this.columns.add(ColumnDef.varchar(name, family, qualifier)); + return this; + } + + public TableSchema build() { + return new TableSchema(tableName, rowKeyType, columns); + } + } +} diff --git a/contrib/storage-accumulo/src/main/resources/bootstrap-storage-plugins.json b/contrib/storage-accumulo/src/main/resources/bootstrap-storage-plugins.json new file mode 100644 index 00000000000..ec1556289d0 --- /dev/null +++ b/contrib/storage-accumulo/src/main/resources/bootstrap-storage-plugins.json @@ -0,0 +1,12 @@ +{ + "storage":{ + "accumulo" : { + "type": "accumulo", + "zookeeperQuorum": "localhost:2181", + "instanceName": "accumulo", + "username": "root", + "password": "secret", + "enabled": false + } + } +} diff --git a/contrib/storage-accumulo/src/main/resources/drill-module.conf b/contrib/storage-accumulo/src/main/resources/drill-module.conf new file mode 100644 index 00000000000..2b005a2e33c --- /dev/null +++ b/contrib/storage-accumulo/src/main/resources/drill-module.conf @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file tells Drill to consider this module when class path scanning. +# This file can also include any supplementary configuration information. +# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. + +drill: { + classpath.scanning: { + packages += "org.apache.drill.exec.store.accumulo" + } + + exec: { + accumulo.scan: { + # Number of rows to sample for schema inference + samplerows.count: 100, + # Default batch size for scanner + batch.size: 4000 + } + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloBasicQueryTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloBasicQueryTest.java new file mode 100644 index 00000000000..c3a4b107824 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloBasicQueryTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import org.junit.Test; + +/** + * Basic query integration tests for Accumulo storage plugin. + * + *

These tests verify basic SELECT queries work correctly against + * real Accumulo tables via MiniAccumuloCluster.

+ */ +public class AccumuloBasicQueryTest extends BaseAccumuloTest { + + @Test + public void testSelectStarFromTable1() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + runAccumuloSQLVerifyCount(sql, 10); + } + + @Test + public void testSelectSpecificColumnsFromTable1() throws Exception { + String sql = "SELECT row_key, cf.name, cf.age FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + runAccumuloSQLVerifyCount(sql, 10); + } + + @Test + public void testSelectRowKeyOnly() throws Exception { + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + runAccumuloSQLVerifyCount(sql, 10); + } + + @Test + public void testSelectFromUsersTable() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + runAccumuloSQLVerifyCount(sql, 20); + } + + @Test + public void testSelectMultipleColumnFamilies() throws Exception { + String sql = "SELECT row_key, personal.first_name, personal.last_name, employment.company " + + "FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + runAccumuloSQLVerifyCount(sql, 20); + } + + @Test + public void testSelectFromLargeTable() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE); + runAccumuloSQLVerifyCount(sql, 1000); + } + + @Test + public void testCountStar() throws Exception { + String sql = "SELECT COUNT(*) FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + runAccumuloSQLVerifyCount(sql, 1); + } + + @Test + public void testCountStarUsersTable() throws Exception { + String sql = "SELECT COUNT(*) FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + runAccumuloSQLVerifyCount(sql, 1); + } + + @Test + public void testDistinctCompany() throws Exception { + String sql = "SELECT DISTINCT employment.company FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + // Should have 3 distinct companies: Acme Corp, TechCo, DataInc + runAccumuloSQLVerifyCount(sql, 3); + } + + @Test + public void testGroupByCompany() throws Exception { + String sql = "SELECT employment.company, COUNT(*) as cnt " + + "FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + + " GROUP BY employment.company"; + runAccumuloSQLVerifyCount(sql, 3); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilderTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilderTest.java new file mode 100644 index 00000000000..ec421bf1a4f --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilderTest.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; + +import org.apache.drill.common.FunctionNames; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.ValueExpressions; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +import com.google.common.collect.ImmutableList; + +/** + * Unit tests for AccumuloCompareFunctionsProcessor and AccumuloFilterBuilder. + */ +public class AccumuloFilterBuilderTest extends BaseTest { + + @Test + public void testIsCompareFunction() { + assertTrue(AccumuloCompareFunctionsProcessor.isCompareFunction(FunctionNames.EQ)); + assertTrue(AccumuloCompareFunctionsProcessor.isCompareFunction(FunctionNames.NE)); + assertTrue(AccumuloCompareFunctionsProcessor.isCompareFunction(FunctionNames.LT)); + assertTrue(AccumuloCompareFunctionsProcessor.isCompareFunction(FunctionNames.LE)); + assertTrue(AccumuloCompareFunctionsProcessor.isCompareFunction(FunctionNames.GT)); + assertTrue(AccumuloCompareFunctionsProcessor.isCompareFunction(FunctionNames.GE)); + assertTrue(AccumuloCompareFunctionsProcessor.isCompareFunction(FunctionNames.IS_NULL)); + assertTrue(AccumuloCompareFunctionsProcessor.isCompareFunction(FunctionNames.IS_NOT_NULL)); + + assertFalse(AccumuloCompareFunctionsProcessor.isCompareFunction("unknown")); + assertFalse(AccumuloCompareFunctionsProcessor.isCompareFunction(FunctionNames.AND)); + assertFalse(AccumuloCompareFunctionsProcessor.isCompareFunction(FunctionNames.OR)); + } + + @Test + public void testProcessEqualFunction() { + // row_key = 'test' + SchemaPath path = SchemaPath.getSimplePath("row_key"); + ValueExpressions.QuotedString value = new ValueExpressions.QuotedString("test", 0, null); + + FunctionCall call = new FunctionCall( + FunctionNames.EQ, + ImmutableList.of(path, value), + null); + + AccumuloCompareFunctionsProcessor processor = + AccumuloCompareFunctionsProcessor.createFunctionsProcessorInstance(call); + + assertTrue(processor.isSuccess()); + assertEquals("row_key", processor.getPath().getRootSegmentPath()); + assertEquals("test", new String(processor.getValue(), StandardCharsets.UTF_8)); + assertEquals(FunctionNames.EQ, processor.getFunctionName()); + } + + @Test + public void testProcessGreaterThanFunction() { + // row_key > 'start' + SchemaPath path = SchemaPath.getSimplePath("row_key"); + ValueExpressions.QuotedString value = new ValueExpressions.QuotedString("start", 0, null); + + FunctionCall call = new FunctionCall( + FunctionNames.GT, + ImmutableList.of(path, value), + null); + + AccumuloCompareFunctionsProcessor processor = + AccumuloCompareFunctionsProcessor.createFunctionsProcessorInstance(call); + + assertTrue(processor.isSuccess()); + assertEquals("row_key", processor.getPath().getRootSegmentPath()); + assertEquals("start", new String(processor.getValue(), StandardCharsets.UTF_8)); + assertEquals(FunctionNames.GT, processor.getFunctionName()); + } + + @Test + public void testProcessLessThanFunction() { + // row_key < 'end' + SchemaPath path = SchemaPath.getSimplePath("row_key"); + ValueExpressions.QuotedString value = new ValueExpressions.QuotedString("end", 0, null); + + FunctionCall call = new FunctionCall( + FunctionNames.LT, + ImmutableList.of(path, value), + null); + + AccumuloCompareFunctionsProcessor processor = + AccumuloCompareFunctionsProcessor.createFunctionsProcessorInstance(call); + + assertTrue(processor.isSuccess()); + assertEquals(FunctionNames.LT, processor.getFunctionName()); + } + + @Test + public void testProcessSwappedOperands() { + // 'test' = row_key (value on left) + SchemaPath path = SchemaPath.getSimplePath("row_key"); + ValueExpressions.QuotedString value = new ValueExpressions.QuotedString("test", 0, null); + + FunctionCall call = new FunctionCall( + FunctionNames.EQ, + ImmutableList.of(value, path), + null); + + AccumuloCompareFunctionsProcessor processor = + AccumuloCompareFunctionsProcessor.createFunctionsProcessorInstance(call); + + assertTrue(processor.isSuccess()); + assertEquals("row_key", processor.getPath().getRootSegmentPath()); + assertEquals("test", new String(processor.getValue(), StandardCharsets.UTF_8)); + // Function should remain EQ since it's symmetric + assertEquals(FunctionNames.EQ, processor.getFunctionName()); + } + + @Test + public void testProcessSwappedGreaterThan() { + // 'value' > row_key should become row_key < 'value' + SchemaPath path = SchemaPath.getSimplePath("row_key"); + ValueExpressions.QuotedString value = new ValueExpressions.QuotedString("value", 0, null); + + FunctionCall call = new FunctionCall( + FunctionNames.GT, + ImmutableList.of(value, path), + null); + + AccumuloCompareFunctionsProcessor processor = + AccumuloCompareFunctionsProcessor.createFunctionsProcessorInstance(call); + + assertTrue(processor.isSuccess()); + assertEquals("row_key", processor.getPath().getRootSegmentPath()); + // GT transposes to LT when operands are swapped + assertEquals(FunctionNames.LT, processor.getFunctionName()); + } + + @Test + public void testProcessIntegerValue() { + // row_key = 123 + SchemaPath path = SchemaPath.getSimplePath("row_key"); + ValueExpressions.IntExpression value = new ValueExpressions.IntExpression(123, null); + + FunctionCall call = new FunctionCall( + FunctionNames.EQ, + ImmutableList.of(path, value), + null); + + AccumuloCompareFunctionsProcessor processor = + AccumuloCompareFunctionsProcessor.createFunctionsProcessorInstance(call); + + assertTrue(processor.isSuccess()); + assertEquals("123", new String(processor.getValue(), StandardCharsets.UTF_8)); + } + + @Test + public void testProcessLongValue() { + // row_key = 9999999999 + SchemaPath path = SchemaPath.getSimplePath("row_key"); + ValueExpressions.LongExpression value = new ValueExpressions.LongExpression(9999999999L, null); + + FunctionCall call = new FunctionCall( + FunctionNames.EQ, + ImmutableList.of(path, value), + null); + + AccumuloCompareFunctionsProcessor processor = + AccumuloCompareFunctionsProcessor.createFunctionsProcessorInstance(call); + + assertTrue(processor.isSuccess()); + assertEquals("9999999999", new String(processor.getValue(), StandardCharsets.UTF_8)); + } + + @Test + public void testCompareTransposeMap() { + // Verify the transpose map entries + assertEquals(FunctionNames.LE, + AccumuloCompareFunctionsProcessor.COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(FunctionNames.GE)); + assertEquals(FunctionNames.LT, + AccumuloCompareFunctionsProcessor.COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(FunctionNames.GT)); + assertEquals(FunctionNames.GE, + AccumuloCompareFunctionsProcessor.COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(FunctionNames.LE)); + assertEquals(FunctionNames.GT, + AccumuloCompareFunctionsProcessor.COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(FunctionNames.LT)); + assertEquals(FunctionNames.EQ, + AccumuloCompareFunctionsProcessor.COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(FunctionNames.EQ)); + assertEquals(FunctionNames.NE, + AccumuloCompareFunctionsProcessor.COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(FunctionNames.NE)); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloIntegrationTestsSuite.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloIntegrationTestsSuite.java new file mode 100644 index 00000000000..5196f867613 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloIntegrationTestsSuite.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.minicluster.MiniAccumuloCluster; +import org.apache.accumulo.minicluster.MiniAccumuloConfig; +import org.apache.drill.test.BaseTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test suite for Accumulo storage plugin. + * + *

This suite manages the lifecycle of a MiniAccumuloCluster and runs + * all integration tests that require a real Accumulo instance.

+ * + *

Run with: {@code mvn test -Dtest=AccumuloIntegrationTestsSuite}

+ */ +@RunWith(Suite.class) +@Suite.SuiteClasses({ + AccumuloBasicQueryTest.class, + AccumuloPushdownIntegrationTest.class +}) +public class AccumuloIntegrationTestsSuite extends BaseTest { + private static final Logger logger = LoggerFactory.getLogger(AccumuloIntegrationTestsSuite.class); + + public static final String ROOT_USER = "root"; + public static final String ROOT_PASSWORD = "drilltest"; + public static final String INSTANCE_NAME = "drill-accumulo-test"; + + private static MiniAccumuloCluster miniCluster; + private static AccumuloClient client; + private static File tempDir; + private static volatile AtomicInteger initCount = new AtomicInteger(0); + private static boolean clusterStarted = false; + private static boolean tablesCreated = false; + + /** + * Whether to manage the MiniAccumuloCluster (start/stop). + * Set to false to use an external Accumulo instance. + */ + private static boolean manageMiniCluster = Boolean.parseBoolean( + System.getProperty("drill.accumulo.tests.managed", "true")); + + /** + * Whether to create test tables. + */ + private static boolean createTables = Boolean.parseBoolean( + System.getProperty("drill.accumulo.tests.createTables", "true")); + + @BeforeClass + public static void initCluster() throws Exception { + if (initCount.get() == 0) { + synchronized (AccumuloIntegrationTestsSuite.class) { + if (initCount.get() == 0) { + if (manageMiniCluster) { + startMiniCluster(); + } else { + connectToExternalCluster(); + } + + if (createTables) { + createTestTables(); + } + + initCount.incrementAndGet(); + return; + } + } + } + initCount.incrementAndGet(); + } + + @AfterClass + public static void tearDownCluster() throws Exception { + synchronized (AccumuloIntegrationTestsSuite.class) { + if (initCount.decrementAndGet() == 0) { + if (createTables && tablesCreated) { + cleanupTestTables(); + } + + if (client != null) { + client.close(); + client = null; + } + + if (clusterStarted && miniCluster != null) { + logger.info("Stopping MiniAccumuloCluster..."); + miniCluster.stop(); + miniCluster = null; + logger.info("MiniAccumuloCluster stopped."); + } + + // Clean up temp directory + if (tempDir != null && tempDir.exists()) { + deleteDirectory(tempDir); + } + } + } + } + + private static void startMiniCluster() throws Exception { + logger.info("Starting MiniAccumuloCluster..."); + + // Create temp directory for cluster data + tempDir = new File(System.getProperty("accumulo.test.root", + System.getProperty("java.io.tmpdir")), "mini-accumulo-" + System.currentTimeMillis()); + if (!tempDir.mkdirs()) { + throw new IOException("Failed to create temp directory: " + tempDir); + } + + MiniAccumuloConfig config = new MiniAccumuloConfig(tempDir, ROOT_PASSWORD); + config.setInstanceName(INSTANCE_NAME); + config.setNumTservers(1); + + miniCluster = new MiniAccumuloCluster(config); + miniCluster.start(); + clusterStarted = true; + + // Create client + client = miniCluster.createAccumuloClient(ROOT_USER, new PasswordToken(ROOT_PASSWORD)); + + logger.info("MiniAccumuloCluster started. Instance: {}, ZooKeepers: {}", + miniCluster.getInstanceName(), miniCluster.getZooKeepers()); + } + + private static void connectToExternalCluster() throws Exception { + String zookeepers = System.getProperty("drill.accumulo.zookeepers", "localhost:2181"); + String instanceName = System.getProperty("drill.accumulo.instance", "accumulo"); + String user = System.getProperty("drill.accumulo.user", "root"); + String password = System.getProperty("drill.accumulo.password", "secret"); + + logger.info("Connecting to external Accumulo instance: {} at {}", instanceName, zookeepers); + + client = org.apache.accumulo.core.client.Accumulo.newClient() + .to(instanceName, zookeepers) + .as(user, password) + .build(); + } + + private static void createTestTables() throws Exception { + logger.info("Creating test tables..."); + AccumuloTestUtils.createAllTestTables(client); + tablesCreated = true; + logger.info("Test tables created."); + } + + private static void cleanupTestTables() { + try { + logger.info("Cleaning up test tables..."); + AccumuloTestUtils.deleteAllTestTables(client); + logger.info("Test tables cleaned up."); + } catch (Exception e) { + logger.warn("Error cleaning up test tables", e); + } + } + + private static void deleteDirectory(File dir) { + File[] files = dir.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + deleteDirectory(file); + } else { + file.delete(); + } + } + } + dir.delete(); + } + + // Public accessors for test classes + + public static MiniAccumuloCluster getMiniCluster() { + return miniCluster; + } + + public static AccumuloClient getClient() { + return client; + } + + public static String getZooKeepers() { + if (miniCluster != null) { + return miniCluster.getZooKeepers(); + } + return System.getProperty("drill.accumulo.zookeepers", "localhost:2181"); + } + + public static String getInstanceName() { + if (miniCluster != null) { + return miniCluster.getInstanceName(); + } + return System.getProperty("drill.accumulo.instance", "accumulo"); + } + + public static String getRootUser() { + return ROOT_USER; + } + + public static String getRootPassword() { + return ROOT_PASSWORD; + } + + public static void configure(boolean manageMiniCluster, boolean createTables) { + AccumuloIntegrationTestsSuite.manageMiniCluster = manageMiniCluster; + AccumuloIntegrationTestsSuite.createTables = createTables; + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloKerberosConfigTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloKerberosConfigTest.java new file mode 100644 index 00000000000..f0178f9bd1d --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloKerberosConfigTest.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.apache.drill.common.logical.StoragePluginConfig.AuthMode; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Unit tests for Kerberos-specific configuration in AccumuloStoragePluginConfig. + */ +public class AccumuloKerberosConfigTest extends BaseTest { + + @Test + public void testAuthTypeEnum() { + assertEquals(AccumuloAuthType.PASSWORD, AccumuloAuthType.parseOrDefault(null, AccumuloAuthType.PASSWORD)); + assertEquals(AccumuloAuthType.PASSWORD, AccumuloAuthType.parseOrDefault("", AccumuloAuthType.PASSWORD)); + assertEquals(AccumuloAuthType.PASSWORD, AccumuloAuthType.parseOrDefault("PASSWORD", AccumuloAuthType.KERBEROS)); + assertEquals(AccumuloAuthType.KERBEROS, AccumuloAuthType.parseOrDefault("KERBEROS", AccumuloAuthType.PASSWORD)); + assertEquals(AccumuloAuthType.KERBEROS, AccumuloAuthType.parseOrDefault("kerberos", AccumuloAuthType.PASSWORD)); + assertEquals(AccumuloAuthType.KERBEROS, AccumuloAuthType.parseOrDefault("Kerberos", AccumuloAuthType.PASSWORD)); + } + + @Test(expected = IllegalArgumentException.class) + public void testAuthTypeEnumInvalidValue() { + AccumuloAuthType.parseOrDefault("INVALID", AccumuloAuthType.PASSWORD); + } + + @Test + public void testKerberosSharedUserConfig() { + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "zk:2181", + "accumulo", + null, + null, + "KERBEROS", + "drill/host@REALM", + "/etc/security/keytabs/drill.keytab", + "auth", + "accumulo", + false, + "SHARED_USER", + null, + null, + null, + null + ); + + assertTrue(config.isKerberosEnabled()); + assertFalse(config.isUserImpersonationEnabled()); + assertFalse(config.isUseDelegationTokens()); + assertEquals(AuthMode.SHARED_USER, config.getAuthMode()); + } + + @Test + public void testKerberosUserImpersonationConfig() { + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "zk:2181", + "accumulo", + null, + null, + "KERBEROS", + "drill/host@REALM", + "/etc/security/keytabs/drill.keytab", + "auth-conf", + "accumulo", + true, + "USER_IMPERSONATION", + null, + null, + null, + null + ); + + assertTrue(config.isKerberosEnabled()); + assertTrue(config.isUserImpersonationEnabled()); + assertTrue(config.isUseDelegationTokens()); + assertEquals(AuthMode.USER_IMPERSONATION, config.getAuthMode()); + assertEquals("auth-conf", config.getSaslQop()); + } + + @Test + public void testSaslQopValues() { + // Test auth + AccumuloStoragePluginConfig authConfig = new AccumuloStoragePluginConfig( + "zk:2181", "accumulo", null, null, + "KERBEROS", "drill@REALM", "/keytab", "auth", null, null, null, null, null, null, null + ); + assertEquals("auth", authConfig.getSaslQop()); + + // Test auth-int + AccumuloStoragePluginConfig authIntConfig = new AccumuloStoragePluginConfig( + "zk:2181", "accumulo", null, null, + "KERBEROS", "drill@REALM", "/keytab", "auth-int", null, null, null, null, null, null, null + ); + assertEquals("auth-int", authIntConfig.getSaslQop()); + + // Test auth-conf + AccumuloStoragePluginConfig authConfConfig = new AccumuloStoragePluginConfig( + "zk:2181", "accumulo", null, null, + "KERBEROS", "drill@REALM", "/keytab", "auth-conf", null, null, null, null, null, null, null + ); + assertEquals("auth-conf", authConfConfig.getSaslQop()); + } + + @Test + public void testBackwardCompatibilityPasswordAuth() { + // Old-style configuration without any Kerberos fields should still work + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "localhost:2181", + "accumulo", + "root", + "secret" + ); + + assertEquals(AccumuloAuthType.PASSWORD, config.getAuthenticationType()); + assertFalse(config.isKerberosEnabled()); + assertFalse(config.isUserImpersonationEnabled()); + assertEquals(AuthMode.SHARED_USER, config.getAuthMode()); + assertEquals("root", config.getUsername()); + assertEquals("secret", config.getPassword()); + } + + @Test + public void testJsonSerializationFullKerberosConfig() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "zk1:2181,zk2:2181", + "accumulo_prod", + null, + null, + "KERBEROS", + "drill/drillserver.example.com@EXAMPLE.COM", + "/etc/security/keytabs/drill.service.keytab", + "auth-conf", + "accumulo", + true, + "USER_IMPERSONATION", + null, + "_drill_schema", + 30000, + 10 + ); + + String json = mapper.writeValueAsString(config); + assertNotNull(json); + + // Verify JSON contains expected fields + assertTrue(json.contains("zk1:2181,zk2:2181")); + assertTrue(json.contains("accumulo_prod")); + assertTrue(json.contains("KERBEROS")); + assertTrue(json.contains("drill/drillserver.example.com@EXAMPLE.COM")); + assertTrue(json.contains("auth-conf")); + assertTrue(json.contains("\"useDelegationTokens\":true")); + + // Deserialize and verify + AccumuloStoragePluginConfig deserialized = mapper.readValue(json, AccumuloStoragePluginConfig.class); + assertEquals(config.getZookeeperQuorum(), deserialized.getZookeeperQuorum()); + assertEquals(config.getInstanceName(), deserialized.getInstanceName()); + assertEquals(config.getAuthenticationType(), deserialized.getAuthenticationType()); + assertEquals(config.getPrincipal(), deserialized.getPrincipal()); + assertEquals(config.getKeytabPath(), deserialized.getKeytabPath()); + assertEquals(config.getSaslQop(), deserialized.getSaslQop()); + assertEquals(config.getAccumuloServicePrimary(), deserialized.getAccumuloServicePrimary()); + assertEquals(config.isUseDelegationTokens(), deserialized.isUseDelegationTokens()); + } + + @Test + public void testMixedAuthConfigInvalid() { + // Config with both password creds and Kerberos settings + // This should be allowed for migration scenarios + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "zk:2181", + "accumulo", + "fallback_user", // password username + "fallback_pass", // password + "KERBEROS", // but auth type is Kerberos + "drill@REALM", + "/keytab", + null, + null, + null, + null, + null, + null, + null, + null + ); + + // When KERBEROS auth type is set, it should use Kerberos + assertEquals(AccumuloAuthType.KERBEROS, config.getAuthenticationType()); + assertTrue(config.isKerberosEnabled()); + // But password fields are still accessible if needed + assertEquals("fallback_user", config.getUsername()); + } + + @Test + public void testUserTranslationConfig() { + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "zk:2181", + "accumulo", + "service_user", // service account for fallback + "service_pass", + "PASSWORD", + null, + null, + null, + null, + false, + "USER_TRANSLATION", + null, + null, + null, + null + ); + + assertFalse(config.isKerberosEnabled()); + assertFalse(config.isUserImpersonationEnabled()); + assertTrue(config.isUserTranslationEnabled()); + assertEquals(AuthMode.USER_TRANSLATION, config.getAuthMode()); + } + + @Test + public void testPrincipalFormats() { + // Test simple principal (just user@REALM) + AccumuloStoragePluginConfig simpleConfig = new AccumuloStoragePluginConfig( + "zk:2181", "accumulo", null, null, + "KERBEROS", "drill@EXAMPLE.COM", "/keytab", null, null, null, null, null, null, null, null + ); + assertEquals("drill@EXAMPLE.COM", simpleConfig.getPrincipal()); + + // Test service principal (primary/instance@REALM) + AccumuloStoragePluginConfig serviceConfig = new AccumuloStoragePluginConfig( + "zk:2181", "accumulo", null, null, + "KERBEROS", "drill/drillserver.example.com@EXAMPLE.COM", "/keytab", + null, null, null, null, null, null, null, null + ); + assertEquals("drill/drillserver.example.com@EXAMPLE.COM", serviceConfig.getPrincipal()); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloLimitPushdownTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloLimitPushdownTest.java new file mode 100644 index 00000000000..faa8815b6db --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloLimitPushdownTest.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.List; + +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.test.BaseTest; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * Unit tests for limit pushdown in AccumuloGroupScan. + */ +public class AccumuloLimitPushdownTest extends BaseTest { + + /** + * Creates a mock AccumuloGroupScan for testing. + */ + private AccumuloGroupScan createTestGroupScan() { + AccumuloStoragePlugin mockPlugin = Mockito.mock(AccumuloStoragePlugin.class); + AccumuloStoragePluginConfig mockConfig = Mockito.mock(AccumuloStoragePluginConfig.class); + Mockito.when(mockPlugin.getConfig()).thenReturn(mockConfig); + + AccumuloScanSpec scanSpec = new AccumuloScanSpec("test_table"); + return new AccumuloGroupScan("testUser", mockPlugin, scanSpec, null, -1); + } + + @Test + public void testSupportsLimitPushdown() { + AccumuloGroupScan scan = createTestGroupScan(); + assertTrue("AccumuloGroupScan should support limit pushdown", scan.supportsLimitPushdown()); + } + + @Test + public void testApplyLimitReturnsNewScan() { + AccumuloGroupScan original = createTestGroupScan(); + + GroupScan newScan = original.applyLimit(100); + + assertNotNull("applyLimit should return a new scan", newScan); + assertNotSame("applyLimit should return a different instance", original, newScan); + assertTrue(newScan instanceof AccumuloGroupScan); + } + + @Test + public void testApplyLimitSetsMaxRecords() { + AccumuloGroupScan original = createTestGroupScan(); + + AccumuloGroupScan newScan = (AccumuloGroupScan) original.applyLimit(100); + + assertEquals(100, newScan.getMaxRecords()); + assertTrue("Limit should be marked as pushed down", newScan.isLimitPushedDown()); + } + + @Test + public void testApplyLimitDoesNotModifyOriginal() { + AccumuloGroupScan original = createTestGroupScan(); + int originalMaxRecords = original.getMaxRecords(); + + original.applyLimit(100); + + assertEquals("Original maxRecords should be unchanged", originalMaxRecords, original.getMaxRecords()); + assertFalse("Original should not have limit pushed down", original.isLimitPushedDown()); + } + + @Test + public void testApplyLimitWithMoreRestrictiveExisting() { + AccumuloStoragePlugin mockPlugin = Mockito.mock(AccumuloStoragePlugin.class); + AccumuloStoragePluginConfig mockConfig = Mockito.mock(AccumuloStoragePluginConfig.class); + Mockito.when(mockPlugin.getConfig()).thenReturn(mockConfig); + + AccumuloScanSpec scanSpec = new AccumuloScanSpec("test_table"); + // Create scan with limit already set to 50 + AccumuloGroupScan original = new AccumuloGroupScan("testUser", mockPlugin, scanSpec, null, 50); + + // Try to apply a higher limit + GroupScan newScan = original.applyLimit(100); + + // Should return null because existing limit is more restrictive + assertNull("Should return null when existing limit is more restrictive", newScan); + } + + @Test + public void testApplyLimitWithLessRestrictiveExisting() { + AccumuloStoragePlugin mockPlugin = Mockito.mock(AccumuloStoragePlugin.class); + AccumuloStoragePluginConfig mockConfig = Mockito.mock(AccumuloStoragePluginConfig.class); + Mockito.when(mockPlugin.getConfig()).thenReturn(mockConfig); + + AccumuloScanSpec scanSpec = new AccumuloScanSpec("test_table"); + // Create scan with limit already set to 100 + AccumuloGroupScan original = new AccumuloGroupScan("testUser", mockPlugin, scanSpec, null, 100); + + // Try to apply a lower limit + AccumuloGroupScan newScan = (AccumuloGroupScan) original.applyLimit(50); + + // Should return new scan with lower limit + assertNotNull("Should return new scan with more restrictive limit", newScan); + assertEquals(50, newScan.getMaxRecords()); + } + + @Test + public void testApplyLimitPreservesOtherPushdowns() { + AccumuloGroupScan original = createTestGroupScan(); + original.setFilterPushedDown(true); + original.setProjectionPushedDown(true); + original.setSortPushedDown(true); + + AccumuloGroupScan newScan = (AccumuloGroupScan) original.applyLimit(100); + + assertTrue("Filter pushdown should be preserved", newScan.isFilterPushedDown()); + assertTrue("Projection pushdown should be preserved", newScan.isProjectionPushedDown()); + assertTrue("Sort pushdown should be preserved", newScan.isSortPushedDown()); + assertTrue("Limit should be marked as pushed down", newScan.isLimitPushedDown()); + } + + @Test + public void testApplyLimitPreservesScanSpec() { + AccumuloGroupScan original = createTestGroupScan(); + + AccumuloGroupScan newScan = (AccumuloGroupScan) original.applyLimit(100); + + assertNotNull("ScanSpec should be preserved", newScan.getScanSpec()); + assertEquals("test_table", newScan.getTableName()); + } + + @Test + public void testApplyLimitPreservesColumns() { + AccumuloGroupScan original = createTestGroupScan(); + List columns = Arrays.asList( + SchemaPath.getSimplePath("row_key"), + SchemaPath.getSimplePath("col1") + ); + AccumuloGroupScan withColumns = (AccumuloGroupScan) original.clone(columns); + + AccumuloGroupScan newScan = (AccumuloGroupScan) withColumns.applyLimit(100); + + assertEquals(columns, newScan.getColumns()); + } + + @Test + public void testScanStatsWithLimitPushdown() { + AccumuloGroupScan scan = createTestGroupScan(); + ScanStats baseStats = scan.getScanStats(); + + AccumuloGroupScan limitedScan = (AccumuloGroupScan) scan.applyLimit(50); + ScanStats limitedStats = limitedScan.getScanStats(); + + // With a limit of 50, row count should be capped at 50 + assertTrue("Row count should be reduced with limit pushdown", + limitedStats.getRecordCount() <= 50); + assertTrue("CPU cost should be lower with limit pushdown", + limitedStats.getCpuCost() < baseStats.getCpuCost()); + } + + @Test + public void testScanStatsWithFilterAndLimitPushdown() { + AccumuloGroupScan scan = createTestGroupScan(); + scan.setFilterPushedDown(true); + + AccumuloGroupScan limitedScan = (AccumuloGroupScan) scan.applyLimit(100); + limitedScan.setFilterPushedDown(true); + ScanStats combinedStats = limitedScan.getScanStats(); + + // Combined pushdowns should result in efficient stats + assertTrue("Row count should be bounded by limit", + combinedStats.getRecordCount() <= 100); + } + + @Test + public void testToStringIncludesLimitInfo() { + AccumuloGroupScan scan = createTestGroupScan(); + AccumuloGroupScan limitedScan = (AccumuloGroupScan) scan.applyLimit(100); + + String toString = limitedScan.toString(); + assertTrue("toString should include maxRecords", toString.contains("maxRecords=100")); + assertTrue("toString should include limitPushedDown=true", toString.contains("limitPushedDown=true")); + } + + @Test + public void testSubScanContainsMaxRecords() { + AccumuloStoragePlugin mockPlugin = Mockito.mock(AccumuloStoragePlugin.class); + AccumuloStoragePluginConfig mockConfig = Mockito.mock(AccumuloStoragePluginConfig.class); + Mockito.when(mockPlugin.getConfig()).thenReturn(mockConfig); + + AccumuloScanSpec scanSpec = new AccumuloScanSpec("test_table"); + AccumuloGroupScan groupScan = new AccumuloGroupScan("testUser", mockPlugin, scanSpec, null, 100); + groupScan.setLimitPushedDown(true); + + AccumuloSubScan subScan = groupScan.getSpecificScan(0); + + assertEquals("SubScan should have maxRecords from GroupScan", 100, subScan.getMaxRecords()); + } + + @Test + public void testSubScanWithNoLimit() { + AccumuloGroupScan scan = createTestGroupScan(); + + AccumuloSubScan subScan = scan.getSpecificScan(0); + + assertEquals("SubScan should have -1 for no limit", -1, subScan.getMaxRecords()); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloProjectionPushdownTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloProjectionPushdownTest.java new file mode 100644 index 00000000000..3d8c4586e07 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloProjectionPushdownTest.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.List; + +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.test.BaseTest; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * Unit tests for projection pushdown in AccumuloGroupScan. + */ +public class AccumuloProjectionPushdownTest extends BaseTest { + + /** + * Creates a mock AccumuloGroupScan for testing. + */ + private AccumuloGroupScan createTestGroupScan() { + AccumuloStoragePlugin mockPlugin = Mockito.mock(AccumuloStoragePlugin.class); + AccumuloStoragePluginConfig mockConfig = Mockito.mock(AccumuloStoragePluginConfig.class); + Mockito.when(mockPlugin.getConfig()).thenReturn(mockConfig); + + AccumuloScanSpec scanSpec = new AccumuloScanSpec("test_table"); + return new AccumuloGroupScan("testUser", mockPlugin, scanSpec, null, -1); + } + + @Test + public void testCloneWithAllColumns() { + AccumuloGroupScan original = createTestGroupScan(); + + // Clone with ALL_COLUMNS should not mark projection as pushed down + GroupScan cloned = original.clone(GroupScan.ALL_COLUMNS); + + assertTrue(cloned instanceof AccumuloGroupScan); + AccumuloGroupScan accumuloCloned = (AccumuloGroupScan) cloned; + + assertNotSame(original, cloned); + assertFalse("Projection should not be marked as pushed down for ALL_COLUMNS", + accumuloCloned.isProjectionPushedDown()); + } + + @Test + public void testCloneWithSpecificColumns() { + AccumuloGroupScan original = createTestGroupScan(); + + List projectedColumns = Arrays.asList( + SchemaPath.getSimplePath("row_key"), + SchemaPath.getSimplePath("name"), + SchemaPath.getSimplePath("age") + ); + + GroupScan cloned = original.clone(projectedColumns); + + assertTrue(cloned instanceof AccumuloGroupScan); + AccumuloGroupScan accumuloCloned = (AccumuloGroupScan) cloned; + + assertNotSame(original, cloned); + assertTrue("Projection should be marked as pushed down for specific columns", + accumuloCloned.isProjectionPushedDown()); + assertEquals(projectedColumns, accumuloCloned.getColumns()); + } + + @Test + public void testCloneWithSingleColumn() { + AccumuloGroupScan original = createTestGroupScan(); + + List projectedColumns = Arrays.asList( + SchemaPath.getSimplePath("row_key") + ); + + GroupScan cloned = original.clone(projectedColumns); + + assertTrue(cloned instanceof AccumuloGroupScan); + AccumuloGroupScan accumuloCloned = (AccumuloGroupScan) cloned; + + assertTrue("Projection should be marked as pushed down for single column", + accumuloCloned.isProjectionPushedDown()); + assertEquals(1, accumuloCloned.getColumns().size()); + } + + @Test + public void testClonePreservesOtherPushdownFlags() { + AccumuloGroupScan original = createTestGroupScan(); + original.setFilterPushedDown(true); + original.setSortPushedDown(true); + original.setLimitPushedDown(true); + + List projectedColumns = Arrays.asList( + SchemaPath.getSimplePath("col1"), + SchemaPath.getSimplePath("col2") + ); + + GroupScan cloned = original.clone(projectedColumns); + AccumuloGroupScan accumuloCloned = (AccumuloGroupScan) cloned; + + // All flags should be preserved + assertTrue("Filter pushdown flag should be preserved", accumuloCloned.isFilterPushedDown()); + assertTrue("Sort pushdown flag should be preserved", accumuloCloned.isSortPushedDown()); + assertTrue("Limit pushdown flag should be preserved", accumuloCloned.isLimitPushedDown()); + assertTrue("Projection should be marked as pushed down", accumuloCloned.isProjectionPushedDown()); + } + + @Test + public void testClonePreservesScanSpec() { + AccumuloGroupScan original = createTestGroupScan(); + + List projectedColumns = Arrays.asList( + SchemaPath.getSimplePath("col1") + ); + + GroupScan cloned = original.clone(projectedColumns); + AccumuloGroupScan accumuloCloned = (AccumuloGroupScan) cloned; + + assertNotNull("ScanSpec should be preserved", accumuloCloned.getScanSpec()); + assertEquals("test_table", accumuloCloned.getTableName()); + } + + @Test + public void testScanStatsWithProjectionPushdown() { + AccumuloGroupScan scan = createTestGroupScan(); + ScanStats statsWithoutProjection = scan.getScanStats(); + + // Clone with specific columns (triggers projection pushdown) + List projectedColumns = Arrays.asList( + SchemaPath.getSimplePath("col1"), + SchemaPath.getSimplePath("col2") + ); + AccumuloGroupScan projectedScan = (AccumuloGroupScan) scan.clone(projectedColumns); + ScanStats statsWithProjection = projectedScan.getScanStats(); + + // CPU cost should be reduced with projection pushdown + assertTrue("CPU cost should be lower with projection pushdown", + statsWithProjection.getCpuCost() < statsWithoutProjection.getCpuCost()); + } + + @Test + public void testScanStatsWithFilterAndProjectionPushdown() { + AccumuloGroupScan scan = createTestGroupScan(); + ScanStats baseStats = scan.getScanStats(); + + // Apply both filter and projection pushdowns + scan.setFilterPushedDown(true); + List projectedColumns = Arrays.asList( + SchemaPath.getSimplePath("col1") + ); + AccumuloGroupScan projectedScan = (AccumuloGroupScan) scan.clone(projectedColumns); + projectedScan.setFilterPushedDown(true); + ScanStats combinedStats = projectedScan.getScanStats(); + + // Combined pushdowns should result in even lower cost + assertTrue("Combined pushdowns should reduce CPU cost significantly", + combinedStats.getCpuCost() < baseStats.getCpuCost()); + assertTrue("Combined pushdowns should reduce row count estimate", + combinedStats.getRecordCount() < baseStats.getRecordCount()); + } + + @Test + public void testOriginalNotModifiedByClone() { + AccumuloGroupScan original = createTestGroupScan(); + assertFalse("Original should not have projection pushed down initially", + original.isProjectionPushedDown()); + + List projectedColumns = Arrays.asList( + SchemaPath.getSimplePath("col1") + ); + + // Clone with projection + original.clone(projectedColumns); + + // Original should remain unchanged + assertFalse("Original should still not have projection pushed down", + original.isProjectionPushedDown()); + } + + @Test + public void testCloneWithNullColumns() { + AccumuloGroupScan original = createTestGroupScan(); + + GroupScan cloned = original.clone(null); + + assertTrue(cloned instanceof AccumuloGroupScan); + AccumuloGroupScan accumuloCloned = (AccumuloGroupScan) cloned; + + // Null columns should be treated as ALL_COLUMNS + assertFalse("Projection should not be marked as pushed down for null columns", + accumuloCloned.isProjectionPushedDown()); + } + + @Test + public void testToStringIncludesProjectionFlag() { + AccumuloGroupScan scan = createTestGroupScan(); + + List projectedColumns = Arrays.asList( + SchemaPath.getSimplePath("col1") + ); + AccumuloGroupScan projectedScan = (AccumuloGroupScan) scan.clone(projectedColumns); + + String toString = projectedScan.toString(); + assertTrue("toString should include projectionPushedDown=true", + toString.contains("projectionPushedDown=true")); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloPushdownIntegrationTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloPushdownIntegrationTest.java new file mode 100644 index 00000000000..73684680e44 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloPushdownIntegrationTest.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import org.junit.Test; + +/** + * Integration tests for Accumulo pushdown capabilities. + * + *

These tests verify that filter, projection, limit, and sort pushdowns + * work correctly with real Accumulo tables.

+ */ +public class AccumuloPushdownIntegrationTest extends BaseAccumuloTest { + + // ========================================================================= + // Filter Pushdown Tests + // ========================================================================= + + @Test + public void testFilterOnRowKeyEquals() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key = 'row_001'"; + runAccumuloSQLVerifyCount(sql, 1); + } + + @Test + public void testFilterOnRowKeyGreaterThan() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key > 'row_005'"; + runAccumuloSQLVerifyCount(sql, 5); // row_006 to row_010 + } + + @Test + public void testFilterOnRowKeyLessThan() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key < 'row_004'"; + runAccumuloSQLVerifyCount(sql, 3); // row_001 to row_003 + } + + @Test + public void testFilterOnRowKeyRange() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key >= 'row_003' AND row_key <= 'row_007'"; + runAccumuloSQLVerifyCount(sql, 5); // row_003 to row_007 + } + + @Test + public void testFilterOnRowKeyRangeLarge() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + + " WHERE row_key >= 'row_0100' AND row_key < 'row_0200'"; + runAccumuloSQLVerifyCount(sql, 100); // row_0100 to row_0199 + } + + @Test + public void testFilterOnColumnValue() throws Exception { + // Note: column value filters may not be pushed down, but should still work + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + + " WHERE employment.company = 'Acme Corp'"; + runAccumuloSQLVerifyCount(sql, 7); // 7 users at Acme Corp + } + + // ========================================================================= + // Projection Pushdown Tests + // ========================================================================= + + @Test + public void testProjectionSingleColumn() throws Exception { + String sql = "SELECT cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + runAccumuloSQLVerifyCount(sql, 10); + } + + @Test + public void testProjectionMultipleColumns() throws Exception { + String sql = "SELECT cf.name, cf.city FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + runAccumuloSQLVerifyCount(sql, 10); + } + + @Test + public void testProjectionWithRowKey() throws Exception { + String sql = "SELECT row_key, personal.first_name FROM " + + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + runAccumuloSQLVerifyCount(sql, 20); + } + + @Test + public void testProjectionSingleColumnFamily() throws Exception { + String sql = "SELECT personal FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + runAccumuloSQLVerifyCount(sql, 20); + } + + // ========================================================================= + // Limit Pushdown Tests + // ========================================================================= + + @Test + public void testLimitSmall() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " LIMIT 5"; + runAccumuloSQLVerifyCount(sql, 5); + } + + @Test + public void testLimitOnLargeTable() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + " LIMIT 50"; + runAccumuloSQLVerifyCount(sql, 50); + } + + @Test + public void testLimitOne() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " LIMIT 1"; + runAccumuloSQLVerifyCount(sql, 1); + } + + @Test + public void testLimitLargerThanTable() throws Exception { + // Limit larger than table size should return all rows + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " LIMIT 100"; + runAccumuloSQLVerifyCount(sql, 10); + } + + // ========================================================================= + // Sort Pushdown Tests + // ========================================================================= + + @Test + public void testOrderByRowKeyAsc() throws Exception { + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " ORDER BY row_key ASC"; + runAccumuloSQLVerifyCount(sql, 10); + } + + @Test + public void testOrderByRowKeyDesc() throws Exception { + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " ORDER BY row_key DESC"; + runAccumuloSQLVerifyCount(sql, 10); + } + + @Test + public void testOrderByRowKeyWithLimit() throws Exception { + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + + " ORDER BY row_key ASC LIMIT 10"; + runAccumuloSQLVerifyCount(sql, 10); + } + + // ========================================================================= + // Combined Pushdown Tests + // ========================================================================= + + @Test + public void testFilterAndProjection() throws Exception { + String sql = "SELECT row_key, cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key > 'row_005'"; + runAccumuloSQLVerifyCount(sql, 5); + } + + @Test + public void testFilterAndLimit() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key > 'row_002' LIMIT 3"; + runAccumuloSQLVerifyCount(sql, 3); + } + + @Test + public void testProjectionAndLimit() throws Exception { + String sql = "SELECT row_key, cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " LIMIT 5"; + runAccumuloSQLVerifyCount(sql, 5); + } + + @Test + public void testFilterProjectionAndLimit() throws Exception { + String sql = "SELECT row_key, cf.name, cf.city FROM " + + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key >= 'row_003' LIMIT 4"; + runAccumuloSQLVerifyCount(sql, 4); + } + + @Test + public void testFilterAndSort() throws Exception { + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key < 'row_006' ORDER BY row_key ASC"; + runAccumuloSQLVerifyCount(sql, 5); + } + + @Test + public void testSortAndLimit() throws Exception { + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " ORDER BY row_key ASC LIMIT 3"; + runAccumuloSQLVerifyCount(sql, 3); + } + + @Test + public void testAllPushdownsCombined() throws Exception { + String sql = "SELECT row_key, cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key >= 'row_002' AND row_key <= 'row_009'" + + " ORDER BY row_key ASC LIMIT 5"; + runAccumuloSQLVerifyCount(sql, 5); + } + + // ========================================================================= + // Edge Case Tests + // ========================================================================= + + @Test + public void testFilterNoResults() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key = 'nonexistent'"; + runAccumuloSQLVerifyCount(sql, 0); + } + + @Test + public void testFilterOutOfRange() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key > 'zzz'"; + runAccumuloSQLVerifyCount(sql, 0); + } + + @Test + public void testLimitZero() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " LIMIT 0"; + runAccumuloSQLVerifyCount(sql, 0); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloScanSpecTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloScanSpecTest.java new file mode 100644 index 00000000000..75cf613e529 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloScanSpecTest.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.List; + +import org.apache.drill.exec.store.accumulo.AccumuloScanSpec.AccumuloColumnSpec; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Unit tests for AccumuloScanSpec. + */ +public class AccumuloScanSpecTest extends BaseTest { + + @Test + public void testSimpleConstruction() { + AccumuloScanSpec spec = new AccumuloScanSpec("test_table"); + + assertEquals("test_table", spec.getTableName()); + assertNull(spec.getStartRow()); + assertNull(spec.getStopRow()); + assertTrue(spec.isStartRowInclusive()); + assertFalse(spec.isStopRowInclusive()); + assertNull(spec.getColumns()); + assertNull(spec.getFilterExpression()); + assertNull(spec.getLimit()); + assertFalse(spec.isUseSortedScanner()); + } + + @Test + public void testFullConstruction() { + byte[] startRow = "row_001".getBytes(); + byte[] stopRow = "row_999".getBytes(); + List columns = Arrays.asList( + new AccumuloColumnSpec("cf1", "name", "name"), + new AccumuloColumnSpec("cf1", "age", "age") + ); + + AccumuloScanSpec spec = new AccumuloScanSpec( + "test_table", + startRow, + stopRow, + true, + false, + columns, + "age > 30", + 100, + true, + false + ); + + assertEquals("test_table", spec.getTableName()); + assertArrayEquals(startRow, spec.getStartRow()); + assertArrayEquals(stopRow, spec.getStopRow()); + assertTrue(spec.isStartRowInclusive()); + assertFalse(spec.isStopRowInclusive()); + assertEquals(2, spec.getColumns().size()); + assertEquals("age > 30", spec.getFilterExpression()); + assertEquals(Integer.valueOf(100), spec.getLimit()); + assertTrue(spec.isUseSortedScanner()); + assertFalse(spec.isSortDescending()); + } + + @Test + public void testHelperMethods() { + AccumuloScanSpec specNoExtras = new AccumuloScanSpec("table1"); + assertFalse(specNoExtras.hasFilter()); + assertFalse(specNoExtras.hasLimit()); + assertFalse(specNoExtras.hasRowRange()); + + AccumuloScanSpec specWithFilter = new AccumuloScanSpec( + "table2", null, null, true, false, null, "col = 'value'", null, false, false); + assertTrue(specWithFilter.hasFilter()); + assertFalse(specWithFilter.hasLimit()); + assertFalse(specWithFilter.hasRowRange()); + + AccumuloScanSpec specWithLimit = new AccumuloScanSpec( + "table3", null, null, true, false, null, null, 50, false, false); + assertFalse(specWithLimit.hasFilter()); + assertTrue(specWithLimit.hasLimit()); + assertFalse(specWithLimit.hasRowRange()); + + AccumuloScanSpec specWithRange = new AccumuloScanSpec( + "table4", "start".getBytes(), "stop".getBytes(), true, false, null, null, null, false, false); + assertFalse(specWithRange.hasFilter()); + assertFalse(specWithRange.hasLimit()); + assertTrue(specWithRange.hasRowRange()); + } + + @Test + public void testWithMethods() { + AccumuloScanSpec original = new AccumuloScanSpec("test_table"); + + AccumuloScanSpec withFilter = original.withFilter("status = 'active'"); + assertEquals("status = 'active'", withFilter.getFilterExpression()); + assertNull(original.getFilterExpression()); // Original unchanged + + AccumuloScanSpec withLimit = original.withLimit(100); + assertEquals(Integer.valueOf(100), withLimit.getLimit()); + assertNull(original.getLimit()); // Original unchanged + + AccumuloScanSpec withSorted = original.withSortedScanner(true); + assertTrue(withSorted.isUseSortedScanner()); + assertFalse(original.isUseSortedScanner()); // Original unchanged + } + + @Test + public void testEquality() { + AccumuloScanSpec spec1 = new AccumuloScanSpec("table1"); + AccumuloScanSpec spec2 = new AccumuloScanSpec("table1"); + AccumuloScanSpec spec3 = new AccumuloScanSpec("table2"); + + assertEquals(spec1, spec2); + assertEquals(spec1.hashCode(), spec2.hashCode()); + assertNotEquals(spec1, spec3); + } + + @Test + public void testJsonSerialization() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + AccumuloScanSpec spec = new AccumuloScanSpec( + "test_table", + "start".getBytes(), + "stop".getBytes(), + true, + false, + Arrays.asList(new AccumuloColumnSpec("cf1", "col1", "col1")), + "col1 > 10", + 50, + true, + true + ); + + String json = mapper.writeValueAsString(spec); + assertNotNull(json); + assertTrue(json.contains("test_table")); + assertTrue(json.contains("col1 > 10")); + + AccumuloScanSpec deserialized = mapper.readValue(json, AccumuloScanSpec.class); + assertEquals(spec.getTableName(), deserialized.getTableName()); + assertEquals(spec.getFilterExpression(), deserialized.getFilterExpression()); + assertEquals(spec.getLimit(), deserialized.getLimit()); + assertEquals(spec.isUseSortedScanner(), deserialized.isUseSortedScanner()); + assertEquals(spec.isSortDescending(), deserialized.isSortDescending()); + } + + @Test + public void testColumnSpec() { + AccumuloColumnSpec colSpec = new AccumuloColumnSpec("family1", "qualifier1", "drill_column"); + + assertEquals("family1", colSpec.getColumnFamily()); + assertEquals("qualifier1", colSpec.getColumnQualifier()); + assertEquals("drill_column", colSpec.getDrillColumnName()); + + AccumuloColumnSpec colSpec2 = new AccumuloColumnSpec("family1", "qualifier1", "drill_column"); + assertEquals(colSpec, colSpec2); + assertEquals(colSpec.hashCode(), colSpec2.hashCode()); + } + + @Test + public void testDigest() { + AccumuloScanSpec spec = new AccumuloScanSpec("my_table"); + String digest = spec.digest(); + + assertNotNull(digest); + assertTrue(digest.contains("my_table")); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSortPushdownTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSortPushdownTest.java new file mode 100644 index 00000000000..1124d1609fd --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSortPushdownTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.test.BaseTest; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * Unit tests for sort pushdown in AccumuloGroupScan. + */ +public class AccumuloSortPushdownTest extends BaseTest { + + /** + * Creates a mock AccumuloGroupScan for testing. + */ + private AccumuloGroupScan createTestGroupScan() { + AccumuloStoragePlugin mockPlugin = Mockito.mock(AccumuloStoragePlugin.class); + AccumuloStoragePluginConfig mockConfig = Mockito.mock(AccumuloStoragePluginConfig.class); + Mockito.when(mockPlugin.getConfig()).thenReturn(mockConfig); + + AccumuloScanSpec scanSpec = new AccumuloScanSpec("test_table"); + return new AccumuloGroupScan("testUser", mockPlugin, scanSpec, null, -1); + } + + @Test + public void testScanSpecWithSortOrderAscending() { + AccumuloScanSpec original = new AccumuloScanSpec("test_table"); + assertFalse("Default should not use sorted scanner", original.isUseSortedScanner()); + assertFalse("Default should not be descending", original.isSortDescending()); + + AccumuloScanSpec withSort = original.withSortOrder(false); + + assertTrue("Should use sorted scanner after withSortOrder", withSort.isUseSortedScanner()); + assertFalse("Should be ascending", withSort.isSortDescending()); + assertEquals("test_table", withSort.getTableName()); + } + + @Test + public void testScanSpecWithSortOrderDescending() { + AccumuloScanSpec original = new AccumuloScanSpec("test_table"); + + AccumuloScanSpec withSort = original.withSortOrder(true); + + assertTrue("Should use sorted scanner after withSortOrder", withSort.isUseSortedScanner()); + assertTrue("Should be descending", withSort.isSortDescending()); + assertEquals("test_table", withSort.getTableName()); + } + + @Test + public void testGroupScanSortPushedDown() { + AccumuloGroupScan scan = createTestGroupScan(); + assertFalse("Sort should not be pushed down initially", scan.isSortPushedDown()); + + scan.setSortPushedDown(true); + assertTrue("Sort should be pushed down after setting", scan.isSortPushedDown()); + } + + @Test + public void testCloneWithNewScanSpecPreservesSortFlag() { + AccumuloGroupScan original = createTestGroupScan(); + original.setSortPushedDown(true); + + AccumuloScanSpec newSpec = original.getScanSpec().withSortOrder(false); + AccumuloGroupScan cloned = original.cloneWithNewScanSpec(newSpec); + + assertTrue("Sort pushdown flag should be preserved", cloned.isSortPushedDown()); + assertTrue("New scan spec should use sorted scanner", cloned.getScanSpec().isUseSortedScanner()); + } + + @Test + public void testScanStatsWithSortPushdown() { + AccumuloGroupScan scan = createTestGroupScan(); + ScanStats baseStats = scan.getScanStats(); + + scan.setSortPushedDown(true); + ScanStats sortedStats = scan.getScanStats(); + + // Sort pushdown has a slight cost penalty because we use Scanner instead of BatchScanner + assertTrue("Sort pushdown should slightly increase CPU cost due to Scanner vs BatchScanner", + sortedStats.getCpuCost() >= baseStats.getCpuCost()); + } + + @Test + public void testToStringIncludesSortFlag() { + AccumuloGroupScan scan = createTestGroupScan(); + scan.setSortPushedDown(true); + + String toString = scan.toString(); + assertTrue("toString should include sortPushedDown=true", + toString.contains("sortPushedDown=true")); + } + + @Test + public void testScanSpecToStringIncludesSortInfo() { + AccumuloScanSpec spec = new AccumuloScanSpec("test_table").withSortOrder(true); + + String toString = spec.toString(); + assertTrue("toString should include useSortedScanner=true", + toString.contains("useSortedScanner=true")); + assertTrue("toString should include sortDescending=true", + toString.contains("sortDescending=true")); + } + + @Test + public void testScanSpecEquality() { + AccumuloScanSpec spec1 = new AccumuloScanSpec("test_table").withSortOrder(false); + AccumuloScanSpec spec2 = new AccumuloScanSpec("test_table").withSortOrder(false); + AccumuloScanSpec spec3 = new AccumuloScanSpec("test_table").withSortOrder(true); + + assertEquals("Same sort order specs should be equal", spec1, spec2); + assertFalse("Different sort order specs should not be equal", spec1.equals(spec3)); + } + + @Test + public void testScanSpecHashCode() { + AccumuloScanSpec spec1 = new AccumuloScanSpec("test_table").withSortOrder(false); + AccumuloScanSpec spec2 = new AccumuloScanSpec("test_table").withSortOrder(false); + AccumuloScanSpec spec3 = new AccumuloScanSpec("test_table").withSortOrder(true); + + assertEquals("Same sort order specs should have equal hash codes", + spec1.hashCode(), spec2.hashCode()); + // Different specs may have different hash codes (not guaranteed, but likely) + assertFalse("Different sort order specs likely have different hash codes", + spec1.hashCode() == spec3.hashCode()); + } + + @Test + public void testScanSpecWithMultiplePushdowns() { + AccumuloScanSpec original = new AccumuloScanSpec("test_table"); + + AccumuloScanSpec withAll = original + .withFilter("row_key > 'a'") + .withLimit(100) + .withSortOrder(true); + + assertEquals("test_table", withAll.getTableName()); + assertEquals("row_key > 'a'", withAll.getFilterExpression()); + assertEquals(Integer.valueOf(100), withAll.getLimit()); + assertTrue(withAll.isUseSortedScanner()); + assertTrue(withAll.isSortDescending()); + } + + @Test + public void testGroupScanCopyPreservesAllFlags() { + AccumuloGroupScan original = createTestGroupScan(); + original.setFilterPushedDown(true); + original.setProjectionPushedDown(true); + original.setLimitPushedDown(true); + original.setSortPushedDown(true); + + AccumuloScanSpec newSpec = original.getScanSpec().withSortOrder(false); + AccumuloGroupScan cloned = original.cloneWithNewScanSpec(newSpec); + + assertTrue("Filter pushdown should be preserved", cloned.isFilterPushedDown()); + assertTrue("Projection pushdown should be preserved", cloned.isProjectionPushedDown()); + assertTrue("Limit pushdown should be preserved", cloned.isLimitPushedDown()); + assertTrue("Sort pushdown should be preserved", cloned.isSortPushedDown()); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePluginConfigTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePluginConfigTest.java new file mode 100644 index 00000000000..3e7055e1852 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePluginConfigTest.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig.AuthMode; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Unit tests for AccumuloStoragePluginConfig. + */ +public class AccumuloStoragePluginConfigTest extends BaseTest { + + @Test + public void testConfigCreation() { + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "localhost:2181", + "accumulo", + "root", + "secret", + null, null, null, null, null, null, null, null, null, null, null + ); + + assertEquals("localhost:2181", config.getZookeeperQuorum()); + assertEquals("accumulo", config.getInstanceName()); + assertEquals("root", config.getUsername()); + assertEquals("secret", config.getPassword()); + assertEquals("_drill_schema", config.getSchemaMetadataTable()); + assertEquals(Integer.valueOf(30000), config.getClientTimeout()); + assertEquals(Integer.valueOf(10), config.getBatchScannerThreads()); + } + + @Test + public void testConfigWithCustomValues() { + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "zk1:2181,zk2:2181,zk3:2181", + "myinstance", + "admin", + "password123", + null, null, null, null, null, null, null, null, + "my_schema_table", + 60000, + 20 + ); + + assertEquals("zk1:2181,zk2:2181,zk3:2181", config.getZookeeperQuorum()); + assertEquals("myinstance", config.getInstanceName()); + assertEquals("admin", config.getUsername()); + assertEquals("password123", config.getPassword()); + assertEquals("my_schema_table", config.getSchemaMetadataTable()); + assertEquals(Integer.valueOf(60000), config.getClientTimeout()); + assertEquals(Integer.valueOf(20), config.getBatchScannerThreads()); + } + + @Test + public void testSimplifiedConstructorBackwardCompatibility() { + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "localhost:2181", + "accumulo", + "root", + "secret" + ); + + assertEquals("localhost:2181", config.getZookeeperQuorum()); + assertEquals("accumulo", config.getInstanceName()); + assertEquals("root", config.getUsername()); + assertEquals("secret", config.getPassword()); + assertEquals(AccumuloAuthType.PASSWORD, config.getAuthenticationType()); + assertEquals(AuthMode.SHARED_USER, config.getAuthMode()); + assertFalse(config.isUseDelegationTokens()); + } + + @Test + public void testKerberosConfigCreation() { + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "zk1:2181,zk2:2181", + "accumulo", + null, // no username for Kerberos + null, // no password for Kerberos + "KERBEROS", + "drill/hostname@REALM", + "/etc/security/keytabs/drill.keytab", + "auth-conf", + "accumulo", + true, // useDelegationTokens + "USER_IMPERSONATION", + null, + null, + null, + null + ); + + assertEquals("zk1:2181,zk2:2181", config.getZookeeperQuorum()); + assertEquals("accumulo", config.getInstanceName()); + assertEquals(AccumuloAuthType.KERBEROS, config.getAuthenticationType()); + assertEquals("drill/hostname@REALM", config.getPrincipal()); + assertEquals("/etc/security/keytabs/drill.keytab", config.getKeytabPath()); + assertEquals("auth-conf", config.getSaslQop()); + assertEquals("accumulo", config.getAccumuloServicePrimary()); + assertTrue(config.isUseDelegationTokens()); + assertEquals(AuthMode.USER_IMPERSONATION, config.getAuthMode()); + assertTrue(config.isKerberosEnabled()); + assertTrue(config.isUserImpersonationEnabled()); + } + + @Test + public void testKerberosDefaults() { + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "localhost:2181", + "accumulo", + null, + null, + "KERBEROS", + "drill/host@REALM", + "/path/to/keytab", + null, // saslQop - should default to "auth" + null, // accumuloServicePrimary - should default to "accumulo" + null, // useDelegationTokens - should default to false + null, + null, + null, + null, + null + ); + + assertEquals(AccumuloAuthType.KERBEROS, config.getAuthenticationType()); + assertEquals("auth", config.getSaslQop()); // default + assertEquals("accumulo", config.getAccumuloServicePrimary()); // default + assertFalse(config.isUseDelegationTokens()); // default + assertEquals(AuthMode.SHARED_USER, config.getAuthMode()); // default + } + + @Test + public void testAuthTypeParsingCaseInsensitive() { + // Test lowercase + AccumuloStoragePluginConfig config1 = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", null, null, + "kerberos", "p", "k", null, null, null, null, null, null, null, null + ); + assertEquals(AccumuloAuthType.KERBEROS, config1.getAuthenticationType()); + + // Test mixed case + AccumuloStoragePluginConfig config2 = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", null, null, + "Kerberos", "p", "k", null, null, null, null, null, null, null, null + ); + assertEquals(AccumuloAuthType.KERBEROS, config2.getAuthenticationType()); + + // Test password (default) + AccumuloStoragePluginConfig config3 = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", "user", "pass", + "password", null, null, null, null, null, null, null, null, null, null + ); + assertEquals(AccumuloAuthType.PASSWORD, config3.getAuthenticationType()); + } + + @Test + public void testConfigEquality() { + AccumuloStoragePluginConfig config1 = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", "root", "secret", + null, null, null, null, null, null, null, null, null, null, null + ); + + AccumuloStoragePluginConfig config2 = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", "root", "secret", + null, null, null, null, null, null, null, null, null, null, null + ); + + AccumuloStoragePluginConfig config3 = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", "different_user", "secret", + null, null, null, null, null, null, null, null, null, null, null + ); + + assertEquals(config1, config2); + assertEquals(config1.hashCode(), config2.hashCode()); + assertNotEquals(config1, config3); + } + + @Test + public void testKerberosConfigEquality() { + AccumuloStoragePluginConfig config1 = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", null, null, + "KERBEROS", "drill@REALM", "/keytab", "auth", "accumulo", true, + "USER_IMPERSONATION", null, null, null, null + ); + + AccumuloStoragePluginConfig config2 = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", null, null, + "KERBEROS", "drill@REALM", "/keytab", "auth", "accumulo", true, + "USER_IMPERSONATION", null, null, null, null + ); + + AccumuloStoragePluginConfig config3 = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", null, null, + "KERBEROS", "different@REALM", "/keytab", "auth", "accumulo", true, + "USER_IMPERSONATION", null, null, null, null + ); + + assertEquals(config1, config2); + assertEquals(config1.hashCode(), config2.hashCode()); + assertNotEquals(config1, config3); + } + + @Test + public void testJsonSerializationPasswordAuth() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", "root", "secret", + null, null, null, null, null, null, null, null, + "my_schema", 45000, 15 + ); + + String json = mapper.writeValueAsString(config); + assertNotNull(json); + assertTrue(json.contains("localhost:2181")); + assertTrue(json.contains("accumulo")); + assertTrue(json.contains("root")); + assertTrue(json.contains("my_schema")); + assertTrue(json.contains("PASSWORD") || json.contains("\"authenticationType\":null")); + + // Deserialize back + AccumuloStoragePluginConfig deserialized = mapper.readValue(json, AccumuloStoragePluginConfig.class); + assertEquals(config.getZookeeperQuorum(), deserialized.getZookeeperQuorum()); + assertEquals(config.getInstanceName(), deserialized.getInstanceName()); + assertEquals(config.getUsername(), deserialized.getUsername()); + assertEquals(config.getPassword(), deserialized.getPassword()); + } + + @Test + public void testJsonSerializationKerberosAuth() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "zk:2181", "accumulo", null, null, + "KERBEROS", "drill/host@REALM", "/etc/keytab", "auth-conf", "accumulo", true, + "USER_IMPERSONATION", null, null, null, null + ); + + String json = mapper.writeValueAsString(config); + assertNotNull(json); + assertTrue(json.contains("KERBEROS")); + assertTrue(json.contains("drill/host@REALM")); + assertTrue(json.contains("/etc/keytab")); + assertTrue(json.contains("auth-conf")); + assertTrue(json.contains("useDelegationTokens")); + + // Deserialize back + AccumuloStoragePluginConfig deserialized = mapper.readValue(json, AccumuloStoragePluginConfig.class); + assertEquals(AccumuloAuthType.KERBEROS, deserialized.getAuthenticationType()); + assertEquals("drill/host@REALM", deserialized.getPrincipal()); + assertEquals("/etc/keytab", deserialized.getKeytabPath()); + assertEquals("auth-conf", deserialized.getSaslQop()); + assertTrue(deserialized.isUseDelegationTokens()); + } + + @Test + public void testToStringMasksPassword() { + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", "root", "supersecret", + null, null, null, null, null, null, null, null, null, null, null + ); + + String toString = config.toString(); + assertTrue(toString.contains("localhost:2181")); + assertTrue(toString.contains("accumulo")); + assertTrue(toString.contains("root")); + // Password should be masked + assertTrue(!toString.contains("supersecret") || toString.contains("*")); + } + + @Test + public void testToStringMasksKeytabPath() { + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", null, null, + "KERBEROS", "drill@REALM", "/secure/path/to/keytab", null, null, null, + null, null, null, null, null + ); + + String toString = config.toString(); + assertTrue(toString.contains("drill@REALM")); + // Keytab path should be masked for security + assertTrue(!toString.contains("/secure/path/to/keytab") || toString.contains("*")); + } + + @Test + public void testExtendsStoragePluginConfig() { + AccumuloStoragePluginConfig config = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", "root", "secret", + null, null, null, null, null, null, null, null, null, null, null + ); + + assertTrue(config instanceof StoragePluginConfig); + } + + @Test + public void testIsKerberosEnabled() { + AccumuloStoragePluginConfig passwordConfig = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", "root", "secret", + "PASSWORD", null, null, null, null, null, null, null, null, null, null + ); + assertFalse(passwordConfig.isKerberosEnabled()); + + AccumuloStoragePluginConfig kerberosConfig = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", null, null, + "KERBEROS", "drill@REALM", "/keytab", null, null, null, null, null, null, null, null + ); + assertTrue(kerberosConfig.isKerberosEnabled()); + } + + @Test + public void testIsUserImpersonationEnabled() { + AccumuloStoragePluginConfig sharedUserConfig = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", "root", "secret", + null, null, null, null, null, null, "SHARED_USER", null, null, null, null + ); + assertFalse(sharedUserConfig.isUserImpersonationEnabled()); + + AccumuloStoragePluginConfig impersonationConfig = new AccumuloStoragePluginConfig( + "localhost:2181", "accumulo", null, null, + "KERBEROS", "drill@REALM", "/keytab", null, null, true, + "USER_IMPERSONATION", null, null, null, null + ); + assertTrue(impersonationConfig.isUserImpersonationEnabled()); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestUtils.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestUtils.java new file mode 100644 index 00000000000..454bdaedae2 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestUtils.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.nio.charset.StandardCharsets; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Value; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for creating test tables and data in Accumulo. + */ +public class AccumuloTestUtils { + private static final Logger logger = LoggerFactory.getLogger(AccumuloTestUtils.class); + + public static final String TEST_TABLE_1 = "drill_test_table_1"; + public static final String TEST_TABLE_USERS = "drill_test_users"; + public static final String TEST_TABLE_LARGE = "drill_test_large"; + + /** + * Creates a simple test table with basic key-value data. + * + *

Table structure:

+ *
    + *
  • row_key: row_001 to row_010
    • + *
  • cf:name - string names
    • + *
  • cf:age - integer ages as strings
    • + *
  • cf:city - city names
    • + *
+ */ + public static void createTestTable1(AccumuloClient client) throws Exception { + String tableName = TEST_TABLE_1; + createTableIfNotExists(client, tableName); + + try (BatchWriter writer = client.createBatchWriter(tableName, new BatchWriterConfig())) { + String[][] data = { + {"row_001", "Alice", "30", "New York"}, + {"row_002", "Bob", "25", "Los Angeles"}, + {"row_003", "Charlie", "35", "Chicago"}, + {"row_004", "Diana", "28", "Houston"}, + {"row_005", "Eve", "32", "Phoenix"}, + {"row_006", "Frank", "45", "Philadelphia"}, + {"row_007", "Grace", "29", "San Antonio"}, + {"row_008", "Henry", "38", "San Diego"}, + {"row_009", "Ivy", "26", "Dallas"}, + {"row_010", "Jack", "41", "San Jose"} + }; + + for (String[] row : data) { + Mutation m = new Mutation(new Text(row[0])); + m.put(new Text("cf"), new Text("name"), new Value(row[1].getBytes(StandardCharsets.UTF_8))); + m.put(new Text("cf"), new Text("age"), new Value(row[2].getBytes(StandardCharsets.UTF_8))); + m.put(new Text("cf"), new Text("city"), new Value(row[3].getBytes(StandardCharsets.UTF_8))); + writer.addMutation(m); + } + } + + logger.info("Created test table: {} with 10 rows", tableName); + } + + /** + * Creates a test table with user data and multiple column families. + * + *

Table structure:

+ *
    + *
  • row_key: user_001 to user_020
    • + *
  • personal:first_name, personal:last_name
    • + *
  • contact:email, contact:phone
    • + *
  • employment:company, employment:title, employment:salary
    • + *
+ */ + public static void createTestTableUsers(AccumuloClient client) throws Exception { + String tableName = TEST_TABLE_USERS; + createTableIfNotExists(client, tableName); + + try (BatchWriter writer = client.createBatchWriter(tableName, new BatchWriterConfig())) { + String[][] data = { + {"user_001", "John", "Doe", "john.doe@email.com", "555-0101", "Acme Corp", "Engineer", "75000"}, + {"user_002", "Jane", "Smith", "jane.smith@email.com", "555-0102", "TechCo", "Manager", "95000"}, + {"user_003", "Bob", "Johnson", "bob.j@email.com", "555-0103", "DataInc", "Analyst", "65000"}, + {"user_004", "Alice", "Williams", "alice.w@email.com", "555-0104", "Acme Corp", "Director", "120000"}, + {"user_005", "Charlie", "Brown", "charlie.b@email.com", "555-0105", "TechCo", "Developer", "80000"}, + {"user_006", "Diana", "Davis", "diana.d@email.com", "555-0106", "DataInc", "Scientist", "90000"}, + {"user_007", "Edward", "Miller", "edward.m@email.com", "555-0107", "Acme Corp", "Engineer", "78000"}, + {"user_008", "Fiona", "Wilson", "fiona.w@email.com", "555-0108", "TechCo", "Designer", "72000"}, + {"user_009", "George", "Moore", "george.m@email.com", "555-0109", "DataInc", "Manager", "98000"}, + {"user_010", "Hannah", "Taylor", "hannah.t@email.com", "555-0110", "Acme Corp", "Analyst", "68000"}, + {"user_011", "Ivan", "Anderson", "ivan.a@email.com", "555-0111", "TechCo", "Developer", "82000"}, + {"user_012", "Julia", "Thomas", "julia.t@email.com", "555-0112", "DataInc", "Engineer", "77000"}, + {"user_013", "Kevin", "Jackson", "kevin.j@email.com", "555-0113", "Acme Corp", "Manager", "105000"}, + {"user_014", "Laura", "White", "laura.w@email.com", "555-0114", "TechCo", "Director", "125000"}, + {"user_015", "Michael", "Harris", "michael.h@email.com", "555-0115", "DataInc", "Analyst", "67000"}, + {"user_016", "Nancy", "Martin", "nancy.m@email.com", "555-0116", "Acme Corp", "Developer", "79000"}, + {"user_017", "Oscar", "Garcia", "oscar.g@email.com", "555-0117", "TechCo", "Scientist", "92000"}, + {"user_018", "Patricia", "Martinez", "patricia.m@email.com", "555-0118", "DataInc", "Designer", "71000"}, + {"user_019", "Quincy", "Robinson", "quincy.r@email.com", "555-0119", "Acme Corp", "Engineer", "76000"}, + {"user_020", "Rachel", "Clark", "rachel.c@email.com", "555-0120", "TechCo", "Manager", "99000"} + }; + + for (String[] row : data) { + Mutation m = new Mutation(new Text(row[0])); + // personal column family + m.put(new Text("personal"), new Text("first_name"), new Value(row[1].getBytes(StandardCharsets.UTF_8))); + m.put(new Text("personal"), new Text("last_name"), new Value(row[2].getBytes(StandardCharsets.UTF_8))); + // contact column family + m.put(new Text("contact"), new Text("email"), new Value(row[3].getBytes(StandardCharsets.UTF_8))); + m.put(new Text("contact"), new Text("phone"), new Value(row[4].getBytes(StandardCharsets.UTF_8))); + // employment column family + m.put(new Text("employment"), new Text("company"), new Value(row[5].getBytes(StandardCharsets.UTF_8))); + m.put(new Text("employment"), new Text("title"), new Value(row[6].getBytes(StandardCharsets.UTF_8))); + m.put(new Text("employment"), new Text("salary"), new Value(row[7].getBytes(StandardCharsets.UTF_8))); + writer.addMutation(m); + } + } + + logger.info("Created test table: {} with 20 rows", tableName); + } + + /** + * Creates a larger test table for testing limit and pagination. + * + *

Table structure:

+ *
    + *
  • row_key: row_0001 to row_1000
    • + *
  • data:value - sequential integer values
    • + *
  • data:description - description string
    • + *
+ */ + public static void createTestTableLarge(AccumuloClient client) throws Exception { + String tableName = TEST_TABLE_LARGE; + createTableIfNotExists(client, tableName); + + try (BatchWriter writer = client.createBatchWriter(tableName, new BatchWriterConfig())) { + for (int i = 1; i <= 1000; i++) { + String rowKey = String.format("row_%04d", i); + Mutation m = new Mutation(new Text(rowKey)); + m.put(new Text("data"), new Text("value"), new Value(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); + m.put(new Text("data"), new Text("description"), new Value(("Item number " + i).getBytes(StandardCharsets.UTF_8))); + writer.addMutation(m); + } + } + + logger.info("Created test table: {} with 1000 rows", tableName); + } + + /** + * Creates all test tables. + */ + public static void createAllTestTables(AccumuloClient client) throws Exception { + createTestTable1(client); + createTestTableUsers(client); + createTestTableLarge(client); + } + + /** + * Deletes all test tables. + */ + public static void deleteAllTestTables(AccumuloClient client) throws Exception { + deleteTableIfExists(client, TEST_TABLE_1); + deleteTableIfExists(client, TEST_TABLE_USERS); + deleteTableIfExists(client, TEST_TABLE_LARGE); + } + + /** + * Creates a table if it doesn't exist. + */ + public static void createTableIfNotExists(AccumuloClient client, String tableName) + throws AccumuloException, AccumuloSecurityException { + try { + if (!client.tableOperations().exists(tableName)) { + client.tableOperations().create(tableName); + logger.debug("Created table: {}", tableName); + } + } catch (TableExistsException e) { + // Table was created by another thread, ignore + logger.debug("Table {} already exists", tableName); + } + } + + /** + * Deletes a table if it exists. + */ + public static void deleteTableIfExists(AccumuloClient client, String tableName) + throws AccumuloException, AccumuloSecurityException { + try { + if (client.tableOperations().exists(tableName)) { + client.tableOperations().delete(tableName); + logger.debug("Deleted table: {}", tableName); + } + } catch (TableNotFoundException e) { + // Table was deleted by another thread, ignore + logger.debug("Table {} not found for deletion", tableName); + } + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestsSuite.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestsSuite.java new file mode 100644 index 00000000000..738f1b70b1f --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestsSuite.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import org.apache.drill.exec.store.accumulo.schema.AccumuloColumnTypeTest; +import org.apache.drill.exec.store.accumulo.schema.TableSchemaTest; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * Test suite for Accumulo storage plugin. + * + *

This suite includes all unit tests for the Accumulo plugin components. + * Integration tests requiring MiniAccumuloCluster will be added in later phases.

+ */ +@RunWith(Suite.class) +@Suite.SuiteClasses({ + // Phase 1: Core components + AccumuloStoragePluginConfigTest.class, + AccumuloScanSpecTest.class, + // Phase 2: Schema discovery + AccumuloColumnTypeTest.class, + TableSchemaTest.class, + // Phase 3: Basic scan capability + DrillAccumuloConstantsTest.class, + AccumuloTypeConverterTest.class, + // Phase 4: Filter pushdown + AccumuloFilterBuilderTest.class, + // Phase 5: Projection pushdown + AccumuloProjectionPushdownTest.class, + // Phase 6: Limit pushdown + AccumuloLimitPushdownTest.class, + // Phase 7: Sort pushdown + AccumuloSortPushdownTest.class, + // Phase 8: Kerberos authentication + AccumuloKerberosConfigTest.class, + DelegationTokenInfoTest.class +}) +public class AccumuloTestsSuite { + // Test suite - no implementation needed +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTypeConverterTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTypeConverterTest.java new file mode 100644 index 00000000000..0f1f3d34d9a --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTypeConverterTest.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +import org.apache.drill.exec.store.accumulo.schema.AccumuloColumnType; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +/** + * Unit tests for AccumuloTypeConverter. + */ +public class AccumuloTypeConverterTest extends BaseTest { + + @Test + public void testConvertVarchar() { + byte[] bytes = "hello world".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.VARCHAR); + assertEquals("hello world", result); + } + + @Test + public void testConvertVarcharEmpty() { + byte[] bytes = new byte[0]; + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.VARCHAR); + assertNull(result); + } + + @Test + public void testConvertVarcharNull() { + Object result = AccumuloTypeConverter.convert(null, AccumuloColumnType.VARCHAR); + assertNull(result); + } + + @Test + public void testConvertIntegerFromString() { + byte[] bytes = "42".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.INT); + assertEquals(42, result); + } + + @Test + public void testConvertIntegerNegative() { + byte[] bytes = "-123".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.INTEGER); + assertEquals(-123, result); + } + + @Test + public void testConvertIntegerFromBinary() { + // Use a value that produces non-ASCII bytes so it falls through to binary parsing + ByteBuffer buffer = ByteBuffer.allocate(4); + buffer.putInt(0x80000001); // Has high bit set, produces non-printable chars + Object result = AccumuloTypeConverter.convert(buffer.array(), AccumuloColumnType.INT); + assertEquals(0x80000001, result); + } + + @Test + public void testConvertIntegerInvalid() { + byte[] bytes = "not a number".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.INT); + assertNull(result); + } + + @Test + public void testConvertLongFromString() { + byte[] bytes = "9223372036854775807".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.BIGINT); + assertEquals(Long.MAX_VALUE, result); + } + + @Test + public void testConvertLongFromBinary() { + ByteBuffer buffer = ByteBuffer.allocate(8); + buffer.putLong(123456789L); + Object result = AccumuloTypeConverter.convert(buffer.array(), AccumuloColumnType.LONG); + assertEquals(123456789L, result); + } + + @Test + public void testConvertFloat() { + byte[] bytes = "3.14".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.FLOAT); + assertEquals(3.14f, (Float) result, 0.001); + } + + @Test + public void testConvertFloatFromBinary() { + ByteBuffer buffer = ByteBuffer.allocate(4); + buffer.putFloat(3.14f); + Object result = AccumuloTypeConverter.convert(buffer.array(), AccumuloColumnType.FLOAT); + assertEquals(3.14f, (Float) result, 0.001); + } + + @Test + public void testConvertDouble() { + byte[] bytes = "3.14159265359".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.DOUBLE); + assertEquals(3.14159265359, (Double) result, 0.00000000001); + } + + @Test + public void testConvertDoubleFromBinary() { + ByteBuffer buffer = ByteBuffer.allocate(8); + buffer.putDouble(3.14159265359); + Object result = AccumuloTypeConverter.convert(buffer.array(), AccumuloColumnType.DOUBLE); + assertEquals(3.14159265359, (Double) result, 0.00000000001); + } + + @Test + public void testConvertDecimal() { + byte[] bytes = "123456.789".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.DECIMAL); + assertEquals(new BigDecimal("123456.789"), result); + } + + @Test + public void testConvertBooleanTrue() { + byte[] bytes = "true".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.BOOLEAN); + assertTrue((Boolean) result); + } + + @Test + public void testConvertBooleanFalse() { + byte[] bytes = "false".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.BOOLEAN); + assertFalse((Boolean) result); + } + + @Test + public void testConvertBooleanOne() { + byte[] bytes = "1".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.BOOLEAN); + assertTrue((Boolean) result); + } + + @Test + public void testConvertBooleanZero() { + byte[] bytes = "0".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.BOOLEAN); + assertFalse((Boolean) result); + } + + @Test + public void testConvertBooleanYes() { + byte[] bytes = "YES".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.BOOLEAN); + assertTrue((Boolean) result); + } + + @Test + public void testConvertBooleanBinary() { + byte[] bytes = new byte[]{1}; + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.BOOLEAN); + assertTrue((Boolean) result); + + bytes = new byte[]{0}; + result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.BOOLEAN); + assertFalse((Boolean) result); + } + + @Test + public void testConvertDateIso() { + byte[] bytes = "2024-01-15".getBytes(StandardCharsets.UTF_8); + Long result = (Long) AccumuloTypeConverter.convert(bytes, AccumuloColumnType.DATE); + // 2024-01-15 00:00:00 UTC + assertEquals(1705276800000L, result.longValue()); + } + + @Test + public void testConvertTimeIso() { + byte[] bytes = "12:30:45".getBytes(StandardCharsets.UTF_8); + Integer result = (Integer) AccumuloTypeConverter.convert(bytes, AccumuloColumnType.TIME); + // 12:30:45 = 12*3600*1000 + 30*60*1000 + 45*1000 = 45045000 ms + assertEquals(45045000, result.intValue()); + } + + @Test + public void testConvertTimestampIso() { + byte[] bytes = "2024-01-15T12:30:45Z".getBytes(StandardCharsets.UTF_8); + Long result = (Long) AccumuloTypeConverter.convert(bytes, AccumuloColumnType.TIMESTAMP); + assertEquals(1705321845000L, result.longValue()); + } + + @Test + public void testConvertTimestampEpoch() { + byte[] bytes = "1705321845000".getBytes(StandardCharsets.UTF_8); + Long result = (Long) AccumuloTypeConverter.convert(bytes, AccumuloColumnType.TIMESTAMP); + assertEquals(1705321845000L, result.longValue()); + } + + @Test + public void testConvertVarbinary() { + byte[] bytes = new byte[]{0x01, 0x02, 0x03, (byte) 0xFF}; + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.VARBINARY); + assertArrayEquals(bytes, (byte[]) result); + } + + @Test + public void testConvertAny() { + byte[] bytes = "some value".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.ANY); + assertEquals("some value", result); + } + + @Test + public void testToDisplayStringPrintable() { + byte[] bytes = "Hello World".getBytes(StandardCharsets.UTF_8); + String result = AccumuloTypeConverter.toDisplayString(bytes); + assertEquals("Hello World", result); + } + + @Test + public void testToDisplayStringBinary() { + byte[] bytes = new byte[]{0x01, 0x02, 0x03}; + String result = AccumuloTypeConverter.toDisplayString(bytes); + assertEquals("0x010203", result); + } + + @Test + public void testToDisplayStringNull() { + String result = AccumuloTypeConverter.toDisplayString(null); + assertEquals("null", result); + } + + @Test + public void testToDisplayStringEmpty() { + String result = AccumuloTypeConverter.toDisplayString(new byte[0]); + assertEquals("", result); + } + + @Test + public void testConvertShort() { + byte[] bytes = "32767".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.SMALLINT); + assertEquals((short) 32767, result); + } + + @Test + public void testConvertByte() { + byte[] bytes = "127".getBytes(StandardCharsets.UTF_8); + Object result = AccumuloTypeConverter.convert(bytes, AccumuloColumnType.TINYINT); + assertEquals((byte) 127, result); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java new file mode 100644 index 00000000000..dedfb5dbf83 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import java.util.List; + +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.rpc.user.QueryDataBatch; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.test.BaseTestQuery; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; + +/** + * Base class for Accumulo integration tests. + * + *

This class sets up the Drill test cluster and registers the Accumulo storage plugin + * configured to connect to the MiniAccumuloCluster.

+ */ +public class BaseAccumuloTest extends BaseTestQuery { + + public static final String ACCUMULO_STORAGE_PLUGIN_NAME = "accumulo"; + + protected static AccumuloStoragePlugin storagePlugin; + protected static AccumuloStoragePluginConfig storagePluginConfig; + + @BeforeClass + public static void setupDefaultTestCluster() throws Exception { + // Initialize the MiniAccumuloCluster + boolean isManaged = Boolean.parseBoolean(System.getProperty("drill.accumulo.tests.managed", "true")); + AccumuloIntegrationTestsSuite.configure(isManaged, true); + AccumuloIntegrationTestsSuite.initCluster(); + + // Start the Drill test cluster + BaseTestQuery.setupDefaultTestCluster(); + + // Register Accumulo storage plugin + StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage(); + storagePluginConfig = new AccumuloStoragePluginConfig( + AccumuloIntegrationTestsSuite.getZooKeepers(), + AccumuloIntegrationTestsSuite.getInstanceName(), + AccumuloIntegrationTestsSuite.getRootUser(), + AccumuloIntegrationTestsSuite.getRootPassword(), + null, // schemaMetadataTable + null, // clientTimeout + null // batchScannerThreads + ); + storagePluginConfig.setEnabled(true); + + pluginRegistry.put(ACCUMULO_STORAGE_PLUGIN_NAME, storagePluginConfig); + storagePlugin = (AccumuloStoragePlugin) pluginRegistry.getPlugin(ACCUMULO_STORAGE_PLUGIN_NAME); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + AccumuloIntegrationTestsSuite.tearDownCluster(); + } + + /** + * Runs a SQL query and verifies the row count. + */ + protected void runAccumuloSQLVerifyCount(String sql, int expectedRowCount) throws Exception { + List results = testSqlWithResults(sql); + logResultAndVerifyRowCount(results, expectedRowCount); + } + + /** + * Runs a SQL query and returns the results for inspection. + */ + protected List runAccumuloSQLWithResults(String sql) throws Exception { + return testSqlWithResults(sql); + } + + /** + * Logs the results and verifies the row count. + */ + private void logResultAndVerifyRowCount(List results, int expectedRowCount) + throws SchemaChangeException { + int rowCount = logResult(results); + if (expectedRowCount != -1) { + Assert.assertEquals(expectedRowCount, rowCount); + } + } + + /** + * Returns the fully qualified table name for Drill queries. + * + * @param tableName the Accumulo table name + * @return the fully qualified name like "accumulo.`tableName`" + */ + protected String fullTableName(String tableName) { + return ACCUMULO_STORAGE_PLUGIN_NAME + ".`" + tableName + "`"; + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/DelegationTokenInfoTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/DelegationTokenInfoTest.java new file mode 100644 index 00000000000..a5323ccde00 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/DelegationTokenInfoTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.Base64; + +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Unit tests for DelegationTokenInfo. + */ +public class DelegationTokenInfoTest extends BaseTest { + + private static final String TEST_TOKEN_CLASS = "org.apache.accumulo.core.clientImpl.DelegationTokenImpl"; + + @Test + public void testBasicConstruction() { + long now = System.currentTimeMillis(); + String serializedToken = Base64.getEncoder().encodeToString("test-token-data".getBytes()); + + DelegationTokenInfo tokenInfo = new DelegationTokenInfo( + "testUser", serializedToken, TEST_TOKEN_CLASS, now); + + assertEquals("testUser", tokenInfo.getUserName()); + assertEquals(serializedToken, tokenInfo.getSerializedToken()); + assertEquals(TEST_TOKEN_CLASS, tokenInfo.getTokenClassName()); + assertEquals(now, tokenInfo.getCreationTime()); + } + + @Test + public void testGetAgeMillis() throws InterruptedException { + long before = System.currentTimeMillis(); + DelegationTokenInfo tokenInfo = new DelegationTokenInfo( + "user", + Base64.getEncoder().encodeToString("data".getBytes()), + TEST_TOKEN_CLASS, + before + ); + + // Sleep a bit to let time pass + Thread.sleep(50); + + long age = tokenInfo.getAgeMillis(); + assertTrue("Age should be at least 50ms", age >= 50); + } + + @Test + public void testIsOlderThan() { + long now = System.currentTimeMillis(); + DelegationTokenInfo tokenInfo = new DelegationTokenInfo( + "user", + Base64.getEncoder().encodeToString("data".getBytes()), + TEST_TOKEN_CLASS, + now - 5000 // Created 5 seconds ago + ); + + assertTrue("Token should be older than 1 second", tokenInfo.isOlderThan(1000)); + assertFalse("Token should not be older than 1 hour", tokenInfo.isOlderThan(3600000)); + } + + @Test + public void testEquality() { + long time = System.currentTimeMillis(); + String token = Base64.getEncoder().encodeToString("token".getBytes()); + + DelegationTokenInfo info1 = new DelegationTokenInfo("user", token, TEST_TOKEN_CLASS, time); + DelegationTokenInfo info2 = new DelegationTokenInfo("user", token, TEST_TOKEN_CLASS, time); + DelegationTokenInfo info3 = new DelegationTokenInfo("differentUser", token, TEST_TOKEN_CLASS, time); + + assertEquals(info1, info2); + assertEquals(info1.hashCode(), info2.hashCode()); + assertNotEquals(info1, info3); + } + + @Test + public void testJsonSerialization() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + long time = 1234567890123L; + String serializedToken = Base64.getEncoder().encodeToString("test-token".getBytes()); + + DelegationTokenInfo original = new DelegationTokenInfo( + "testUser", serializedToken, TEST_TOKEN_CLASS, time); + + // Serialize to JSON + String json = mapper.writeValueAsString(original); + assertNotNull(json); + assertTrue(json.contains("testUser")); + assertTrue(json.contains(serializedToken)); + assertTrue(json.contains("1234567890123")); + assertTrue(json.contains("tokenClassName")); + + // Deserialize back + DelegationTokenInfo deserialized = mapper.readValue(json, DelegationTokenInfo.class); + + assertEquals(original.getUserName(), deserialized.getUserName()); + assertEquals(original.getSerializedToken(), deserialized.getSerializedToken()); + assertEquals(original.getTokenClassName(), deserialized.getTokenClassName()); + assertEquals(original.getCreationTime(), deserialized.getCreationTime()); + assertEquals(original, deserialized); + } + + @Test + public void testJsonRoundTrip() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + // Create with current time + DelegationTokenInfo original = new DelegationTokenInfo( + "drillUser", + Base64.getEncoder().encodeToString("serialized-delegation-token".getBytes()), + TEST_TOKEN_CLASS, + System.currentTimeMillis() + ); + + // Round-trip through JSON + String json = mapper.writeValueAsString(original); + DelegationTokenInfo roundTripped = mapper.readValue(json, DelegationTokenInfo.class); + + assertEquals(original, roundTripped); + } + + @Test + public void testToString() { + String serializedToken = Base64.getEncoder().encodeToString("token-data".getBytes()); + DelegationTokenInfo tokenInfo = new DelegationTokenInfo( + "user", serializedToken, TEST_TOKEN_CLASS, 1234567890L); + + String toString = tokenInfo.toString(); + + assertTrue(toString.contains("user")); + assertTrue(toString.contains("1234567890")); + assertTrue(toString.contains("tokenClassName")); + // Should contain token length, not the actual token + assertTrue(toString.contains("tokenLength")); + // Should not contain the actual serialized token for security + assertFalse(toString.contains(serializedToken)); + } + + @Test + public void testTokenLengthInToString() { + String shortToken = Base64.getEncoder().encodeToString("short".getBytes()); + String longToken = Base64.getEncoder().encodeToString("this-is-a-much-longer-token".getBytes()); + + DelegationTokenInfo shortInfo = new DelegationTokenInfo("user", shortToken, TEST_TOKEN_CLASS, 0); + DelegationTokenInfo longInfo = new DelegationTokenInfo("user", longToken, TEST_TOKEN_CLASS, 0); + + // toString should show different token lengths + assertTrue(shortInfo.toString().contains(String.valueOf(shortToken.length()))); + assertTrue(longInfo.toString().contains(String.valueOf(longToken.length()))); + } + + @Test + public void testDifferentTokenClassNames() { + String token = Base64.getEncoder().encodeToString("token".getBytes()); + long time = System.currentTimeMillis(); + + DelegationTokenInfo info1 = new DelegationTokenInfo("user", token, "ClassA", time); + DelegationTokenInfo info2 = new DelegationTokenInfo("user", token, "ClassB", time); + + // Different token class names should result in different objects + assertNotEquals(info1, info2); + assertNotEquals(info1.hashCode(), info2.hashCode()); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/DrillAccumuloConstantsTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/DrillAccumuloConstantsTest.java new file mode 100644 index 00000000000..f98538b00e9 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/DrillAccumuloConstantsTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +/** + * Unit tests for DrillAccumuloConstants. + */ +public class DrillAccumuloConstantsTest extends BaseTest { + + @Test + public void testRowKeyConstant() { + assertEquals("row_key", DrillAccumuloConstants.ROW_KEY); + } + + @Test + public void testRowKeyPath() { + assertNotNull(DrillAccumuloConstants.ROW_KEY_PATH); + assertEquals("row_key", DrillAccumuloConstants.ROW_KEY_PATH.getRootSegment().getPath()); + } + + @Test + public void testRowKeyType() { + assertNotNull(DrillAccumuloConstants.ROW_KEY_TYPE); + assertEquals(MinorType.VARBINARY, DrillAccumuloConstants.ROW_KEY_TYPE.getMinorType()); + assertEquals(DataMode.REQUIRED, DrillAccumuloConstants.ROW_KEY_TYPE.getMode()); + } + + @Test + public void testColumnFamilyType() { + assertNotNull(DrillAccumuloConstants.COLUMN_FAMILY_TYPE); + assertEquals(MinorType.MAP, DrillAccumuloConstants.COLUMN_FAMILY_TYPE.getMinorType()); + assertEquals(DataMode.REQUIRED, DrillAccumuloConstants.COLUMN_FAMILY_TYPE.getMode()); + } + + @Test + public void testColumnType() { + assertNotNull(DrillAccumuloConstants.COLUMN_TYPE); + assertEquals(MinorType.VARBINARY, DrillAccumuloConstants.COLUMN_TYPE.getMinorType()); + assertEquals(DataMode.OPTIONAL, DrillAccumuloConstants.COLUMN_TYPE.getMode()); + } + + @Test + public void testColumnSeparator() { + assertEquals(":", DrillAccumuloConstants.COLUMN_SEPARATOR); + } + + @Test + public void testDefaultBatchSize() { + assertEquals(4000, DrillAccumuloConstants.DEFAULT_BATCH_SIZE); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/schema/AccumuloColumnTypeTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/schema/AccumuloColumnTypeTest.java new file mode 100644 index 00000000000..1118e587ffa --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/schema/AccumuloColumnTypeTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo.schema; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +/** + * Unit tests for AccumuloColumnType. + */ +public class AccumuloColumnTypeTest extends BaseTest { + + @Test + public void testSqlTypeMapping() { + assertEquals(SqlTypeName.VARCHAR, AccumuloColumnType.VARCHAR.getSqlTypeName()); + assertEquals(SqlTypeName.INTEGER, AccumuloColumnType.INT.getSqlTypeName()); + assertEquals(SqlTypeName.INTEGER, AccumuloColumnType.INTEGER.getSqlTypeName()); + assertEquals(SqlTypeName.BIGINT, AccumuloColumnType.BIGINT.getSqlTypeName()); + assertEquals(SqlTypeName.BIGINT, AccumuloColumnType.LONG.getSqlTypeName()); + assertEquals(SqlTypeName.FLOAT, AccumuloColumnType.FLOAT.getSqlTypeName()); + assertEquals(SqlTypeName.DOUBLE, AccumuloColumnType.DOUBLE.getSqlTypeName()); + assertEquals(SqlTypeName.BOOLEAN, AccumuloColumnType.BOOLEAN.getSqlTypeName()); + assertEquals(SqlTypeName.DATE, AccumuloColumnType.DATE.getSqlTypeName()); + assertEquals(SqlTypeName.TIME, AccumuloColumnType.TIME.getSqlTypeName()); + assertEquals(SqlTypeName.TIMESTAMP, AccumuloColumnType.TIMESTAMP.getSqlTypeName()); + assertEquals(SqlTypeName.VARBINARY, AccumuloColumnType.VARBINARY.getSqlTypeName()); + assertEquals(SqlTypeName.ANY, AccumuloColumnType.ANY.getSqlTypeName()); + } + + @Test + public void testFromString() { + // Direct matches + assertEquals(AccumuloColumnType.VARCHAR, AccumuloColumnType.fromString("VARCHAR")); + assertEquals(AccumuloColumnType.INTEGER, AccumuloColumnType.fromString("INTEGER")); + assertEquals(AccumuloColumnType.BIGINT, AccumuloColumnType.fromString("BIGINT")); + assertEquals(AccumuloColumnType.DOUBLE, AccumuloColumnType.fromString("DOUBLE")); + assertEquals(AccumuloColumnType.BOOLEAN, AccumuloColumnType.fromString("BOOLEAN")); + + // Case insensitive + assertEquals(AccumuloColumnType.VARCHAR, AccumuloColumnType.fromString("varchar")); + assertEquals(AccumuloColumnType.INTEGER, AccumuloColumnType.fromString("integer")); + assertEquals(AccumuloColumnType.BOOLEAN, AccumuloColumnType.fromString("Boolean")); + } + + @Test + public void testFromStringAliases() { + // String aliases + assertEquals(AccumuloColumnType.VARCHAR, AccumuloColumnType.fromString("STRING")); + assertEquals(AccumuloColumnType.VARCHAR, AccumuloColumnType.fromString("TEXT")); + + // Integer aliases + assertEquals(AccumuloColumnType.INTEGER, AccumuloColumnType.fromString("INT")); + + // Long aliases + assertEquals(AccumuloColumnType.BIGINT, AccumuloColumnType.fromString("LONG")); + + // Boolean aliases + assertEquals(AccumuloColumnType.BOOLEAN, AccumuloColumnType.fromString("BOOL")); + + // Binary aliases + assertEquals(AccumuloColumnType.VARBINARY, AccumuloColumnType.fromString("BYTES")); + assertEquals(AccumuloColumnType.VARBINARY, AccumuloColumnType.fromString("BINARY")); + } + + @Test + public void testFromStringDefault() { + // Unknown types should default to VARCHAR + assertEquals(AccumuloColumnType.VARCHAR, AccumuloColumnType.fromString("UNKNOWN")); + assertEquals(AccumuloColumnType.VARCHAR, AccumuloColumnType.fromString("")); + assertEquals(AccumuloColumnType.VARCHAR, AccumuloColumnType.fromString(null)); + assertEquals(AccumuloColumnType.VARCHAR, AccumuloColumnType.fromString(" ")); + } + + @Test + public void testIsNumeric() { + assertTrue(AccumuloColumnType.INT.isNumeric()); + assertTrue(AccumuloColumnType.INTEGER.isNumeric()); + assertTrue(AccumuloColumnType.BIGINT.isNumeric()); + assertTrue(AccumuloColumnType.LONG.isNumeric()); + assertTrue(AccumuloColumnType.SMALLINT.isNumeric()); + assertTrue(AccumuloColumnType.TINYINT.isNumeric()); + assertTrue(AccumuloColumnType.FLOAT.isNumeric()); + assertTrue(AccumuloColumnType.DOUBLE.isNumeric()); + assertTrue(AccumuloColumnType.DECIMAL.isNumeric()); + + assertFalse(AccumuloColumnType.VARCHAR.isNumeric()); + assertFalse(AccumuloColumnType.BOOLEAN.isNumeric()); + assertFalse(AccumuloColumnType.DATE.isNumeric()); + assertFalse(AccumuloColumnType.VARBINARY.isNumeric()); + } + + @Test + public void testIsTemporal() { + assertTrue(AccumuloColumnType.DATE.isTemporal()); + assertTrue(AccumuloColumnType.TIME.isTemporal()); + assertTrue(AccumuloColumnType.TIMESTAMP.isTemporal()); + + assertFalse(AccumuloColumnType.VARCHAR.isTemporal()); + assertFalse(AccumuloColumnType.INT.isTemporal()); + assertFalse(AccumuloColumnType.BOOLEAN.isTemporal()); + assertFalse(AccumuloColumnType.VARBINARY.isTemporal()); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/schema/TableSchemaTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/schema/TableSchemaTest.java new file mode 100644 index 00000000000..f6c146128ac --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/schema/TableSchemaTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo.schema; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Unit tests for ColumnDef and TableSchema. + */ +public class TableSchemaTest extends BaseTest { + + @Test + public void testColumnDefCreation() { + ColumnDef col = new ColumnDef("name", "cf1", "name", AccumuloColumnType.VARCHAR, true); + + assertEquals("name", col.getName()); + assertEquals("cf1", col.getColumnFamily()); + assertEquals("name", col.getColumnQualifier()); + assertEquals(AccumuloColumnType.VARCHAR, col.getType()); + assertTrue(col.isNullable()); + assertEquals(SqlTypeName.VARCHAR, col.getSqlTypeName()); + assertEquals("cf1:name", col.getFullColumnName()); + } + + @Test + public void testColumnDefFactoryMethods() { + ColumnDef col1 = ColumnDef.create("age", "cf1", "age", AccumuloColumnType.INT); + assertEquals("age", col1.getName()); + assertEquals(AccumuloColumnType.INT, col1.getType()); + assertTrue(col1.isNullable()); + + ColumnDef col2 = ColumnDef.varchar("email", "cf2", "email"); + assertEquals("email", col2.getName()); + assertEquals(AccumuloColumnType.VARCHAR, col2.getType()); + } + + @Test + public void testColumnDefFullColumnName() { + ColumnDef col1 = ColumnDef.varchar("name", "cf1", "name"); + assertEquals("cf1:name", col1.getFullColumnName()); + + ColumnDef col2 = ColumnDef.varchar("data", "cf1", ""); + assertEquals("cf1", col2.getFullColumnName()); + + ColumnDef col3 = ColumnDef.varchar("data", "cf1", null); + assertEquals("cf1", col3.getFullColumnName()); + } + + @Test + public void testTableSchemaBuilder() { + TableSchema schema = TableSchema.builder("users") + .rowKeyType(AccumuloColumnType.VARCHAR) + .addColumn("name", "cf1", "name", AccumuloColumnType.VARCHAR) + .addColumn("age", "cf1", "age", AccumuloColumnType.INT) + .addVarcharColumn("email", "cf2", "email") + .build(); + + assertEquals("users", schema.getTableName()); + assertEquals(AccumuloColumnType.VARCHAR, schema.getRowKeyType()); + assertEquals(3, schema.getColumnCount()); + assertTrue(schema.hasExplicitColumns()); + } + + @Test + public void testTableSchemaColumnLookup() { + TableSchema schema = TableSchema.builder("test") + .addColumn("name", "cf1", "name", AccumuloColumnType.VARCHAR) + .addColumn("age", "cf1", "age", AccumuloColumnType.INT) + .build(); + + // By name + ColumnDef byName = schema.getColumnByName("name"); + assertNotNull(byName); + assertEquals("name", byName.getName()); + + ColumnDef byNameCase = schema.getColumnByName("NAME"); + assertNotNull(byNameCase); + assertEquals("name", byNameCase.getName()); + + ColumnDef notFound = schema.getColumnByName("notexist"); + assertNull(notFound); + + // By Accumulo key + ColumnDef byKey = schema.getColumnByAccumuloKey("cf1:name"); + assertNotNull(byKey); + assertEquals("name", byKey.getName()); + + ColumnDef byKeyNotFound = schema.getColumnByAccumuloKey("cf2:name"); + assertNull(byKeyNotFound); + } + + @Test + public void testDynamicSchema() { + TableSchema schema = TableSchema.dynamic("dynamic_table"); + + assertEquals("dynamic_table", schema.getTableName()); + assertEquals(AccumuloColumnType.VARBINARY, schema.getRowKeyType()); + assertFalse(schema.hasExplicitColumns()); + assertEquals(0, schema.getColumnCount()); + } + + @Test + public void testTableSchemaEquality() { + TableSchema schema1 = TableSchema.builder("test") + .addVarcharColumn("name", "cf1", "name") + .build(); + + TableSchema schema2 = TableSchema.builder("test") + .addVarcharColumn("name", "cf1", "name") + .build(); + + TableSchema schema3 = TableSchema.builder("test") + .addColumn("name", "cf1", "name", AccumuloColumnType.INT) + .build(); + + assertEquals(schema1, schema2); + assertEquals(schema1.hashCode(), schema2.hashCode()); + assertFalse(schema1.equals(schema3)); + } + + @Test + public void testTableSchemaJsonSerialization() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + TableSchema schema = TableSchema.builder("users") + .rowKeyType(AccumuloColumnType.VARCHAR) + .addColumn("name", "cf1", "name", AccumuloColumnType.VARCHAR) + .addColumn("age", "cf1", "age", AccumuloColumnType.INT) + .build(); + + String json = mapper.writeValueAsString(schema); + assertNotNull(json); + assertTrue(json.contains("users")); + assertTrue(json.contains("name")); + assertTrue(json.contains("age")); + + TableSchema deserialized = mapper.readValue(json, TableSchema.class); + assertEquals(schema.getTableName(), deserialized.getTableName()); + assertEquals(schema.getRowKeyType(), deserialized.getRowKeyType()); + assertEquals(schema.getColumnCount(), deserialized.getColumnCount()); + } + + @Test + public void testColumnDefJsonSerialization() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + ColumnDef col = new ColumnDef("name", "cf1", "qualifier1", AccumuloColumnType.VARCHAR, false); + + String json = mapper.writeValueAsString(col); + assertNotNull(json); + assertTrue(json.contains("name")); + assertTrue(json.contains("cf1")); + assertTrue(json.contains("qualifier1")); + assertTrue(json.contains("VARCHAR")); + + ColumnDef deserialized = mapper.readValue(json, ColumnDef.class); + assertEquals(col, deserialized); + } + + @Test + public void testTableSchemaDefaultValues() { + // Null rowKeyType should default to VARBINARY + TableSchema schema = new TableSchema("test", null, null); + assertEquals(AccumuloColumnType.VARBINARY, schema.getRowKeyType()); + assertFalse(schema.hasExplicitColumns()); + } +} diff --git a/distribution/pom.xml b/distribution/pom.xml index 10e27292638..f16424432bd 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -436,6 +436,11 @@ drill-storage-kafka ${project.version} + + org.apache.drill.contrib + drill-storage-accumulo + ${project.version} + org.apache.drill.contrib drill-storage-cassandra diff --git a/distribution/src/assemble/component.xml b/distribution/src/assemble/component.xml index 71974cdd127..c74dbad934d 100644 --- a/distribution/src/assemble/component.xml +++ b/distribution/src/assemble/component.xml @@ -50,6 +50,7 @@ org.apache.drill.contrib:drill-mongo-storage:jar org.apache.drill.contrib:drill-opentsdb-storage:jar org.apache.drill.contrib:drill-paimon-format:jar + org.apache.drill.contrib:drill-storage-accumulo:jar org.apache.drill.contrib:drill-storage-cassandra:jar org.apache.drill.contrib:drill-storage-elasticsearch:jar org.apache.drill.contrib:drill-storage-googlesheets:jar From b63f92cbe2074b4db6c4abd3eaa01d801fb8987e Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sun, 2 Aug 2026 00:38:58 -0700 Subject: [PATCH 2/6] DRILL-8552: Fix AccumuloScanSpec constructor calls in filter builder Pass the sortDescending flag to the 10-arg AccumuloScanSpec constructor in createScanSpec() and mergeScanSpecs(), which were still using the old 9-arg signature and failing compilation. --- .../drill/exec/store/accumulo/AccumuloFilterBuilder.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilder.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilder.java index 0b38d5b1fc8..89ab3244a38 100644 --- a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilder.java +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilder.java @@ -211,7 +211,8 @@ private AccumuloScanSpec createScanSpecFromComparison( groupScan.getScanSpec().getColumns(), null, // No filter expression needed when using row ranges groupScan.getScanSpec().getLimit(), - groupScan.getScanSpec().isUseSortedScanner()); + groupScan.getScanSpec().isUseSortedScanner(), + groupScan.getScanSpec().isSortDescending()); } /** @@ -253,7 +254,8 @@ private AccumuloScanSpec mergeScanSpecs( leftSpec.getColumns(), leftSpec.getFilterExpression(), leftSpec.getLimit(), - leftSpec.isUseSortedScanner()); + leftSpec.isUseSortedScanner(), + leftSpec.isSortDescending()); } /** From 61654ba6719fb0987506d73fa9332cb6ee5f5147 Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sun, 2 Aug 2026 00:39:48 -0700 Subject: [PATCH 3/6] Add result-cache/ to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 67e5cbbb905..e3d906196e0 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ Makefile .mvn/maven.config *.patch .project +result-cache/ .settings/ *.swp TAGS From ede8ae12479cd9ab9cbb9cfb7334d7310054668d Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sun, 2 Aug 2026 05:21:02 -0700 Subject: [PATCH 4/6] Fix test issues --- contrib/storage-accumulo/pom.xml | 16 ++++++++++++++++ .../store/accumulo/AccumuloRecordReader.java | 1 - .../store/accumulo/AccumuloStoragePlugin.java | 1 - .../accumulo/AccumuloFilterBuilderTest.java | 2 -- .../store/accumulo/AccumuloSortPushdownTest.java | 2 -- .../exec/store/accumulo/BaseAccumuloTest.java | 5 +---- 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/contrib/storage-accumulo/pom.xml b/contrib/storage-accumulo/pom.xml index 09d3417b2ad..1ef7c52b473 100644 --- a/contrib/storage-accumulo/pom.xml +++ b/contrib/storage-accumulo/pom.xml @@ -95,6 +95,22 @@ log4j log4j + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-web + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.apache.logging.log4j + log4j-1.2-api + diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java index 4fd11af4dee..e327afa0b98 100644 --- a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.store.accumulo; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.HashMap; diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePlugin.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePlugin.java index acbf80dc3b4..1da3007aafc 100644 --- a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePlugin.java +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloStoragePlugin.java @@ -24,7 +24,6 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.drill.common.JSONOptions; import org.apache.drill.common.exceptions.UserException; -import org.apache.drill.common.logical.StoragePluginConfig.AuthMode; import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.store.accumulo.schema.AccumuloSchemaProvider; import org.apache.drill.exec.store.accumulo.schema.MetadataTableSchemaProvider; diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilderTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilderTest.java index ec421bf1a4f..515c98ad481 100644 --- a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilderTest.java +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloFilterBuilderTest.java @@ -19,8 +19,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.nio.charset.StandardCharsets; diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSortPushdownTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSortPushdownTest.java index 1124d1609fd..5e41b67aa0c 100644 --- a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSortPushdownTest.java +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSortPushdownTest.java @@ -19,8 +19,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; import org.apache.drill.exec.physical.base.ScanStats; diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java index dedfb5dbf83..d6794bfced6 100644 --- a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java @@ -56,10 +56,7 @@ public static void setupDefaultTestCluster() throws Exception { AccumuloIntegrationTestsSuite.getZooKeepers(), AccumuloIntegrationTestsSuite.getInstanceName(), AccumuloIntegrationTestsSuite.getRootUser(), - AccumuloIntegrationTestsSuite.getRootPassword(), - null, // schemaMetadataTable - null, // clientTimeout - null // batchScannerThreads + AccumuloIntegrationTestsSuite.getRootPassword() ); storagePluginConfig.setEnabled(true); From b3c09bb70ce8bff3ae3c5c9b8808318bd47b9313 Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sun, 2 Aug 2026 05:56:39 -0700 Subject: [PATCH 5/6] Fix CodeQL issue --- .../org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java index 14e2196d019..d2615814049 100644 --- a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java @@ -172,7 +172,7 @@ public ScanStats getScanStats() { // Adjust cost for pushdowns if (filterPushedDown) { cpuCost *= 0.5; - rowCount *= 0.5; + rowCount /= 2; } if (projectionPushedDown) { // Projection reduces network I/O significantly From 553bd18769e3d6654197f60ae2a224191371e460 Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Tue, 4 Aug 2026 07:55:23 -0700 Subject: [PATCH 6/6] Fixed Unit Tests --- contrib/storage-accumulo/README.md | 30 +- contrib/storage-accumulo/pom.xml | 19 +- .../store/accumulo/AccumuloGroupScan.java | 6 +- .../store/accumulo/AccumuloRecordReader.java | 19 +- .../store/accumulo/DrillAccumuloTable.java | 68 +++- .../accumulo/AccumuloBasicQueryTest.java | 26 +- .../AccumuloIntegrationTestsSuite.java | 4 +- .../accumulo/AccumuloLimitPushdownTest.java | 24 ++ .../AccumuloPushdownIntegrationTest.java | 60 +-- .../AccumuloResultVerificationTest.java | 347 ++++++++++++++++++ .../store/accumulo/AccumuloSerDeTest.java | 247 +++++++++++++ .../store/accumulo/AccumuloTestUtils.java | 50 +++ .../exec/store/accumulo/BaseAccumuloTest.java | 100 +++-- 13 files changed, 901 insertions(+), 99 deletions(-) create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloResultVerificationTest.java create mode 100644 contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSerDeTest.java diff --git a/contrib/storage-accumulo/README.md b/contrib/storage-accumulo/README.md index 5bc261c2d2b..4534ef71871 100644 --- a/contrib/storage-accumulo/README.md +++ b/contrib/storage-accumulo/README.md @@ -165,13 +165,23 @@ For production environments, `auth-conf` is recommended for full encryption of d -- Select all columns from a table SELECT * FROM accumulo.`my_table`; --- Select specific columns -SELECT row_key, cf.column1, cf.column2 FROM accumulo.`my_table`; +-- Select specific columns. Reaching into a column family requires a table alias +-- (`t` below), the same as for the HBase plugin. +SELECT row_key, t.cf.column1, t.cf.column2 FROM accumulo.`my_table` t; -- Select entire column family SELECT personal FROM accumulo.`users`; ``` +Values, including the row key, come back as `VARBINARY`. Decode them with +`CONVERT_FROM` to compare or display them as text: + +```sql +SELECT CONVERT_FROM(row_key, 'UTF8') AS row_key, + CONVERT_FROM(t.cf.name, 'UTF8') AS name +FROM accumulo.`my_table` t; +``` + ### Filter Queries ```sql @@ -201,8 +211,8 @@ SELECT * FROM accumulo.`my_table` ORDER BY row_key ASC; ```sql -- Filter, project, sort, and limit -SELECT row_key, cf.name, cf.value -FROM accumulo.`my_table` +SELECT row_key, t.cf.name, t.cf.value +FROM accumulo.`my_table` t WHERE row_key >= 'row_100' ORDER BY row_key ASC LIMIT 50; @@ -223,6 +233,12 @@ Accumulo: row_001 -> personal:first_name = "John", personal:last_name = "Doe" Drill: row_key = 'row_001', personal = {first_name: "John", last_name: "Doe"} ``` +Accumulo, unlike HBase, keeps no catalog of its column families, so when a table has +no entry in the schema metadata table the plugin infers them by reading the first +1000 entries of the table at plan time. A family that appears only later in the table +will not be visible to the planner; define an explicit schema for tables where that +matters. + ### Data Types All values are stored as `VARBINARY` by default. Use Drill's CAST functions for type conversion: @@ -230,9 +246,9 @@ All values are stored as `VARBINARY` by default. Use Drill's CAST functions for ```sql SELECT row_key, - CAST(cf.age AS INT) as age, - CAST(cf.salary AS DOUBLE) as salary -FROM accumulo.`employees`; + CAST(CONVERT_FROM(t.cf.age, 'UTF8') AS INT) as age, + CAST(CONVERT_FROM(t.cf.salary, 'UTF8') AS DOUBLE) as salary +FROM accumulo.`employees` t; ``` ## Schema Metadata (Optional) diff --git a/contrib/storage-accumulo/pom.xml b/contrib/storage-accumulo/pom.xml index 1ef7c52b473..bc56942e971 100644 --- a/contrib/storage-accumulo/pom.xml +++ b/contrib/storage-accumulo/pom.xml @@ -32,7 +32,11 @@ 2.1.4 - **/AccumuloTestsSuite.class + + **/Accumulo*TestsSuite.class @@ -64,6 +68,19 @@ + + + org.apache.zookeeper + zookeeper-jute + ${zookeeper.version} + + org.apache.drill.exec diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java index d2615814049..af14200ec32 100644 --- a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloGroupScan.java @@ -198,8 +198,10 @@ public boolean supportsLimitPushdown() { @Override public GroupScan applyLimit(int maxRecords) { - // If limit is already set and is more restrictive, keep the current one - if (this.maxRecords > 0 && this.maxRecords <= maxRecords) { + // If a limit is already set and is at least as restrictive, keep the current one. + // The bound must include zero: returning a new scan for an unchanged limit makes + // the planner rule fire on its own output forever (e.g. for LIMIT 0). + if (this.maxRecords >= 0 && this.maxRecords <= maxRecords) { return null; } diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java index e327afa0b98..c0686db6af7 100644 --- a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/AccumuloRecordReader.java @@ -224,16 +224,17 @@ private void configureRange() { byte[] startRow = scanSpec.getStartRow(); byte[] stopRow = scanSpec.getStopRow(); - if (startRow != null && stopRow != null) { - scanner.setRange(new Range( - new Text(startRow), true, - new Text(stopRow), false)); - } else if (startRow != null) { - scanner.setRange(new Range(new Text(startRow), null)); - } else if (stopRow != null) { - scanner.setRange(new Range(null, new Text(stopRow))); + if (startRow == null && stopRow == null) { + // Full table scan (default) + return; } - // else: full table scan (default) + + // The inclusive flags carry the difference between, say, `row_key < 'x'` and + // `row_key <= 'x'`, so they have to be passed through rather than assumed. A null + // bound means unbounded on that side. + scanner.setRange(new Range( + startRow == null ? null : new Text(startRow), scanSpec.isStartRowInclusive(), + stopRow == null ? null : new Text(stopRow), scanSpec.isStopRowInclusive())); } /** diff --git a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloTable.java b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloTable.java index f7d5699588c..5f136d512fa 100644 --- a/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloTable.java +++ b/contrib/storage-accumulo/src/main/java/org/apache/drill/exec/store/accumulo/DrillAccumuloTable.java @@ -18,7 +18,14 @@ package org.apache.drill.exec.store.accumulo; import java.util.ArrayList; - +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.sql.type.SqlTypeName; @@ -38,18 +45,27 @@ *

Schema resolution follows this order:

*
    *
  1. If explicit schema is defined in the metadata table, use that
    2. - *
  2. Otherwise, use a dynamic schema with row_key and a columns map
    3. + *
  3. Otherwise, expose row_key plus one map column per Accumulo column family, + * with the families inferred from a bounded sample of the table's data
    4. *
*/ public class DrillAccumuloTable extends DrillTable { private static final Logger logger = LoggerFactory.getLogger(DrillAccumuloTable.class); public static final String ROW_KEY_COLUMN = "row_key"; - public static final String COLUMNS_MAP_COLUMN = "columns"; private final AccumuloStoragePlugin plugin; private final AccumuloScanSpec scanSpec; + + /** + * Maximum number of Accumulo entries examined when inferring which column families + * a table contains. Accumulo has no catalog of column families, so they have to be + * read off the data itself; the cap keeps planning cheap on large tables. + */ + private static final int COLUMN_FAMILY_SAMPLE_SIZE = 1000; + private TableSchema tableSchema; + private Set columnFamilies; public DrillAccumuloTable( AccumuloStoragePlugin plugin, @@ -83,12 +99,16 @@ public RelDataType getRowType(RelDataTypeFactory typeFactory) { logger.debug("Using explicit schema for table '{}' with {} columns", scanSpec.getTableName(), schema.getColumnCount()); } else { - // Use dynamic schema with columns map - fieldNameList.add(COLUMNS_MAP_COLUMN); - typeList.add(typeFactory.createMapType( - typeFactory.createSqlType(SqlTypeName.VARCHAR), - typeFactory.createSqlType(SqlTypeName.ANY))); - logger.debug("Using dynamic schema for table '{}'", scanSpec.getTableName()); + // No explicit schema: expose one map column per Accumulo column family, which + // is the shape the record reader produces (row_key plus a MapVector per family). + for (String family : getColumnFamilies()) { + fieldNameList.add(family); + typeList.add(typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY))); + } + logger.debug("Using inferred schema for table '{}' with column families {}", + scanSpec.getTableName(), fieldNameList); } return typeFactory.createStructType(typeList, fieldNameList); @@ -113,6 +133,36 @@ private RelDataType createColumnType(RelDataTypeFactory typeFactory, ColumnDef c } } + /** + * Returns the column families present in the table, inferring them from a bounded + * sample of the table's data. + * + *

Unlike HBase, Accumulo does not declare its column families up front, so they + * are read from the first {@value #COLUMN_FAMILY_SAMPLE_SIZE} entries. A family that + * appears only beyond that point will not be visible to the planner; define an + * explicit schema in the metadata table for tables where that matters.

+ */ + private Set getColumnFamilies() { + if (columnFamilies == null) { + Set families = new LinkedHashSet<>(); + try (Scanner scanner = plugin.getClient() + .createScanner(scanSpec.getTableName(), Authorizations.EMPTY)) { + int examined = 0; + for (Map.Entry entry : scanner) { + families.add(entry.getKey().getColumnFamily().toString()); + if (++examined >= COLUMN_FAMILY_SAMPLE_SIZE) { + break; + } + } + } catch (Exception e) { + logger.warn("Failed to infer column families for table '{}'", + scanSpec.getTableName(), e); + } + columnFamilies = families; + } + return columnFamilies; + } + /** * Returns the table schema, loading it from the schema provider if needed. */ diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloBasicQueryTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloBasicQueryTest.java index c3a4b107824..50764b851d1 100644 --- a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloBasicQueryTest.java +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloBasicQueryTest.java @@ -29,65 +29,65 @@ public class AccumuloBasicQueryTest extends BaseAccumuloTest { @Test public void testSelectStarFromTable1() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t"; runAccumuloSQLVerifyCount(sql, 10); } @Test public void testSelectSpecificColumnsFromTable1() throws Exception { - String sql = "SELECT row_key, cf.name, cf.age FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + String sql = "SELECT row_key, t.cf.name, t.cf.age FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t"; runAccumuloSQLVerifyCount(sql, 10); } @Test public void testSelectRowKeyOnly() throws Exception { - String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t"; runAccumuloSQLVerifyCount(sql, 10); } @Test public void testSelectFromUsersTable() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + " t"; runAccumuloSQLVerifyCount(sql, 20); } @Test public void testSelectMultipleColumnFamilies() throws Exception { - String sql = "SELECT row_key, personal.first_name, personal.last_name, employment.company " + - "FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + String sql = "SELECT row_key, t.personal.first_name, t.personal.last_name, t.employment.company " + + "FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + " t"; runAccumuloSQLVerifyCount(sql, 20); } @Test public void testSelectFromLargeTable() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE); + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + " t"; runAccumuloSQLVerifyCount(sql, 1000); } @Test public void testCountStar() throws Exception { - String sql = "SELECT COUNT(*) FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + String sql = "SELECT COUNT(*) FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t"; runAccumuloSQLVerifyCount(sql, 1); } @Test public void testCountStarUsersTable() throws Exception { - String sql = "SELECT COUNT(*) FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + String sql = "SELECT COUNT(*) FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + " t"; runAccumuloSQLVerifyCount(sql, 1); } @Test public void testDistinctCompany() throws Exception { - String sql = "SELECT DISTINCT employment.company FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + String sql = "SELECT DISTINCT t.employment.company FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + " t"; // Should have 3 distinct companies: Acme Corp, TechCo, DataInc runAccumuloSQLVerifyCount(sql, 3); } @Test public void testGroupByCompany() throws Exception { - String sql = "SELECT employment.company, COUNT(*) as cnt " + - "FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + - " GROUP BY employment.company"; + String sql = "SELECT t.employment.company, COUNT(*) as cnt " + + "FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + " t" + + " GROUP BY t.employment.company"; runAccumuloSQLVerifyCount(sql, 3); } } diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloIntegrationTestsSuite.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloIntegrationTestsSuite.java index 5196f867613..b50ea4141b7 100644 --- a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloIntegrationTestsSuite.java +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloIntegrationTestsSuite.java @@ -44,7 +44,9 @@ @RunWith(Suite.class) @Suite.SuiteClasses({ AccumuloBasicQueryTest.class, - AccumuloPushdownIntegrationTest.class + AccumuloPushdownIntegrationTest.class, + AccumuloResultVerificationTest.class, + AccumuloSerDeTest.class }) public class AccumuloIntegrationTestsSuite extends BaseTest { private static final Logger logger = LoggerFactory.getLogger(AccumuloIntegrationTestsSuite.class); diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloLimitPushdownTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloLimitPushdownTest.java index faa8815b6db..b7c7d1744d9 100644 --- a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloLimitPushdownTest.java +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloLimitPushdownTest.java @@ -124,6 +124,30 @@ public void testApplyLimitWithLessRestrictiveExisting() { assertEquals(50, newScan.getMaxRecords()); } + @Test + public void testApplyLimitIsNotReappliedToItsOwnResult() { + // The planner rule keeps firing as long as applyLimit hands back a new scan, so + // re-applying the same limit must return null or planning never terminates. + AccumuloGroupScan original = createTestGroupScan(); + + AccumuloGroupScan limited = (AccumuloGroupScan) original.applyLimit(100); + + assertNull("Re-applying the same limit should return null", limited.applyLimit(100)); + } + + @Test + public void testApplyLimitZeroIsNotReapplied() { + // LIMIT 0 is the case that regressed: a zero limit must still be recognised as + // already pushed down. + AccumuloGroupScan original = createTestGroupScan(); + + AccumuloGroupScan limited = (AccumuloGroupScan) original.applyLimit(0); + + assertNotNull("A zero limit should still be pushed down", limited); + assertEquals(0, limited.getMaxRecords()); + assertNull("Re-applying a zero limit should return null", limited.applyLimit(0)); + } + @Test public void testApplyLimitPreservesOtherPushdowns() { AccumuloGroupScan original = createTestGroupScan(); diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloPushdownIntegrationTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloPushdownIntegrationTest.java index 73684680e44..a2e6e8244d4 100644 --- a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloPushdownIntegrationTest.java +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloPushdownIntegrationTest.java @@ -33,35 +33,35 @@ public class AccumuloPushdownIntegrationTest extends BaseAccumuloTest { @Test public void testFilterOnRowKeyEquals() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " WHERE row_key = 'row_001'"; runAccumuloSQLVerifyCount(sql, 1); } @Test public void testFilterOnRowKeyGreaterThan() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " WHERE row_key > 'row_005'"; runAccumuloSQLVerifyCount(sql, 5); // row_006 to row_010 } @Test public void testFilterOnRowKeyLessThan() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " WHERE row_key < 'row_004'"; runAccumuloSQLVerifyCount(sql, 3); // row_001 to row_003 } @Test public void testFilterOnRowKeyRange() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " WHERE row_key >= 'row_003' AND row_key <= 'row_007'"; runAccumuloSQLVerifyCount(sql, 5); // row_003 to row_007 } @Test public void testFilterOnRowKeyRangeLarge() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + " t" + " WHERE row_key >= 'row_0100' AND row_key < 'row_0200'"; runAccumuloSQLVerifyCount(sql, 100); // row_0100 to row_0199 } @@ -69,8 +69,8 @@ public void testFilterOnRowKeyRangeLarge() throws Exception { @Test public void testFilterOnColumnValue() throws Exception { // Note: column value filters may not be pushed down, but should still work - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + - " WHERE employment.company = 'Acme Corp'"; + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + " t" + + " WHERE t.employment.company = 'Acme Corp'"; runAccumuloSQLVerifyCount(sql, 7); // 7 users at Acme Corp } @@ -80,26 +80,26 @@ public void testFilterOnColumnValue() throws Exception { @Test public void testProjectionSingleColumn() throws Exception { - String sql = "SELECT cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + String sql = "SELECT t.cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t"; runAccumuloSQLVerifyCount(sql, 10); } @Test public void testProjectionMultipleColumns() throws Exception { - String sql = "SELECT cf.name, cf.city FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + String sql = "SELECT t.cf.name, t.cf.city FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t"; runAccumuloSQLVerifyCount(sql, 10); } @Test public void testProjectionWithRowKey() throws Exception { - String sql = "SELECT row_key, personal.first_name FROM " + - fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + String sql = "SELECT row_key, t.personal.first_name FROM " + + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + " t"; runAccumuloSQLVerifyCount(sql, 20); } @Test public void testProjectionSingleColumnFamily() throws Exception { - String sql = "SELECT personal FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + String sql = "SELECT personal FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS) + " t"; runAccumuloSQLVerifyCount(sql, 20); } @@ -109,26 +109,26 @@ public void testProjectionSingleColumnFamily() throws Exception { @Test public void testLimitSmall() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " LIMIT 5"; + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " LIMIT 5"; runAccumuloSQLVerifyCount(sql, 5); } @Test public void testLimitOnLargeTable() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + " LIMIT 50"; + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + " t" + " LIMIT 50"; runAccumuloSQLVerifyCount(sql, 50); } @Test public void testLimitOne() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " LIMIT 1"; + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " LIMIT 1"; runAccumuloSQLVerifyCount(sql, 1); } @Test public void testLimitLargerThanTable() throws Exception { // Limit larger than table size should return all rows - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " LIMIT 100"; + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " LIMIT 100"; runAccumuloSQLVerifyCount(sql, 10); } @@ -138,21 +138,21 @@ public void testLimitLargerThanTable() throws Exception { @Test public void testOrderByRowKeyAsc() throws Exception { - String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " ORDER BY row_key ASC"; runAccumuloSQLVerifyCount(sql, 10); } @Test public void testOrderByRowKeyDesc() throws Exception { - String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " ORDER BY row_key DESC"; runAccumuloSQLVerifyCount(sql, 10); } @Test public void testOrderByRowKeyWithLimit() throws Exception { - String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + " t" + " ORDER BY row_key ASC LIMIT 10"; runAccumuloSQLVerifyCount(sql, 10); } @@ -163,50 +163,50 @@ public void testOrderByRowKeyWithLimit() throws Exception { @Test public void testFilterAndProjection() throws Exception { - String sql = "SELECT row_key, cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT row_key, t.cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " WHERE row_key > 'row_005'"; runAccumuloSQLVerifyCount(sql, 5); } @Test public void testFilterAndLimit() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " WHERE row_key > 'row_002' LIMIT 3"; runAccumuloSQLVerifyCount(sql, 3); } @Test public void testProjectionAndLimit() throws Exception { - String sql = "SELECT row_key, cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT row_key, t.cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " LIMIT 5"; runAccumuloSQLVerifyCount(sql, 5); } @Test public void testFilterProjectionAndLimit() throws Exception { - String sql = "SELECT row_key, cf.name, cf.city FROM " + - fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT row_key, t.cf.name, t.cf.city FROM " + + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " WHERE row_key >= 'row_003' LIMIT 4"; runAccumuloSQLVerifyCount(sql, 4); } @Test public void testFilterAndSort() throws Exception { - String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " WHERE row_key < 'row_006' ORDER BY row_key ASC"; runAccumuloSQLVerifyCount(sql, 5); } @Test public void testSortAndLimit() throws Exception { - String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " ORDER BY row_key ASC LIMIT 3"; runAccumuloSQLVerifyCount(sql, 3); } @Test public void testAllPushdownsCombined() throws Exception { - String sql = "SELECT row_key, cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT row_key, t.cf.name FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " WHERE row_key >= 'row_002' AND row_key <= 'row_009'" + " ORDER BY row_key ASC LIMIT 5"; runAccumuloSQLVerifyCount(sql, 5); @@ -218,21 +218,21 @@ public void testAllPushdownsCombined() throws Exception { @Test public void testFilterNoResults() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " WHERE row_key = 'nonexistent'"; runAccumuloSQLVerifyCount(sql, 0); } @Test public void testFilterOutOfRange() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " WHERE row_key > 'zzz'"; runAccumuloSQLVerifyCount(sql, 0); } @Test public void testLimitZero() throws Exception { - String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " LIMIT 0"; + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1) + " t" + " LIMIT 0"; runAccumuloSQLVerifyCount(sql, 0); } } diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloResultVerificationTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloResultVerificationTest.java new file mode 100644 index 00000000000..b1dd39e7755 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloResultVerificationTest.java @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +/** + * End-to-end tests that verify the actual values Drill returns from Accumulo, + * not just the number of rows. + * + *

Accumulo row keys and values are surfaced to Drill as VARBINARY, so the queries + * here decode them with {@code CONVERT_FROM(..., 'UTF8')} before comparing against + * the data written by {@link AccumuloTestUtils}.

+ */ +public class AccumuloResultVerificationTest extends BaseAccumuloTest { + + // ========================================================================= + // Full table content + // ========================================================================= + + @Test + public void testAllRowsAndValuesFromTable1() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + ", " + + utf8("t.cf.name", "name") + ", " + + utf8("t.cf.age", "age") + ", " + + utf8("t.cf.city", "city") + + fromTable(AccumuloTestUtils.TEST_TABLE_1) + + " ORDER BY row_key"; + + testBuilder() + .sqlQuery(sql) + .ordered() + .baselineColumns("row_key", "name", "age", "city") + .baselineValues("row_001", "Alice", "30", "New York") + .baselineValues("row_002", "Bob", "25", "Los Angeles") + .baselineValues("row_003", "Charlie", "35", "Chicago") + .baselineValues("row_004", "Diana", "28", "Houston") + .baselineValues("row_005", "Eve", "32", "Phoenix") + .baselineValues("row_006", "Frank", "45", "Philadelphia") + .baselineValues("row_007", "Grace", "29", "San Antonio") + .baselineValues("row_008", "Henry", "38", "San Diego") + .baselineValues("row_009", "Ivy", "26", "Dallas") + .baselineValues("row_010", "Jack", "41", "San Jose") + .go(); + } + + @Test + public void testValuesAcrossMultipleColumnFamilies() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + ", " + + utf8("t.personal.first_name", "first_name") + ", " + + utf8("t.personal.last_name", "last_name") + ", " + + utf8("t.contact.email", "email") + ", " + + utf8("t.employment.company", "company") + ", " + + utf8("t.employment.salary", "salary") + + fromTable(AccumuloTestUtils.TEST_TABLE_USERS) + + " WHERE row_key IN ('user_001', 'user_014', 'user_020')" + + " ORDER BY row_key"; + + testBuilder() + .sqlQuery(sql) + .ordered() + .baselineColumns("row_key", "first_name", "last_name", "email", "company", "salary") + .baselineValues("user_001", "John", "Doe", "john.doe@email.com", "Acme Corp", "75000") + .baselineValues("user_014", "Laura", "White", "laura.w@email.com", "TechCo", "125000") + .baselineValues("user_020", "Rachel", "Clark", "rachel.c@email.com", "TechCo", "99000") + .go(); + } + + // ========================================================================= + // Filter pushdown: verify the correct rows come back, not just the count + // ========================================================================= + + @Test + public void testRowKeyEqualsReturnsMatchingRow() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + ", " + utf8("t.cf.name", "name") + + fromTable(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key = 'row_003'"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("row_key", "name") + .baselineValues("row_003", "Charlie") + .go(); + } + + @Test + public void testRowKeyRangeReturnsExactRows() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + ", " + utf8("t.cf.city", "city") + + fromTable(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key >= 'row_003' AND row_key <= 'row_005'" + + " ORDER BY row_key"; + + testBuilder() + .sqlQuery(sql) + .ordered() + .baselineColumns("row_key", "city") + .baselineValues("row_003", "Chicago") + .baselineValues("row_004", "Houston") + .baselineValues("row_005", "Phoenix") + .go(); + } + + @Test + public void testRowKeyGreaterThanReturnsExactRows() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + + fromTable(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key > 'row_007'" + + " ORDER BY row_key"; + + testBuilder() + .sqlQuery(sql) + .ordered() + .baselineColumns("row_key") + .baselineValues("row_008") + .baselineValues("row_009") + .baselineValues("row_010") + .go(); + } + + @Test + public void testValueFilterReturnsMatchingRows() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + + fromTable(AccumuloTestUtils.TEST_TABLE_USERS) + + " WHERE CONVERT_FROM(t.employment.title, 'UTF8') = 'Director'" + + " ORDER BY row_key"; + + testBuilder() + .sqlQuery(sql) + .ordered() + .baselineColumns("row_key") + .baselineValues("user_004") + .baselineValues("user_014") + .go(); + } + + @Test + public void testRowKeyRangeOnLargeTableBoundaries() throws Exception { + // Exercises a range that spans many rows: verify both endpoints and the count. + String sql = "SELECT MIN(rk) AS min_rk, MAX(rk) AS max_rk, COUNT(*) AS cnt FROM (" + + " SELECT " + utf8("row_key", "rk") + + " " + fromTable(AccumuloTestUtils.TEST_TABLE_LARGE) + + " WHERE row_key >= 'row_0100' AND row_key < 'row_0200')"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("min_rk", "max_rk", "cnt") + .baselineValues("row_0100", "row_0199", 100L) + .go(); + } + + // ========================================================================= + // Sort and limit + // ========================================================================= + + @Test + public void testOrderByRowKeyDescReturnsRowsInOrder() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + + fromTable(AccumuloTestUtils.TEST_TABLE_1) + + " ORDER BY row_key DESC LIMIT 3"; + + testBuilder() + .sqlQuery(sql) + .ordered() + .baselineColumns("row_key") + .baselineValues("row_010") + .baselineValues("row_009") + .baselineValues("row_008") + .go(); + } + + @Test + public void testOrderByWithLimitOnLargeTable() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + ", " + utf8("t.data.value", "value") + + fromTable(AccumuloTestUtils.TEST_TABLE_LARGE) + + " ORDER BY row_key LIMIT 3"; + + testBuilder() + .sqlQuery(sql) + .ordered() + .baselineColumns("row_key", "value") + .baselineValues("row_0001", "1") + .baselineValues("row_0002", "2") + .baselineValues("row_0003", "3") + .go(); + } + + @Test + public void testLimitReturnsDistinctRowsFromTheTable() throws Exception { + // A pushed-down limit must return whole, distinct rows rather than repeating or + // truncating them, so check the returned keys against the full key set. + String sql = "SELECT " + utf8("row_key", "row_key") + + fromTable(AccumuloTestUtils.TEST_TABLE_LARGE) + " LIMIT 25"; + + List keys = new ArrayList<>(); + for (List row : runAndReadStrings(sql)) { + keys.add(row.get(0)); + } + + assertEquals(25, keys.size()); + assertEquals("Limit must not return duplicate rows", 25, keys.stream().distinct().count()); + for (String key : keys) { + assertTrue("Unexpected row key returned: " + key, key.matches("row_0\\d{3}")); + int index = Integer.parseInt(key.substring(4)); + assertTrue("Row key out of range: " + key, index >= 1 && index <= 1000); + } + } + + // ========================================================================= + // Aggregates over Accumulo data + // ========================================================================= + + @Test + public void testCountStarReturnsRowCount() throws Exception { + String sql = "SELECT COUNT(*) AS cnt FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE); + assertEquals(1000L, queryBuilder().sql(sql).singletonLong()); + } + + @Test + public void testSumOverConvertedValues() throws Exception { + // Sum of 1..1000 + String sql = "SELECT SUM(CAST(CONVERT_FROM(t.data.value, 'UTF8') AS INT)) AS total" + + fromTable(AccumuloTestUtils.TEST_TABLE_LARGE); + assertEquals(500500L, queryBuilder().sql(sql).singletonLong()); + } + + @Test + public void testGroupByReturnsCorrectCounts() throws Exception { + String sql = "SELECT CONVERT_FROM(t.employment.company, 'UTF8') AS company, COUNT(*) AS cnt" + + fromTable(AccumuloTestUtils.TEST_TABLE_USERS) + + " GROUP BY CONVERT_FROM(t.employment.company, 'UTF8')" + + " ORDER BY company"; + + testBuilder() + .sqlQuery(sql) + .ordered() + .baselineColumns("company", "cnt") + .baselineValues("Acme Corp", 7L) + .baselineValues("DataInc", 6L) + .baselineValues("TechCo", 7L) + .go(); + } + + // ========================================================================= + // Sparse rows: missing qualifiers must read back as NULL + // ========================================================================= + + @Test + public void testMissingQualifiersReadBackAsNull() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + ", " + + utf8("t.cf.a", "a") + ", " + utf8("t.cf.b", "b") + ", " + utf8("t.cf.c", "c") + + fromTable(AccumuloTestUtils.TEST_TABLE_SPARSE) + + " ORDER BY row_key"; + + testBuilder() + .sqlQuery(sql) + .ordered() + .baselineColumns("row_key", "a", "b", "c") + .baselineValues("sparse_001", "a1", "b1", "c1") + .baselineValues("sparse_002", "a2", null, null) + .baselineValues("sparse_003", null, "b3", null) + .baselineValues("sparse_004", null, null, "c4") + .baselineValues("sparse_005", "a5", null, "c5") + .go(); + } + + @Test + public void testSparseTableIsNotNullFilter() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + + fromTable(AccumuloTestUtils.TEST_TABLE_SPARSE) + + " WHERE t.cf.b IS NOT NULL" + + " ORDER BY row_key"; + + testBuilder() + .sqlQuery(sql) + .ordered() + .baselineColumns("row_key") + .baselineValues("sparse_001") + .baselineValues("sparse_003") + .go(); + } + + // ========================================================================= + // Projection: unprojected data must not leak into the result + // ========================================================================= + + @Test + public void testProjectedFamilyContainsAllQualifiers() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + ", " + + utf8("t.personal.first_name", "first_name") + ", " + + utf8("t.personal.last_name", "last_name") + + fromTable(AccumuloTestUtils.TEST_TABLE_USERS) + + " WHERE row_key = 'user_007'"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("row_key", "first_name", "last_name") + .baselineValues("user_007", "Edward", "Miller") + .go(); + } + + @Test + public void testRowKeyOnlyProjectionReturnsAllKeys() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + + fromTable(AccumuloTestUtils.TEST_TABLE_1) + + " ORDER BY row_key"; + + testBuilder() + .sqlQuery(sql) + .ordered() + .baselineColumns("row_key") + .baselineValues("row_001") + .baselineValues("row_002") + .baselineValues("row_003") + .baselineValues("row_004") + .baselineValues("row_005") + .baselineValues("row_006") + .baselineValues("row_007") + .baselineValues("row_008") + .baselineValues("row_009") + .baselineValues("row_010") + .go(); + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSerDeTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSerDeTest.java new file mode 100644 index 00000000000..7d5a76f1c71 --- /dev/null +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloSerDeTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.accumulo; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.physical.base.FragmentLeaf; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.store.accumulo.AccumuloScanSpec.AccumuloColumnSpec; +import org.junit.Test; + +/** + * Serialization/deserialization tests for the Accumulo physical operators. + * + *

Drill serializes physical operators to JSON when it distributes fragments to + * other Drillbits, so anything that survives planning must survive a JSON round trip. + * These tests cover both the whole-plan path (plan the query, serialize it, then submit + * the serialized plan and check the results) and a direct round trip of + * {@link AccumuloSubScan} through Drill's {@link PhysicalPlanReader}.

+ */ +public class AccumuloSerDeTest extends BaseAccumuloTest { + + // ========================================================================= + // Whole-plan round trip: plan -> JSON -> execute -> verify results + // ========================================================================= + + @Test + public void testSerDeSelectStar() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_1); + String plan = queryBuilder().sql(sql).explainJson(); + + assertTrue("Plan should contain the Accumulo scan", plan.contains("accumulo-scan")); + assertEquals(10, queryBuilder().physical(plan).run().recordCount()); + } + + @Test + public void testSerDePreservesValues() throws Exception { + String sql = "SELECT " + utf8("row_key", "row_key") + ", " + utf8("t.cf.name", "name") + + fromTable(AccumuloTestUtils.TEST_TABLE_1) + + " WHERE row_key >= 'row_008' ORDER BY row_key"; + String plan = queryBuilder().sql(sql).explainJson(); + + assertEquals( + Arrays.asList( + Arrays.asList("row_008", "Henry"), + Arrays.asList("row_009", "Ivy"), + Arrays.asList("row_010", "Jack")), + readStringsFromPlan(plan)); + } + + @Test + public void testSerDePreservesRowRangePushdown() throws Exception { + String sql = "SELECT row_key FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + + " WHERE row_key >= 'row_0100' AND row_key < 'row_0200'"; + String plan = queryBuilder().sql(sql).explainJson(); + + // The deserialized plan must scan the same range, not the whole table. + assertEquals(100, queryBuilder().physical(plan).run().recordCount()); + } + + @Test + public void testSerDePreservesLimitPushdown() throws Exception { + String sql = "SELECT * FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_LARGE) + " LIMIT 17"; + String plan = queryBuilder().sql(sql).explainJson(); + + assertEquals(17, queryBuilder().physical(plan).run().recordCount()); + } + + @Test + public void testSerDeAggregate() throws Exception { + String sql = "SELECT COUNT(*) FROM " + fullTableName(AccumuloTestUtils.TEST_TABLE_USERS); + String plan = queryBuilder().sql(sql).explainJson(); + + assertEquals(20L, queryBuilder().physical(plan).singletonLong()); + } + + @Test + public void testFragmentSerDe() throws Exception { + // A slice target of 1 forces the plan to be split into fragments, which are + // serialized individually before being handed to the executor. + client.alterSession(ExecConstants.SLICE_TARGET, 1); + try { + String sql = "SELECT CONVERT_FROM(t.employment.company, 'UTF8') AS company, COUNT(*) AS cnt" + + fromTable(AccumuloTestUtils.TEST_TABLE_USERS) + + " GROUP BY CONVERT_FROM(t.employment.company, 'UTF8')"; + String plan = queryBuilder().sql(sql).explainJson(); + + List> rows = readStringsFromPlan(plan); + rows.sort(Comparator.comparing(row -> row.get(0))); + assertEquals( + Arrays.asList( + Arrays.asList("Acme Corp", "7"), + Arrays.asList("DataInc", "6"), + Arrays.asList("TechCo", "7")), + rows); + } finally { + client.resetSession(ExecConstants.SLICE_TARGET); + } + } + + // ========================================================================= + // Direct operator round trip + // ========================================================================= + + @Test + public void testSubScanSerDeRoundTrip() throws Exception { + AccumuloScanSpec scanSpec = new AccumuloScanSpec( + AccumuloTestUtils.TEST_TABLE_1, + "row_002".getBytes(StandardCharsets.UTF_8), + "row_008".getBytes(StandardCharsets.UTF_8), + true, + false, + Collections.singletonList(new AccumuloColumnSpec("cf", "name", "cf.name")), + "cf.age > 30", + 25, + true, + true); + + List columns = Arrays.asList( + SchemaPath.getSimplePath("row_key"), + SchemaPath.getCompoundPath("cf", "name")); + + AccumuloSubScan subScan = new AccumuloSubScan( + "testUser", storagePlugin, scanSpec, columns, 25, null); + + AccumuloSubScan deserialized = roundTrip(subScan); + + assertEquals("testUser", deserialized.getUserName()); + assertEquals(25, deserialized.getMaxRecords()); + assertEquals(columns, deserialized.getColumns()); + assertEquals(storagePluginConfig, deserialized.getStoragePluginConfig()); + assertNull(deserialized.getDelegationTokenInfo()); + + AccumuloScanSpec deserializedSpec = deserialized.getScanSpec(); + assertEquals(scanSpec, deserializedSpec); + assertEquals(AccumuloTestUtils.TEST_TABLE_1, deserializedSpec.getTableName()); + assertArrayEquals("row_002".getBytes(StandardCharsets.UTF_8), deserializedSpec.getStartRow()); + assertArrayEquals("row_008".getBytes(StandardCharsets.UTF_8), deserializedSpec.getStopRow()); + assertTrue(deserializedSpec.isStartRowInclusive()); + assertFalse(deserializedSpec.isStopRowInclusive()); + assertEquals("cf.age > 30", deserializedSpec.getFilterExpression()); + assertEquals(Integer.valueOf(25), deserializedSpec.getLimit()); + assertTrue(deserializedSpec.isUseSortedScanner()); + assertTrue(deserializedSpec.isSortDescending()); + assertEquals(1, deserializedSpec.getColumns().size()); + assertEquals("cf", deserializedSpec.getColumns().get(0).getColumnFamily()); + assertEquals("name", deserializedSpec.getColumns().get(0).getColumnQualifier()); + } + + @Test + public void testSubScanSerDeRoundTripWithDefaults() throws Exception { + AccumuloSubScan subScan = new AccumuloSubScan( + "testUser", + storagePlugin, + new AccumuloScanSpec(AccumuloTestUtils.TEST_TABLE_USERS), + null); + + AccumuloSubScan deserialized = roundTrip(subScan); + + assertEquals(-1, deserialized.getMaxRecords()); + assertNull(deserialized.getColumns()); + assertNull(deserialized.getDelegationTokenInfo()); + assertEquals(AccumuloTestUtils.TEST_TABLE_USERS, deserialized.getScanSpec().getTableName()); + assertNull(deserialized.getScanSpec().getStartRow()); + assertNull(deserialized.getScanSpec().getStopRow()); + } + + @Test + public void testSubScanSerDeRoundTripWithDelegationToken() throws Exception { + String serializedToken = Base64.getEncoder() + .encodeToString("fake-token-bytes".getBytes(StandardCharsets.UTF_8)); + long creationTime = System.currentTimeMillis(); + DelegationTokenInfo tokenInfo = new DelegationTokenInfo( + "alice", + serializedToken, + "org.apache.accumulo.core.client.security.tokens.DelegationTokenImpl", + creationTime); + + AccumuloSubScan subScan = new AccumuloSubScan( + "alice", + storagePlugin, + new AccumuloScanSpec(AccumuloTestUtils.TEST_TABLE_1), + null, + -1, + tokenInfo); + + AccumuloSubScan deserialized = roundTrip(subScan); + + assertTrue(deserialized.hasDelegationToken()); + DelegationTokenInfo deserializedToken = deserialized.getDelegationTokenInfo(); + assertNotNull(deserializedToken); + assertEquals("alice", deserializedToken.getUserName()); + assertEquals(serializedToken, deserializedToken.getSerializedToken()); + assertEquals("org.apache.accumulo.core.client.security.tokens.DelegationTokenImpl", + deserializedToken.getTokenClassName()); + assertEquals(creationTime, deserializedToken.getCreationTime()); + } + + /** + * Submits a serialized physical plan and returns the results as rows of strings. + */ + private List> readStringsFromPlan(String plan) throws Exception { + return readStrings(queryBuilder().physical(plan).rowSetIterator()); + } + + /** + * Writes the operator to JSON with Drill's physical plan mapper and reads it back, + * which is exactly what happens when a fragment is shipped to another Drillbit. + */ + private AccumuloSubScan roundTrip(AccumuloSubScan subScan) throws Exception { + PhysicalPlanReader reader = cluster.drillbit().getContext().getPlanReader(); + String json = reader.writeJson(subScan); + FragmentLeaf leaf = reader.readFragmentLeaf(json); + assertTrue("Expected an AccumuloSubScan, got " + leaf.getClass().getName(), + leaf instanceof AccumuloSubScan); + return (AccumuloSubScan) leaf; + } +} diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestUtils.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestUtils.java index 454bdaedae2..5d0e2077808 100644 --- a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestUtils.java +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/AccumuloTestUtils.java @@ -41,6 +41,7 @@ public class AccumuloTestUtils { public static final String TEST_TABLE_1 = "drill_test_table_1"; public static final String TEST_TABLE_USERS = "drill_test_users"; public static final String TEST_TABLE_LARGE = "drill_test_large"; + public static final String TEST_TABLE_SPARSE = "drill_test_sparse"; /** * Creates a simple test table with basic key-value data. @@ -168,6 +169,53 @@ public static void createTestTableLarge(AccumuloClient client) throws Exception logger.info("Created test table: {} with 1000 rows", tableName); } + /** + * Creates a test table where rows have different sets of column qualifiers. + * + *

This exercises the record reader's handling of columns that are absent from + * some rows: every missing qualifier must come back as NULL rather than shifting + * values between rows.

+ * + *

Table structure:

+ *
    + *
  • sparse_001: cf:a, cf:b, cf:c
    • + *
  • sparse_002: cf:a only
    • + *
  • sparse_003: cf:b only
    • + *
  • sparse_004: cf:c only
    • + *
  • sparse_005: cf:a, cf:c
    • + *
+ */ + public static void createTestTableSparse(AccumuloClient client) throws Exception { + String tableName = TEST_TABLE_SPARSE; + createTableIfNotExists(client, tableName); + + try (BatchWriter writer = client.createBatchWriter(tableName, new BatchWriterConfig())) { + // null means the qualifier is absent for that row + String[][] data = { + {"sparse_001", "a1", "b1", "c1"}, + {"sparse_002", "a2", null, null}, + {"sparse_003", null, "b3", null}, + {"sparse_004", null, null, "c4"}, + {"sparse_005", "a5", null, "c5"} + }; + + String[] qualifiers = {"a", "b", "c"}; + for (String[] row : data) { + Mutation m = new Mutation(new Text(row[0])); + for (int i = 0; i < qualifiers.length; i++) { + String value = row[i + 1]; + if (value != null) { + m.put(new Text("cf"), new Text(qualifiers[i]), + new Value(value.getBytes(StandardCharsets.UTF_8))); + } + } + writer.addMutation(m); + } + } + + logger.info("Created test table: {} with 5 sparse rows", tableName); + } + /** * Creates all test tables. */ @@ -175,6 +223,7 @@ public static void createAllTestTables(AccumuloClient client) throws Exception { createTestTable1(client); createTestTableUsers(client); createTestTableLarge(client); + createTestTableSparse(client); } /** @@ -184,6 +233,7 @@ public static void deleteAllTestTables(AccumuloClient client) throws Exception { deleteTableIfExists(client, TEST_TABLE_1); deleteTableIfExists(client, TEST_TABLE_USERS); deleteTableIfExists(client, TEST_TABLE_LARGE); + deleteTableIfExists(client, TEST_TABLE_SPARSE); } /** diff --git a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java index d6794bfced6..08c7f0644ab 100644 --- a/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java +++ b/contrib/storage-accumulo/src/test/java/org/apache/drill/exec/store/accumulo/BaseAccumuloTest.java @@ -17,14 +17,19 @@ */ package org.apache.drill.exec.store.accumulo; +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; import java.util.List; -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.rpc.user.QueryDataBatch; +import org.apache.drill.exec.physical.rowSet.DirectRowSet; +import org.apache.drill.exec.physical.rowSet.RowSetReader; import org.apache.drill.exec.store.StoragePluginRegistry; -import org.apache.drill.test.BaseTestQuery; +import org.apache.drill.exec.vector.accessor.ScalarReader; +import org.apache.drill.test.ClusterFixture; +import org.apache.drill.test.ClusterTest; +import org.apache.drill.test.QueryRowSetIterator; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.BeforeClass; /** @@ -33,7 +38,7 @@ *

This class sets up the Drill test cluster and registers the Accumulo storage plugin * configured to connect to the MiniAccumuloCluster.

*/ -public class BaseAccumuloTest extends BaseTestQuery { +public class BaseAccumuloTest extends ClusterTest { public static final String ACCUMULO_STORAGE_PLUGIN_NAME = "accumulo"; @@ -41,17 +46,17 @@ public class BaseAccumuloTest extends BaseTestQuery { protected static AccumuloStoragePluginConfig storagePluginConfig; @BeforeClass - public static void setupDefaultTestCluster() throws Exception { + public static void setupAccumuloTestCluster() throws Exception { // Initialize the MiniAccumuloCluster boolean isManaged = Boolean.parseBoolean(System.getProperty("drill.accumulo.tests.managed", "true")); AccumuloIntegrationTestsSuite.configure(isManaged, true); AccumuloIntegrationTestsSuite.initCluster(); // Start the Drill test cluster - BaseTestQuery.setupDefaultTestCluster(); + startCluster(ClusterFixture.builder(dirTestWatcher)); // Register Accumulo storage plugin - StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage(); + StoragePluginRegistry pluginRegistry = cluster.drillbit().getContext().getStorage(); storagePluginConfig = new AccumuloStoragePluginConfig( AccumuloIntegrationTestsSuite.getZooKeepers(), AccumuloIntegrationTestsSuite.getInstanceName(), @@ -65,43 +70,84 @@ public static void setupDefaultTestCluster() throws Exception { } @AfterClass - public static void tearDownAfterClass() throws Exception { + public static void tearDownAccumuloTestCluster() throws Exception { AccumuloIntegrationTestsSuite.tearDownCluster(); } /** - * Runs a SQL query and verifies the row count. + * Runs a SQL query and verifies the row count. Pass {@code -1} to skip the check. */ protected void runAccumuloSQLVerifyCount(String sql, int expectedRowCount) throws Exception { - List results = testSqlWithResults(sql); - logResultAndVerifyRowCount(results, expectedRowCount); + long rowCount = queryBuilder().sql(sql).run().recordCount(); + if (expectedRowCount != -1) { + assertEquals(expectedRowCount, rowCount); + } } /** - * Runs a SQL query and returns the results for inspection. + * Returns the fully qualified table name for Drill queries. + * + * @param tableName the Accumulo table name + * @return the fully qualified name like "accumulo.`tableName`" */ - protected List runAccumuloSQLWithResults(String sql) throws Exception { - return testSqlWithResults(sql); + protected String fullTableName(String tableName) { + return ACCUMULO_STORAGE_PLUGIN_NAME + ".`" + tableName + "`"; } /** - * Logs the results and verifies the row count. + * Returns a {@code FROM} clause that aliases the table as {@code t}. Referring to a + * qualifier inside a column family requires the table alias ({@code t.cf.name}), + * the same as for the HBase plugin. */ - private void logResultAndVerifyRowCount(List results, int expectedRowCount) - throws SchemaChangeException { - int rowCount = logResult(results); - if (expectedRowCount != -1) { - Assert.assertEquals(expectedRowCount, rowCount); - } + protected String fromTable(String tableName) { + return " FROM " + fullTableName(tableName) + " t"; } /** - * Returns the fully qualified table name for Drill queries. + * Wraps a column reference in a {@code CONVERT_FROM(..., 'UTF8')} call. Accumulo row + * keys and values are surfaced to Drill as VARBINARY, so they must be decoded before + * they can be compared against string baselines. * - * @param tableName the Accumulo table name - * @return the fully qualified name like "accumulo.`tableName`" + * @param column the column reference, e.g. {@code row_key} or {@code cf.name} + * @param alias the alias to give the decoded column */ - protected String fullTableName(String tableName) { - return ACCUMULO_STORAGE_PLUGIN_NAME + ".`" + tableName + "`"; + protected static String utf8(String column, String alias) { + return "CONVERT_FROM(" + column + ", 'UTF8') AS " + alias; + } + + /** + * Runs a query and returns its results as rows of strings, with {@code null} for + * NULL values. Reading the values back as strings keeps the assertions independent + * of whether a column comes back as required or nullable. + */ + protected List> runAndReadStrings(String sql) throws Exception { + return readStrings(queryBuilder().sql(sql).rowSetIterator()); + } + + /** + * Drains a query's row sets into rows of strings, with {@code null} for NULL values. + * + *

Every batch is read, so this stays correct for queries that return their results + * across several batches, and each row set is released as it is consumed.

+ */ + protected static List> readStrings(QueryRowSetIterator batches) { + List> rows = new ArrayList<>(); + for (DirectRowSet rowSet : batches) { + try { + int columnCount = rowSet.schema().size(); + RowSetReader reader = rowSet.reader(); + while (reader.next()) { + List row = new ArrayList<>(columnCount); + for (int i = 0; i < columnCount; i++) { + ScalarReader scalar = reader.scalar(i); + row.add(scalar.isNull() ? null : String.valueOf(scalar.getObject())); + } + rows.add(row); + } + } finally { + rowSet.clear(); + } + } + return rows; } }