Add experimental Harper-backed Fulltext runtime

Cargo.toml

@@ -19,7 +19,8 @@ required-features = ["phase0"]
 default = []
 node-api = ["dep:napi", "dep:napi-derive"]
 phase0 = ["dep:async-trait", "dep:stable_deref_trait"]
-test-panic = ["node-api", "phase0"]
+host-storage = ["node-api", "phase0"]
+test-panic = ["host-storage"]
 
 [dependencies]
 async-trait = { version = "=0.1.92", optional = true }

README.md

@@ -1,10 +1,11 @@
 # @harperfast/fulltext
 
-Native Tantivy full-text indexing for Node.js, with a native filesystem backend and a planned
-caller-owned rocksdb-js backend for Harper.
+Native Tantivy full-text indexing for Node.js, with a standalone filesystem backend and an
+experimental Harper-owned storage integration.
 
 This repository is under active development. The native entry point provides a standalone Tantivy
-index backed by `MmapDirectory`. Harper releases will use only the planned RocksDB entry point.
+index backed by `MmapDirectory`. Harper releases will use only the Harper integration backed by
+Harper's existing RocksDB lifecycle; they will not use Tantivy's filesystem storage.
 
 ## Requirements
 
@@ -67,15 +68,30 @@ handle's searches.
 
 ## Storage boundaries
 
-The package is designed around two explicit entry points:
+The package has two explicit entry points:
 
 - `@harperfast/fulltext/native` uses Tantivy's native directory implementation and has no
   rocksdb-js dependency.
-- `@harperfast/fulltext/rocks` will use a caller-owned rocksdb-js database through a versioned
-  native capability lease. It is not exported until that contract is implemented and tested.
+- `@harperfast/fulltext/harper` is an experimental integration surface that stores Tantivy objects
+  through a synchronous Harper-owned key-value view. It exists to prove and measure the real
+  derived-index path before Harper enables a customer-facing feature.
 
 There is no generic storage selector and no fallback between backends. Harper will consume only the
-Rocks entry point. The fulltext addon will not link its own copy of RocksDB.
+Harper entry point. The fulltext addon does not link RocksDB or depend on rocksdb-js; Harper owns the
+database, durability, and store lifecycle.
+
+The Harper opener requires a process-lifetime store identity, persistent generation, byte namespace,
+bounded transport limits, and a `HostStorage` implementation. `publish(payload)` commits the index
+and opaque payload into one Tantivy `meta.json` generation, then reloads the local reader before it
+resolves. Harper uses that payload for its derived-index cursor. `committedPayload` exposes the
+payload recovered at open or the newest successful publish. If a publish poisons the generation,
+its durable outcome can be ambiguous, so the getter throws until the index is reopened. Host storage
+methods are strictly synchronous; `write` and `sync` must return `undefined`, and Promise-returning
+implementations are rejected rather than acknowledged.
+
+The current Harper path is owner-worker-only. It does not yet provide non-owner read handles,
+cross-worker refresh, generation retirement, or scheduled physical reclamation. Those lifecycle
+pieces and representative performance results are required before release enablement.
 
 ## Development
 

dependencies.md

@@ -15,8 +15,8 @@ reviewed deliberately.
 | `libloading` 0.8.9         | Windows runtime                  | Resolves the Node-API type-tag check from the host, matching napi-rs's Windows strategy. |
 | `stable_deref_trait` 1.2.1 | optional Phase 0 runtime         | Lets Tantivy `OwnedBytes` retain and directly read provider-owned buffers.               |
 
-The Rust dependency graph must not include RocksDB. The future Rocks backend calls a C-ABI
-capability table owned by rocksdb-js rather than linking a second RocksDB runtime.
+The Rust dependency graph must not include RocksDB. The Harper integration reaches Harper-owned
+storage through its narrow host interface rather than linking a second RocksDB runtime.
 
 napi-rs 2.16 generates an outer unwind boundary only for exports marked `catch_unwind`. Every
 fulltext function, method, and constructor uses that option to contain argument and result
@@ -32,5 +32,6 @@ error codes and per-handle poison state for package-owned operations.
 | `prettier` 3.6.2      | development | Repository formatting checks.                                         |
 | `typescript` 5.9.3    | development | Compiles the public façade and declarations.                          |
 
-The native entry point has no production npm dependencies. rocksdb-js will be an optional peer
-dependency only when the Rocks entry point is implemented.
+Neither entry point has production npm dependencies. The Harper integration accepts a narrow host
+storage interface and does not depend on rocksdb-js; Harper supplies the implementation backed by
+its own RocksDB lifecycle.

package.json

@@ -7,6 +7,10 @@
 		"./native": {
 			"types": "./dist/native.d.ts",
 			"import": "./dist/native.js"
+		},
+		"./harper": {
+			"types": "./dist/harper.d.ts",
+			"import": "./dist/harper.js"
 		}
 	},
 	"files": [
@@ -29,8 +33,8 @@
 	},
 	"scripts": {
 		"build": "npm run build:typescript && npm run build:native",
-		"build:debug": "npm run build:typescript && napi build --platform --js false --dts ts/addon.d.ts --features node-api",
-		"build:native": "napi build --platform --js false --dts ts/addon.d.ts --release --features node-api",
+		"build:debug": "npm run build:typescript && napi build --platform --js false --dts ts/addon.d.ts --features host-storage",
+		"build:native": "napi build --platform --js false --dts ts/addon.d.ts --release --features host-storage",
 		"build:test-native": "napi build --platform --js false --dts ts/addon.d.ts --features test-panic",
 		"build:typescript": "tsc -p tsconfig.json",
 		"benchmark:native": "npm run build && node benchmarks/native.mjs",

src/engine.rs

@@ -17,6 +17,7 @@ const ID_FIELD_NAME: &str = "__fulltext_id";
 const IDENTITY_PATH: &str = ".harper-fulltext-identity";
 const META_PATH: &str = "meta.json";
 const ANALYZER_NAME: &str = "english@1";
+pub const MAX_COMMIT_PAYLOAD_BYTES: usize = 64 * 1024;
 
 #[derive(Clone)]
 pub struct Engine {
@@ -141,6 +142,10 @@ impl Engine {
 			.map_err(index_error)
 	}
 
+	pub fn committed_payload(&self) -> Result<Option<String>> {
+		Ok(self.index.load_metas().map_err(index_error)?.payload)
+	}
+
 	pub fn search(&self, searcher: &Searcher, request: &SearchRequest) -> Result<SearchResult> {
 		let selected = self.selected_fields(&request.fields)?;
 		let query = self.query(&request.text, request.operator, &selected)?;
@@ -326,7 +331,20 @@ impl Writer {
 	}
 
 	pub fn commit(&mut self) -> Result<u64> {
-		self.inner.commit().map_err(index_error)
+		self.commit_with_payload(None)
+	}
+
+	pub fn commit_with_payload(&mut self, payload: Option<&str>) -> Result<u64> {
+		if payload.is_some_and(|payload| payload.len() > MAX_COMMIT_PAYLOAD_BYTES) {
+			return Err(FulltextError::invalid(format!(
+				"commit payload exceeds {MAX_COMMIT_PAYLOAD_BYTES} UTF-8 bytes"
+			)));
+		}
+		let mut commit = self.inner.prepare_commit().map_err(index_error)?;
+		if let Some(payload) = payload {
+			commit.set_payload(payload);
+		}
+		commit.commit().map_err(index_error)
 	}
 
 	pub fn rollback(&mut self) -> Result<u64> {
@@ -429,7 +447,6 @@ mod tests {
 
 	fn config() -> EngineConfig {
 		EngineConfig {
-			path: "unused".to_owned(),
 			index_id: "products".to_owned(),
 			generation: "one".to_owned(),
 			fields: vec![
@@ -550,6 +567,37 @@ mod tests {
 		);
 	}
 
+	#[test]
+	fn commit_payload_survives_merge_and_reopen() {
+		let directory = RamDirectory::create();
+		let config = config();
+		let engine = Engine::open(directory.clone(), &config).unwrap();
+		let mut writer = engine.writer(&config).unwrap();
+		writer
+			.inner
+			.set_merge_policy(Box::new(tantivy::merge_policy::NoMergePolicy));
+		writer.apply(batch()).unwrap();
+		writer.commit_with_payload(Some("cursor-v1:42")).unwrap();
+		writer
+			.apply(MutationBatch {
+				upserts: vec![crate::protocol::Upsert {
+					id: "three".to_owned(),
+					fields: vec![("title".to_owned(), vec!["Hiking Boots".to_owned()])],
+				}],
+				deletes: Vec::new(),
+			})
+			.unwrap();
+		writer.commit_with_payload(Some("cursor-v1:43")).unwrap();
+		let segments = engine.index.searchable_segment_ids().unwrap();
+		assert_eq!(segments.len(), 2);
+		writer.inner.merge(&segments).wait().unwrap();
+		assert_eq!(engine.committed_payload().unwrap().as_deref(), Some("cursor-v1:43"));
+		writer.close().unwrap();
+
+		let reopened = Engine::open(directory, &config).unwrap();
+		assert_eq!(reopened.committed_payload().unwrap().as_deref(), Some("cursor-v1:43"));
+	}
+
 	#[test]
 	fn completes_an_interrupted_sidecar_first_create() {
 		let directory = RamDirectory::create();