diff --git a/.agents/skills/espipe/SKILL.md b/.agents/skills/espipe/SKILL.md index d49a2db..c034702 100644 --- a/.agents/skills/espipe/SKILL.md +++ b/.agents/skills/espipe/SKILL.md @@ -65,9 +65,12 @@ For Elasticsearch outputs, configuration options are available before bulk inges - `--pipeline ` installs a JSON or YAML ingest pipeline - `--pipeline-name ` overrides the pipeline name; `_none` disables a request-level default when compatible -- `--template ` installs a composable index template from JSON, JSONC, JSON5, YAML, or YML -- `--template-name ` overrides the template name -- `--template-overwrite=true|false` controls replacement of an existing template +- `--template ` installs a file-backed composable index template or selects a template compiled into espipe +- `--template _okf` selects the bundled Open Knowledge Format v0.2 mapping; its default Elasticsearch name is `open-knowledge-format` +- `--template-name ` overrides a file-derived or bundled default template name +- `--template-overwrite=true|false` controls file-template replacement and bundled-template pattern updates + +Bundled templates read their selected Elasticsearch template before ingestion. A missing template is created with the target index in `index_patterns`. An existing template gains the exact target index when absent while preserving its stored body. With `--template-overwrite=false`, a missing bundled template uses create-only semantics, an existing exact target proceeds without a write, and an existing template missing the exact target fails. Keep bundled-template preflight serial when separate processes add indices because concurrent read and update cycles are last-write-wins. Pipeline and template preflight errors abort before bulk ingestion starts. These options require an Elasticsearch output. @@ -90,6 +93,7 @@ Examples: - `espipe accounts.csv records:customers` - `espipe users.csv https://host:9200/users` - `espipe --action upsert --generate-id=true 'docs/**/*.md' env:/documents` +- `espipe 'knowledge/**/*.md' env:/knowledge --template _okf --content markdown` - `espipe --split /hits response.json output.ndjson` Use only flags the user requests or that are required to express the destination. Do not reinterpret `--action index` as an overwrite-by-source-ID option; IDs are used only when explicit or generated according to the rules above. diff --git a/CHANGELOG.md b/CHANGELOG.md index 81b1f1f..f2347d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added `.env` fallback for missing `ELASTIC_ES_URL` and `ELASTIC_ES_API_KEY` settings used by `env:/` outputs. +- Added the bundled `_okf` Elasticsearch template with explicit Open Knowledge Format v0.2 mappings, shared index-pattern maintenance, and `--template-name` overrides. ### Changed diff --git a/Cargo.lock b/Cargo.lock index 8bb76ff..495b2f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -809,6 +809,7 @@ dependencies = [ "log", "rcgen", "reqwest 0.13.4", + "rust-embed", "rustls", "serde", "serde_json", @@ -1638,6 +1639,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2244,6 +2255,41 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rust-embed" +version = "8.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e7760e252aaba7b09f4be00e36476cf585bdb68a53552ac954cdf504ab4bc9" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bcfc4d6f53af43755f7a723e4b6b8794fcce052a178dd8c6c1dadc5f5343097" +dependencies = [ + "mime_guess", + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.119", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ffa149f6aa81b58a5b3011d01a857c4ed12c7a732d2c51947a4c7c692185f0" +dependencies = [ + "sha2 0.11.0", + "walkdir", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -2969,6 +3015,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-bidi" version = "0.3.18" diff --git a/Cargo.toml b/Cargo.toml index e5c70df..e55a493 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ include = [ "CHANGELOG.md", "LICENSE.md", "README.md", + "assets/templates/*.yaml", "src/**/*.rs", "tests/**/*.rs", "tests/fixtures/anydoc/*", @@ -41,6 +42,7 @@ futures = "^0.3.34" glob = "^0.3.4" log = "^0.4.33" reqwest = { version = "^0.13.4", features = ["blocking"] } +rust-embed = "^8.7.2" serde_json = { version = "^1.0.151", features = ["arbitrary_precision", "raw_value"] } serde = { version = "^1.0.229", features = ["derive"] } yaml_serde = "^0.10.7" diff --git a/README.md b/README.md index 44047a2..6d22dde 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,22 @@ Elasticsearch requests use gzip compression by default. `espipe` retries `429 To `400 Bad Request` bulk responses are logged and counted as zero successful documents for that batch. +#### Index templates + +`--template` accepts a JSON, JSONC, JSON5, YAML, or YML composable index template file. A value beginning with `_` selects a template compiled into `espipe` instead. File-backed templates keep their own `index_patterns`; a mismatch with the output index produces a warning. + +The bundled `_okf` template maps Open Knowledge Format v0.2 metadata and the `content.body`, `content.markdown`, and `origin` fields emitted by local document ingestion. Official identifiers and categorical metadata use `keyword`, prose uses `text`, timestamps use `date`, and repeated source, verifier, and parameter records use `nested`. Automatic date detection is disabled. Unknown strings become one `keyword` field with `ignore_above: 2048`, without a `text` plus `keyword` multifield. + +```bash +espipe 'knowledge/**/*.md' env:/team-knowledge --template _okf --content markdown +``` + +`_okf` installs as `open-knowledge-format` by default. Use `--template-name team-okf` to select another cluster-side name. On each run, `espipe` reads that template. If it does not exist, `espipe` creates it with the output index in `index_patterns`. If it exists, `espipe` appends the exact output index when absent and writes the stored template body back without replacing its mappings, settings, aliases, priority, version, metadata, or component references. An existing wildcard does not suppress the exact index entry. + +`--template-overwrite=false` uses create-only semantics when the selected template is absent. It accepts an existing template only when the exact output index is already listed. Reading and writing bundled templates requires the corresponding Elasticsearch index-template privileges. + +Concurrent processes can lose one another's appended index because Elasticsearch has no atomic index-pattern append operation. Run bundled-template preflight serially when separate processes target new indices. An overridden name should identify a template dedicated to that bundled asset; selecting an unrelated template broadens its index coverage. + Local import summaries separate discovered files from documents: `Piped 5,850 of 5,850 docs from 6,246 files ...`. Skipped files count as files, not documents. ### File and stdout output diff --git a/assets/templates/_okf.yaml b/assets/templates/_okf.yaml new file mode 100644 index 0000000..db7cb28 --- /dev/null +++ b/assets/templates/_okf.yaml @@ -0,0 +1,124 @@ +index_patterns: [] +_meta: + description: Open Knowledge Format v0.2 documents + okf_version: "0.2" + espipe: + template_revision: 1 + default_template_name: open-knowledge-format +template: + mappings: + date_detection: false + dynamic_templates: + - unknown_strings: + match_mapping_type: string + mapping: + type: keyword + ignore_above: 2048 + properties: + content: + properties: + type: + type: keyword + title: + type: text + description: + type: text + resource: + type: keyword + ignore_above: 2048 + tags: + type: keyword + ignore_above: 2048 + okf_version: + type: keyword + status: + type: keyword + stale_after: + type: date + runtime: + type: keyword + ignore_above: 2048 + computation: + type: keyword + ignore_above: 2048 + body: + type: text + markdown: + type: text + sources: + type: nested + properties: + id: + type: keyword + ignore_above: 2048 + resource: + type: keyword + ignore_above: 2048 + title: + type: text + author: + type: keyword + ignore_above: 2048 + usage_count: + type: long + last_modified: + type: date + usage_window: + properties: + from: + type: date + to: + type: date + usage_window: + properties: + from: + type: date + to: + type: date + generated: + properties: + by: + type: keyword + ignore_above: 2048 + at: + type: date + verified: + type: nested + properties: + by: + type: keyword + ignore_above: 2048 + at: + type: date + parameters: + type: nested + properties: + name: + type: keyword + type: + type: keyword + required: + type: boolean + executor: + properties: + resource: + type: keyword + ignore_above: 2048 + receipt: + type: keyword + ignore_above: 2048 + attester: + properties: + resource: + type: keyword + ignore_above: 2048 + origin: + properties: + scheme: + type: keyword + path: + type: keyword + ignore_above: 2048 + filename: + type: keyword + ignore_above: 2048 diff --git a/examples/steam-games/readme.md b/examples/steam-games/readme.md index 7f19536..cd40a73 100644 --- a/examples/steam-games/readme.md +++ b/examples/steam-games/readme.md @@ -25,9 +25,9 @@ Then run from the repository root directory against a new `steam-games` index. I ```bash espipe ~/Downloads/steam-games-dataset-march-2026/games.csv \ http://localhost:9200/steam-games \ - --pipeline examples/steam-games/steam-games-pipeline.yml \ + --pipeline examples/steam-games/steam-games-pipeline.yaml \ --pipeline-name steam-games \ - --template examples/steam-games/steam-games-template.yml + --template examples/steam-games/steam-games-template.yaml ``` The pipeline splits comma-delimited `Tags` and `Screenshots` values into arrays and converts `Windows`, `Mac`, and `Linux` from title-case strings into booleans. diff --git a/examples/steam-games/steam-games-pipeline.yml b/examples/steam-games/steam-games-pipeline.yaml similarity index 100% rename from examples/steam-games/steam-games-pipeline.yml rename to examples/steam-games/steam-games-pipeline.yaml diff --git a/examples/steam-games/steam-games-template.yml b/examples/steam-games/steam-games-template.yaml similarity index 100% rename from examples/steam-games/steam-games-template.yml rename to examples/steam-games/steam-games-template.yaml diff --git a/openspec/changes/archive/2026-08-27-add-bundled-okf-template/.openspec.yaml b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/.openspec.yaml new file mode 100644 index 0000000..701445b --- /dev/null +++ b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-26 diff --git a/openspec/changes/archive/2026-08-27-add-bundled-okf-template/design.md b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/design.md new file mode 100644 index 0000000..7facb8a --- /dev/null +++ b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/design.md @@ -0,0 +1,93 @@ +## Context + +`TemplateConfig` currently stores a `PathBuf`, and preflight reads, parses, names, checks, and installs that file before input access. File-backed templates own their `index_patterns`; a mismatch only warns. The new selector must share the parsing and installation path without weakening those rules. + +Markdown frontmatter is emitted under `content`, alongside the configured body field. Local file identity is emitted under `origin`. OKF v0.2 defines core concept fields, provenance and trust structures, lifecycle fields, and Attested Computation fields, but it permits producer-defined extensions and two shapes for `verified`. + +## Goals / Non-Goals + +**Goals:** + +- Keep one parsed-template representation and one Elasticsearch installation path for embedded and file-backed sources. +- Make the template asset readable and reviewable as ordinary YAML in the repository. +- Preserve structured associations in repeated OKF objects and avoid automatic dual string mappings. +- Maintain one Elasticsearch template per selected installation name across every index loaded with the same bundled template invocation. + +**Non-Goals:** + +- Validate whether input documents conform to OKF. +- Infer that Markdown input is OKF or select `_okf` automatically. +- Rewrite file-backed template patterns. +- Support aliases for bundled selectors or remote template registries. +- Promise automatic compatibility with OKF versions after v0.2. + +## Decisions + +### Resolve `--template` into an explicit source type + +Replace the path-only internal value with a source enum containing `File(PathBuf)` and `Bundled(String)`. Classify the raw CLI value by its first character: a leading `_` means bundled, all other values mean file. This keeps the public CLI to one option and leaves relative paths such as `templates/_okf.json` unambiguous. + +Resolution returns bytes plus source metadata. Both sources then use the existing config parser and produce the same `ParsedTemplate`. File extensions continue to select JSON, JSON5, or YAML parsing. Bundled assets use their known YAML format rather than pretending to have a user path. + +The alternative was a separate `--bundled-template` option. That makes the source explicit but gives users two mutually exclusive ways to perform one operation and does not match the requested `_okf` convention. + +### Embed an assets directory with `rust-embed` + +Add `rust-embed` and derive one asset collection for `assets/templates/`. Store `_okf.yaml` there and add the asset directory to the Cargo package include list. Each asset declares its default Elasticsearch template name in `_meta.espipe.default_template_name`. The lookup layer exposes logical selector names without extensions, so `_okf` resolves `_okf.yaml` and `open-knowledge-format`, while error reporting can enumerate the embedded catalog. + +This follows `esdiag`'s asset model and, unlike `include_str!` per template, gives future bundled templates one catalog and one lookup path. Compression is unnecessary for the first small JSON asset; it can be enabled later without changing behavior. + +### Resolve a selected Elasticsearch template name + +Read the bundled asset's default installation name, then replace it when the user supplies `--template-name`. `_okf` declares `open-knowledge-format`. Future assets follow the same rule without selector-specific code. Validate the selected name with the existing template-name checks before sending a request. + +Preserve `--template-overwrite=false`: it creates a missing selected template with create-only semantics, accepts an existing template that already lists the target, and fails when adding the target would require an update. File-backed templates keep both controls unchanged. + +During bundled-template preflight, send `GET /_index_template/{selected_name}` before the template write: + +1. On `404`, parse the bundled asset, append the exact target index to its initially empty `index_patterns`, and create `{selected_name}`. Use `PUT` by default or the existing create-only request when overwrite is disabled. +2. On `200`, extract the one exact-name `index_template` body from Elasticsearch's `index_templates` response. Require an array of string `index_patterns`. +3. If the exact target index string is absent and overwrite is enabled, append it without sorting or deduplicating other entries, then `PUT` the full stored template body back to `{selected_name}`. If overwrite is disabled, fail without writing. +4. If the exact string is present, skip the `PUT`. Preflight has already proved the template exists and covers the explicit target entry. + +Exact membership, rather than wildcard matching, makes the merge deterministic and produces an audit-friendly list of indices loaded through `_okf`. Preserve the stored body rather than rebuilding it from the embedded asset. This retains mappings, settings, aliases, priority, version, `_meta`, and `composed_of`, including cluster-side edits. The trade-off is that installing a newer `espipe` does not upgrade an existing template's mappings. Mapping upgrades need their own versioned migration policy. + +Treat authentication failures, transport errors, unexpected statuses, ambiguous response entries, and invalid stored patterns as fatal preflight errors. Replacing an unreadable stored template with the bundled default could erase cluster configuration, so the safe response is to stop. + +File-backed templates do not perform a lookup or merge. Their names still come from the file stem, and mismatched patterns still warn. + +The alternative was one template per index. That avoids a read before installation but leaves many identical templates and makes a mapping revision harder to manage. Forcing bundled templates to use only their default names was also considered, but it would make future assets less reusable across teams and environments. Rebuilding the shared template from the bundled asset on every run would silently discard cluster-side edits. + +### Map OKF v0.2 according to query role + +The asset records `okf_version: "0.2"` and an integer template revision in `_meta`. Explicit mappings cover: + +- text search: `content.title`, `content.description`, `content.body`, `content.markdown`, and `content.sources.title`; +- exact filtering and identity: the remaining string-valued official fields, string arrays, and `origin` fields; +- typed values: OKF timestamps as `date`, `sources.usage_count` as `long`, and `parameters.required` as `boolean`; +- repeated structures: `sources`, `verified`, and `parameters` as `nested`, with their children explicitly mapped. + +Elasticsearch accepts a single object or an array for an object mapping, so the OKF shorthand form of `verified` remains compatible with a `nested` mapping. Ordinary objects such as `generated`, `usage_window`, `executor`, and `attester` use `object` properties. + +Set `date_detection` to `false`. A final dynamic template maps any undeclared string to one `keyword` field with a finite `ignore_above` limit. Producer extensions stay filterable without doubling every string. The explicitly mapped prose fields take precedence. Users who choose a custom body field can provide a file-backed template or add a future explicit bundled mapping; silently treating every unknown string as prose would recreate the mapping growth this change is meant to stop. + +The alternative was `dynamic: false`. That avoids mapping growth but makes producer-defined OKF metadata unqueryable. Mapping unknown strings as `text` would favor full-text search at the cost of aggregations and exact filtering, which is the less common role for frontmatter extensions. + +### Test the asset at three boundaries + +Unit tests decode the embedded asset and assert its complete field mapping, default installation name, and dynamic template. Preflight tests cover default and overridden names, missing and existing template lookups, conditional updates, exact membership, malformed responses, and unchanged file behavior. A distribution-style test runs the compiled binary with a working directory outside the repository and inspects the template requests, proving runtime lookup does not touch the asset directory. + +## Risks / Trade-offs + +- [OKF changes after v0.2] The mappings can become stale. Store the supported spec version and template revision in `_meta`, document the pin, and update the asset through a reviewed change. +- [A producer extension needs full-text search] The string fallback maps it as `keyword`. Keep known prose explicit and document file-backed templates as the escape hatch. +- [Very long URI or path values exceed the keyword limit] Values remain in `_source` but may not be indexed. Choose and test a limit suitable for expected OKF paths and URIs, and state it in the asset comments or documentation. +- [Concurrent runs can lose one appended index] Two processes may read the same pattern list and race to update it. Elasticsearch's index template API does not expose an atomic append, so document this last-write-wins limit instead of promising unsafe retry logic. +- [Existing templates do not receive mapping upgrades] Preserving the stored template protects cluster-side edits but leaves its original mapping revision in place. Handle mapping upgrades in a separate versioned migration design. +- [Template lookup needs another cluster privilege] `_okf` now reads the existing template before any write. Document the required Elasticsearch template read and management privileges and return the lookup response details on authorization failure. +- [An override can name an unrelated existing template] The merge preserves that template and appends the target index, which can broaden where its mappings apply. Treat the explicit override as user intent and document that it should name a template dedicated to the selected bundled asset. +- [Nested mappings cost more than plain objects] OKF's repeated source, verifier, and parameter records need per-entry association for correct queries. The extra hidden documents are bounded by those metadata lists. + +## Migration Plan + +This is additive. Existing file-backed commands and runs without `--template` keep their behavior. Release packaging adds the asset directory and dependency in the same change; rollback removes selector support and the embedded asset without changing any indexed document. Templates created under default or overridden names remain cluster resources and can be removed by an administrator if desired. diff --git a/openspec/changes/archive/2026-08-27-add-bundled-okf-template/proposal.md b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/proposal.md new file mode 100644 index 0000000..d1d0784 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/proposal.md @@ -0,0 +1,33 @@ +## Why + +OKF documents currently rely on Elasticsearch dynamic mapping or a user-maintained template, which produces wasteful `text` plus `keyword` mappings for metadata and leaves official OKF fields inconsistently typed. A bundled OKF template gives every `espipe` binary a versioned, ready-to-use mapping without requiring a separate template file. + +## What Changes + +- Ship a composable Elasticsearch index template for the official OKF v0.2 frontmatter fields and the document shape emitted by `espipe` Markdown ingestion. +- Give official metadata explicit field mappings, including nested provenance, trust, lifecycle, and attested-computation fields. +- Add dynamic templates that map undeclared strings once according to their role instead of creating a `text` field with a `keyword` multifield for every string. +- Extend `--template` so values beginning with `_` select a bundled template, starting with `--template _okf`; filesystem paths keep their existing behavior. +- Give each bundled template a default Elasticsearch template name. `_okf` defaults to `open-knowledge-format`. +- Allow `--template-name` to override the Elasticsearch name for `_okf` and future bundled templates. +- Before each bundled-template ingestion, read the selected Elasticsearch template name. Create it with the target index when absent, or append the target index and update the existing template when the exact index name is not already listed. +- Embed template assets into the executable with `rust-embed` so crates, release archives, and platform packages all contain the same template. + +## Capabilities + +### New Capabilities + +- `okf-index-template`: Defines the bundled OKF template, its supported OKF specification version, explicit mappings, and default dynamic mapping policy. + +### Modified Capabilities + +- `elasticsearch-index-template`: Allow bundled template selectors, default and overridden installation names, and shared pattern maintenance across target indices alongside existing file-backed templates. + +## Impact + +- Affected CLI: `--template` accepts a bundled selector such as `_okf` in addition to a path. +- Affected Elasticsearch preflight: bundled template resolution gains an embedded source, a default or overridden installation name, and a read, merge, and conditional update flow. +- Affected cluster permissions: `_okf` requires permission to read the existing composable index template as well as create or update it. +- Affected packaging: `Cargo.toml` gains `rust-embed`, template assets become package inputs, and release builds must prove the asset is available without source-tree files. +- Affected tests and docs: mapping assertions, bundled selector errors, default and overridden template names, shared-template creation and update behavior, binary embedding, CLI help, README examples, and the `espipe` skill. +- External contract: mappings track the official Open Knowledge Format v0.2 specification; future OKF revisions require an intentional template update. diff --git a/openspec/changes/archive/2026-08-27-add-bundled-okf-template/specs/elasticsearch-index-template/spec.md b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/specs/elasticsearch-index-template/spec.md new file mode 100644 index 0000000..8662b2f --- /dev/null +++ b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/specs/elasticsearch-index-template/spec.md @@ -0,0 +1,105 @@ +## ADDED Requirements + +### Requirement: Underscore template values select bundled templates +The system SHALL interpret a `--template` value beginning with `_` as a bundled template selector and SHALL continue to interpret every other value as a filesystem path. + +#### Scenario: OKF bundled template is selected +- **WHEN** the user passes `--template _okf` with an Elasticsearch output +- **THEN** the system resolves the bundled `_okf` template without reading a template file from the filesystem +- **AND** template preflight completes before any input content is opened or any bulk request is sent + +#### Scenario: File-backed template behavior is preserved +- **WHEN** the user passes `--template templates/_okf.json` +- **THEN** the system treats the value as a filesystem path because the complete value does not begin with `_` +- **AND** it preserves existing file parsing, naming, index-pattern warning, and installation behavior + +#### Scenario: Unknown bundled template is rejected +- **WHEN** the user passes a bundled selector that the executable does not contain +- **THEN** startup fails before input access and bulk ingestion +- **AND** the error identifies the unknown selector and lists the available bundled template names + +### Requirement: Bundled templates have default Elasticsearch names +The system SHALL associate every bundled template selector with a default Elasticsearch template name and SHALL use `--template-name` as an override. `_okf` SHALL default to `open-knowledge-format`. Before installing or updating a bundled template, the system SHALL request the selected name from Elasticsearch. + +#### Scenario: OKF uses its default name +- **WHEN** the user passes `--template _okf` and the output target index is `team-knowledge` +- **AND** Elasticsearch reports that `open-knowledge-format` does not exist +- **THEN** the system adds `team-knowledge` to the bundled template's `index_patterns` +- **AND** it creates the template as `open-knowledge-format` +- **AND** it creates no target-specific template + +#### Scenario: Bundled template name is overridden +- **WHEN** the user passes `--template _okf --template-name team-okf` +- **THEN** the system requests, creates, or updates `team-okf` +- **AND** it does not request or write `open-knowledge-format` + +#### Scenario: Future bundled template name is overridden +- **WHEN** the executable contains a bundled selector `_catalog` +- **AND** the user passes `--template _catalog --template-name company-catalog` +- **THEN** the system requests, creates, or updates `company-catalog` +- **AND** the selected `_catalog` asset supplies the initial template body when `company-catalog` is absent + +#### Scenario: Selected template lookup fails +- **WHEN** the request for the default or overridden template name fails because of authentication, TLS, transport, timeout, or an unexpected Elasticsearch response +- **THEN** startup fails before input access and bulk ingestion +- **AND** the error identifies the template lookup failure + +### Requirement: Existing bundled template gains new target indices +When the selected default or overridden Elasticsearch template exists, the system SHALL read its stored composable template body and SHALL append the exact output index name to its `index_patterns` only when that exact value is absent. The system SHALL preserve the existing pattern order and all other stored template fields. + +#### Scenario: New target index is appended +- **WHEN** the user passes `--template _okf` for target index `team-knowledge` +- **AND** `open-knowledge-format` exists with `index_patterns` equal to `["company-knowledge"]` +- **THEN** the system updates the same template with `index_patterns` equal to `["company-knowledge", "team-knowledge"]` +- **AND** it preserves the template's existing mappings, settings, aliases, priority, version, metadata, and composed component references +- **AND** it sends no bulk request until Elasticsearch accepts the update + +#### Scenario: Target index is already listed +- **WHEN** the user passes `--template _okf` for target index `team-knowledge` +- **AND** `open-knowledge-format` already lists the exact `team-knowledge` value in `index_patterns` +- **THEN** the system does not append a duplicate value +- **AND** it does not send a template update request +- **AND** bulk ingestion may proceed after preflight + +#### Scenario: Existing patterns contain a wildcard match but not the exact index +- **WHEN** the target index is `team-knowledge` +- **AND** the existing `index_patterns` contains `team-*` but does not contain the exact value `team-knowledge` +- **THEN** the system appends `team-knowledge` + +#### Scenario: Existing selected template cannot be merged +- **WHEN** the selected Elasticsearch template exists but its response lacks one unambiguous composable template body or has an invalid `index_patterns` value +- **THEN** startup fails before input access and bulk ingestion +- **AND** the system does not replace the malformed stored template with the bundled asset + +#### Scenario: File template pattern remains user-owned +- **WHEN** the user selects a file-backed template whose patterns do not match the target index +- **THEN** the system does not add the target index to that template +- **AND** it preserves the existing mismatch warning behavior + +### Requirement: Overwrite control applies to the selected bundled template name +The existing `--template-overwrite` control SHALL determine whether `espipe` may create or update the default or overridden Elasticsearch template selected for a bundled asset. + +#### Scenario: Create-only mode creates a missing shared template +- **WHEN** the user passes `--template _okf --template-overwrite=false` +- **AND** the selected default or overridden Elasticsearch template does not exist +- **THEN** the system creates the selected template with Elasticsearch create-only semantics + +#### Scenario: Create-only mode cannot append a missing target +- **WHEN** the user passes `--template _okf --template-overwrite=false` +- **AND** the selected default or overridden Elasticsearch template exists without the exact target index in `index_patterns` +- **THEN** startup fails before input access and bulk ingestion +- **AND** the system does not update the template +- **AND** the error explains that appending the target requires template overwrite behavior + +#### Scenario: Create-only mode accepts an existing target +- **WHEN** the user passes `--template _okf --template-overwrite=false` +- **AND** the selected default or overridden Elasticsearch template already lists the exact target index +- **THEN** the system sends no template write request +- **AND** bulk ingestion may proceed after preflight + +### Requirement: Bundled templates are present in distributed executables +The system SHALL include every bundled template asset in the compiled executable and SHALL resolve it at runtime without a source checkout, working-directory asset folder, or network access. + +#### Scenario: Bundled template works outside the source tree +- **WHEN** a packaged `espipe` executable runs from a directory that contains no template assets +- **THEN** `--template _okf` resolves and installs the bundled template diff --git a/openspec/changes/archive/2026-08-27-add-bundled-okf-template/specs/okf-index-template/spec.md b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/specs/okf-index-template/spec.md new file mode 100644 index 0000000..44c64ac --- /dev/null +++ b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/specs/okf-index-template/spec.md @@ -0,0 +1,70 @@ +## Purpose + +Define the Elasticsearch mappings and dynamic mapping policy supplied by the bundled template for OKF documents ingested through `espipe`. + +## ADDED Requirements + +### Requirement: Bundled OKF template targets the current supported specification +The system SHALL bundle an Elasticsearch composable index template named `_okf` whose metadata identifies Open Knowledge Format v0.2 and a template revision controlled by `espipe`. + +#### Scenario: Bundled template version is inspectable +- **WHEN** the `_okf` asset is decoded as a composable index template +- **THEN** its `_meta` identifies OKF specification version `0.2` +- **AND** its `_meta` contains an `espipe` template revision + +### Requirement: Official OKF scalar and list metadata fields have explicit mappings +The `_okf` template SHALL explicitly map `content.type`, `content.title`, `content.description`, `content.resource`, `content.tags`, `content.okf_version`, `content.status`, `content.stale_after`, `content.runtime`, and `content.computation`. Categorical values, identifiers, paths, URIs, and tags SHALL be `keyword`; prose SHALL be `text`; and lifecycle instants SHALL be `date`. + +#### Scenario: Core concept metadata mappings are installed +- **WHEN** Elasticsearch installs a materialized `_okf` template +- **THEN** `content.type`, `content.resource`, `content.tags`, `content.okf_version`, `content.status`, `content.runtime`, and `content.computation` are mapped as `keyword` +- **AND** `content.title` and `content.description` are mapped as `text` +- **AND** `content.stale_after` is mapped as `date` + +#### Scenario: Markdown content is searchable +- **WHEN** `espipe` emits OKF document content in `content.body` or `content.markdown` +- **THEN** the template maps both fields as `text` + +### Requirement: Official OKF structured metadata fields have explicit mappings +The `_okf` template SHALL explicitly map the complete v0.2 structures for `content.sources`, `content.usage_window`, `content.generated`, `content.verified`, `content.parameters`, `content.executor`, and `content.attester`, including every child field defined by the specification. + +#### Scenario: Provenance fields preserve source associations +- **WHEN** an OKF concept contains `sources` +- **THEN** `content.sources` is mapped as `nested` +- **AND** each source's `id`, `resource`, and `author` are `keyword` +- **AND** each source's `title` is `text` +- **AND** each source's `usage_count` is `long` +- **AND** each source's `last_modified`, `usage_window.from`, and `usage_window.to` are `date` +- **AND** top-level `content.usage_window.from` and `content.usage_window.to` are `date` + +#### Scenario: Trust and lifecycle structures are mapped +- **WHEN** an OKF concept contains generation or verification metadata +- **THEN** `content.generated.by` is `keyword` and `content.generated.at` is `date` +- **AND** `content.verified` is `nested` with `by` as `keyword` and `at` as `date` +- **AND** Elasticsearch accepts either the specification's single-object or list representation of `verified` + +#### Scenario: Attested computation structures are mapped +- **WHEN** an OKF Attested Computation contains contract metadata +- **THEN** `content.parameters` is `nested` with `name` and `type` as `keyword` and `required` as `boolean` +- **AND** `content.executor.resource` and every `content.executor.receipt` value are `keyword` +- **AND** `content.attester.resource` is `keyword` + +### Requirement: Unknown strings do not receive automatic text and keyword multifields +The `_okf` template SHALL disable automatic date detection and SHALL use a final string dynamic template that maps undeclared string fields once as `keyword`, without an automatically generated `text` or `keyword` multifield. + +#### Scenario: Producer extension string is mapped once +- **WHEN** an OKF concept contains an undeclared producer field such as `content.owner` +- **THEN** Elasticsearch dynamically maps `content.owner` as `keyword` +- **AND** the mapping does not add a `text` representation or a multifield + +#### Scenario: Date-like extension remains a string +- **WHEN** an undeclared producer string resembles a date +- **THEN** automatic date detection does not map it as `date` +- **AND** the string dynamic template maps it as `keyword` + +### Requirement: Espipe origin metadata has stable mappings +The `_okf` template SHALL explicitly map `origin.scheme`, `origin.path`, and `origin.filename` as `keyword` so file identity fields remain filterable and do not receive text multifields. + +#### Scenario: Local file origin is mapped for filtering +- **WHEN** `espipe` indexes an OKF document with local-file origin metadata +- **THEN** `origin.scheme`, `origin.path`, and `origin.filename` are mapped as `keyword` diff --git a/openspec/changes/archive/2026-08-27-add-bundled-okf-template/tasks.md b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/tasks.md new file mode 100644 index 0000000..be1597e --- /dev/null +++ b/openspec/changes/archive/2026-08-27-add-bundled-okf-template/tasks.md @@ -0,0 +1,39 @@ +## 1. Embedded template catalog + +- [x] 1.1 Add `rust-embed` to `Cargo.toml` and include the template asset directory in the published crate file list. +- [x] 1.2 Add an embedded-template module that resolves logical selector names, reads each asset's default Elasticsearch template name, and enumerates available bundled templates. +- [x] 1.3 Add `assets/templates/_okf.yaml` with OKF v0.2 metadata, an `espipe` revision, and `open-knowledge-format` as its default installation name. +- [x] 1.4 Add unit coverage proving `_okf` is available from the compiled asset catalog and unknown selectors report the available names. + +## 2. OKF mapping contract + +- [x] 2.1 Explicitly map core OKF metadata, searchable body fields, and `origin` identity fields with their specified Elasticsearch types. +- [x] 2.2 Explicitly map all provenance, trust, lifecycle, and Attested Computation structures, including nested source, verifier, and parameter records. +- [x] 2.3 Disable automatic date detection and add a final dynamic template that maps undeclared strings once as bounded `keyword` fields. +- [x] 2.4 Add mapping tests that account for every official OKF v0.2 field and reject automatic `text` plus `keyword` mappings for extension strings. +- [x] 2.5 Add a representative OKF document fixture covering single-object `verified`, list `verified`, provenance usage windows, lifecycle metadata, and attested computation fields. + +## 3. Template source resolution + +- [x] 3.1 Refactor template configuration to distinguish file-backed paths from leading-underscore bundled selectors while preserving existing CLI validation timing. +- [x] 3.2 Parse bundled YAML through the shared parsed-template representation without filesystem access. +- [x] 3.3 Preserve all existing JSON, JSONC, JSON5, YAML, naming, pipeline compatibility, and index-pattern warning behavior for file-backed templates. +- [x] 3.4 Add CLI and preflight tests for `_okf`, unknown bundled selectors, underscore-containing file paths, and unchanged non-Elasticsearch rejection. + +## 4. Shared template maintenance + +- [x] 4.1 Resolve the selected Elasticsearch template name from the bundled asset default or `--template-name`, with `_okf` defaulting to `open-knowledge-format`. +- [x] 4.2 Add exact-name template lookup for the selected name and parse the Elasticsearch `index_templates` response into one stored composable template body. +- [x] 4.3 When the selected template is absent, append the target index to the bundled asset and create it under the default or overridden name before bulk ingestion. +- [x] 4.4 When the shared template exists and lacks the exact target index, append it to the stored `index_patterns` and update the same template without changing other fields. +- [x] 4.5 Skip the update when the exact target index is already listed, while preserving file-backed template behavior. +- [x] 4.6 Add request-capture tests for lookup failures, default and overridden names, initial creation, append updates, exact-value no-ops, wildcard-only patterns, malformed stored templates, create-only branches, and preserved stored fields. +- [x] 4.7 Document the last-write-wins limitation when concurrent processes update the shared template for different targets. +- [x] 4.8 Add a binary-level test that runs outside the source tree and proves `_okf` resolves without local assets or network lookup. + +## 5. Documentation and verification + +- [x] 5.1 Update CLI help and README template documentation with bundled selector syntax, default and overridden template names, merge behavior, the OKF v0.2 version pin, mapping defaults, and an OKF ingestion example. +- [x] 5.2 Update the repository `espipe` skill so ingestion requests can select `--template _okf` accurately. +- [x] 5.3 Add an Unreleased changelog entry for bundled OKF template support. +- [x] 5.4 Run formatting, unit tests, index-template integration tests, strict OpenSpec validation, and package-content verification. diff --git a/openspec/specs/elasticsearch-index-template/spec.md b/openspec/specs/elasticsearch-index-template/spec.md index 3bf5f0b..c5e4f3f 100644 --- a/openspec/specs/elasticsearch-index-template/spec.md +++ b/openspec/specs/elasticsearch-index-template/spec.md @@ -210,3 +210,107 @@ The system SHALL preserve existing output behavior when `--template` is not prov - **WHEN** the user runs `espipe` with an Elasticsearch output and no `--template` - **THEN** the system does not send an index template request - **AND** document bulk indexing starts using the existing output flow + +### Requirement: Underscore template values select bundled templates +The system SHALL interpret a `--template` value beginning with `_` as a bundled template selector and SHALL continue to interpret every other value as a filesystem path. + +#### Scenario: OKF bundled template is selected +- **WHEN** the user passes `--template _okf` with an Elasticsearch output +- **THEN** the system resolves the bundled `_okf` template without reading a template file from the filesystem +- **AND** template preflight completes before any input content is opened or any bulk request is sent + +#### Scenario: File-backed template behavior is preserved +- **WHEN** the user passes `--template templates/_okf.json` +- **THEN** the system treats the value as a filesystem path because the complete value does not begin with `_` +- **AND** it preserves existing file parsing, naming, index-pattern warning, and installation behavior + +#### Scenario: Unknown bundled template is rejected +- **WHEN** the user passes a bundled selector that the executable does not contain +- **THEN** startup fails before input access and bulk ingestion +- **AND** the error identifies the unknown selector and lists the available bundled template names + +### Requirement: Bundled templates have default Elasticsearch names +The system SHALL associate every bundled template selector with a default Elasticsearch template name and SHALL use `--template-name` as an override. `_okf` SHALL default to `open-knowledge-format`. Before installing or updating a bundled template, the system SHALL request the selected name from Elasticsearch. + +#### Scenario: OKF uses its default name +- **WHEN** the user passes `--template _okf` and the output target index is `team-knowledge` +- **AND** Elasticsearch reports that `open-knowledge-format` does not exist +- **THEN** the system adds `team-knowledge` to the bundled template's `index_patterns` +- **AND** it creates the template as `open-knowledge-format` +- **AND** it creates no target-specific template + +#### Scenario: Bundled template name is overridden +- **WHEN** the user passes `--template _okf --template-name team-okf` +- **THEN** the system requests, creates, or updates `team-okf` +- **AND** it does not request or write `open-knowledge-format` + +#### Scenario: Future bundled template name is overridden +- **WHEN** the executable contains a bundled selector `_catalog` +- **AND** the user passes `--template _catalog --template-name company-catalog` +- **THEN** the system requests, creates, or updates `company-catalog` +- **AND** the selected `_catalog` asset supplies the initial template body when `company-catalog` is absent + +#### Scenario: Selected template lookup fails +- **WHEN** the request for the default or overridden template name fails because of authentication, TLS, transport, timeout, or an unexpected Elasticsearch response +- **THEN** startup fails before input access and bulk ingestion +- **AND** the error identifies the template lookup failure + +### Requirement: Existing bundled template gains new target indices +When the selected default or overridden Elasticsearch template exists, the system SHALL read its stored composable template body and SHALL append the exact output index name to its `index_patterns` only when that exact value is absent. The system SHALL preserve the existing pattern order and all other stored template fields. + +#### Scenario: New target index is appended +- **WHEN** the user passes `--template _okf` for target index `team-knowledge` +- **AND** `open-knowledge-format` exists with `index_patterns` equal to `["company-knowledge"]` +- **THEN** the system updates the same template with `index_patterns` equal to `["company-knowledge", "team-knowledge"]` +- **AND** it preserves the template's existing mappings, settings, aliases, priority, version, metadata, and composed component references +- **AND** it sends no bulk request until Elasticsearch accepts the update + +#### Scenario: Target index is already listed +- **WHEN** the user passes `--template _okf` for target index `team-knowledge` +- **AND** `open-knowledge-format` already lists the exact `team-knowledge` value in `index_patterns` +- **THEN** the system does not append a duplicate value +- **AND** it does not send a template update request +- **AND** bulk ingestion may proceed after preflight + +#### Scenario: Existing patterns contain a wildcard match but not the exact index +- **WHEN** the target index is `team-knowledge` +- **AND** the existing `index_patterns` contains `team-*` but does not contain the exact value `team-knowledge` +- **THEN** the system appends `team-knowledge` + +#### Scenario: Existing selected template cannot be merged +- **WHEN** the selected Elasticsearch template exists but its response lacks one unambiguous composable template body or has an invalid `index_patterns` value +- **THEN** startup fails before input access and bulk ingestion +- **AND** the system does not replace the malformed stored template with the bundled asset + +#### Scenario: File template pattern remains user-owned +- **WHEN** the user selects a file-backed template whose patterns do not match the target index +- **THEN** the system does not add the target index to that template +- **AND** it preserves the existing mismatch warning behavior + +### Requirement: Overwrite control applies to the selected bundled template name +The existing `--template-overwrite` control SHALL determine whether `espipe` may create or update the default or overridden Elasticsearch template selected for a bundled asset. + +#### Scenario: Create-only mode creates a missing shared template +- **WHEN** the user passes `--template _okf --template-overwrite=false` +- **AND** the selected default or overridden Elasticsearch template does not exist +- **THEN** the system creates the selected template with Elasticsearch create-only semantics + +#### Scenario: Create-only mode cannot append a missing target +- **WHEN** the user passes `--template _okf --template-overwrite=false` +- **AND** the selected default or overridden Elasticsearch template exists without the exact target index in `index_patterns` +- **THEN** startup fails before input access and bulk ingestion +- **AND** the system does not update the template +- **AND** the error explains that appending the target requires template overwrite behavior + +#### Scenario: Create-only mode accepts an existing target +- **WHEN** the user passes `--template _okf --template-overwrite=false` +- **AND** the selected default or overridden Elasticsearch template already lists the exact target index +- **THEN** the system sends no template write request +- **AND** bulk ingestion may proceed after preflight + +### Requirement: Bundled templates are present in distributed executables +The system SHALL include every bundled template asset in the compiled executable and SHALL resolve it at runtime without a source checkout, working-directory asset folder, or network access. + +#### Scenario: Bundled template works outside the source tree +- **WHEN** a packaged `espipe` executable runs from a directory that contains no template assets +- **THEN** `--template _okf` resolves and installs the bundled template diff --git a/openspec/specs/okf-index-template/spec.md b/openspec/specs/okf-index-template/spec.md new file mode 100644 index 0000000..c535273 --- /dev/null +++ b/openspec/specs/okf-index-template/spec.md @@ -0,0 +1,70 @@ +## Purpose + +Define the Elasticsearch mappings and dynamic mapping policy supplied by the bundled template for OKF documents ingested through `espipe`. + +## Requirements + +### Requirement: Bundled OKF template targets the current supported specification +The system SHALL bundle an Elasticsearch composable index template named `_okf` whose metadata identifies Open Knowledge Format v0.2 and a template revision controlled by `espipe`. + +#### Scenario: Bundled template version is inspectable +- **WHEN** the `_okf` asset is decoded as a composable index template +- **THEN** its `_meta` identifies OKF specification version `0.2` +- **AND** its `_meta` contains an `espipe` template revision + +### Requirement: Official OKF scalar and list metadata fields have explicit mappings +The `_okf` template SHALL explicitly map `content.type`, `content.title`, `content.description`, `content.resource`, `content.tags`, `content.okf_version`, `content.status`, `content.stale_after`, `content.runtime`, and `content.computation`. Categorical values, identifiers, paths, URIs, and tags SHALL be `keyword`; prose SHALL be `text`; and lifecycle instants SHALL be `date`. + +#### Scenario: Core concept metadata mappings are installed +- **WHEN** Elasticsearch installs a materialized `_okf` template +- **THEN** `content.type`, `content.resource`, `content.tags`, `content.okf_version`, `content.status`, `content.runtime`, and `content.computation` are mapped as `keyword` +- **AND** `content.title` and `content.description` are mapped as `text` +- **AND** `content.stale_after` is mapped as `date` + +#### Scenario: Markdown content is searchable +- **WHEN** `espipe` emits OKF document content in `content.body` or `content.markdown` +- **THEN** the template maps both fields as `text` + +### Requirement: Official OKF structured metadata fields have explicit mappings +The `_okf` template SHALL explicitly map the complete v0.2 structures for `content.sources`, `content.usage_window`, `content.generated`, `content.verified`, `content.parameters`, `content.executor`, and `content.attester`, including every child field defined by the specification. + +#### Scenario: Provenance fields preserve source associations +- **WHEN** an OKF concept contains `sources` +- **THEN** `content.sources` is mapped as `nested` +- **AND** each source's `id`, `resource`, and `author` are `keyword` +- **AND** each source's `title` is `text` +- **AND** each source's `usage_count` is `long` +- **AND** each source's `last_modified`, `usage_window.from`, and `usage_window.to` are `date` +- **AND** top-level `content.usage_window.from` and `content.usage_window.to` are `date` + +#### Scenario: Trust and lifecycle structures are mapped +- **WHEN** an OKF concept contains generation or verification metadata +- **THEN** `content.generated.by` is `keyword` and `content.generated.at` is `date` +- **AND** `content.verified` is `nested` with `by` as `keyword` and `at` as `date` +- **AND** Elasticsearch accepts either the specification's single-object or list representation of `verified` + +#### Scenario: Attested computation structures are mapped +- **WHEN** an OKF Attested Computation contains contract metadata +- **THEN** `content.parameters` is `nested` with `name` and `type` as `keyword` and `required` as `boolean` +- **AND** `content.executor.resource` and every `content.executor.receipt` value are `keyword` +- **AND** `content.attester.resource` is `keyword` + +### Requirement: Unknown strings do not receive automatic text and keyword multifields +The `_okf` template SHALL disable automatic date detection and SHALL use a final string dynamic template that maps undeclared string fields once as `keyword`, without an automatically generated `text` or `keyword` multifield. + +#### Scenario: Producer extension string is mapped once +- **WHEN** an OKF concept contains an undeclared producer field such as `content.owner` +- **THEN** Elasticsearch dynamically maps `content.owner` as `keyword` +- **AND** the mapping does not add a `text` representation or a multifield + +#### Scenario: Date-like extension remains a string +- **WHEN** an undeclared producer string resembles a date +- **THEN** automatic date detection does not map it as `date` +- **AND** the string dynamic template maps it as `keyword` + +### Requirement: Espipe origin metadata has stable mappings +The `_okf` template SHALL explicitly map `origin.scheme`, `origin.path`, and `origin.filename` as `keyword` so file identity fields remain filterable and do not receive text multifields. + +#### Scenario: Local file origin is mapped for filtering +- **WHEN** `espipe` indexes an OKF document with local-file origin metadata +- **THEN** `origin.scheme`, `origin.path`, and `origin.filename` are mapped as `keyword` diff --git a/src/input.rs b/src/input.rs index 01013c8..7413b3f 100644 --- a/src/input.rs +++ b/src/input.rs @@ -373,6 +373,26 @@ enum InputKind { } impl Input { + pub(crate) fn discover_is_multi_source_local( + uris: &[UriRef], + discovery_options: DiscoveryOptions, + ) -> Result { + if uris.is_empty() + || uris.iter().any(|uri| { + !matches!( + uri.scheme().map(|scheme| scheme.as_str()), + Some("file") | None + ) || uri.path().as_str() == "-" + }) + { + return Ok(false); + } + + let (paths, _) = + resolve_file_document_paths_with_options(uris.to_vec(), discovery_options)?; + Ok(paths.len() > 1) + } + pub async fn try_new( uris: Vec>, content_field: String, @@ -2567,6 +2587,28 @@ mod tests { encoder.finish().unwrap(); } + #[test] + fn multi_source_discovery_counts_paths_without_constructing_input() { + let directory = workspace_tempdir(); + let first = directory.path().join("first.ndjson"); + let second = directory.path().join("second.ndjson"); + fs::write(&first, "{\"value\":1}\n").unwrap(); + fs::write(&second, "{\"value\":2}\n").unwrap(); + let first = UriRef::parse(first.display().to_string()).unwrap(); + let second = UriRef::parse(second.display().to_string()).unwrap(); + + assert!( + Input::discover_is_multi_source_local( + &[first.clone(), second], + DiscoveryOptions::default() + ) + .unwrap() + ); + assert!( + !Input::discover_is_multi_source_local(&[first], DiscoveryOptions::default()).unwrap() + ); + } + #[test] fn input_kind_detects_supported_compressed_suffixes() { assert_eq!( diff --git a/src/main.rs b/src/main.rs index f079292..fedde84 100644 --- a/src/main.rs +++ b/src/main.rs @@ -132,17 +132,23 @@ struct Cli { /// Elasticsearch ingest pipeline name override #[arg(help = "Elasticsearch ingest pipeline name", long)] pipeline_name: Option, - /// Composable index template file to install before Elasticsearch bulk ingestion + /// Composable index template file or bundled selector to install before ingestion #[arg( - help = "Composable index template file for Elasticsearch outputs; .json, .jsonc, .json5, .yml, and .yaml are detected by extension, and other extensions are parsed as strict JSON", + help = "Composable index template file or bundled selector such as _okf for Elasticsearch outputs; file extensions .json, .jsonc, .json5, .yml, and .yaml are detected, and other files are parsed as strict JSON", long )] template: Option, - /// Override the template name; defaults to the template file name without its final extension - #[arg(help = "Composable index template name override", long)] + /// Override the file-derived or bundled default template name + #[arg( + help = "Composable index template name override; bundled templates otherwise use their embedded default name", + long + )] template_name: Option, - /// Overwrite an existing composable index template - #[arg(help = "Overwrite an existing composable index template", long)] + /// Allow replacement or bundled index-pattern updates + #[arg( + help = "Allow replacement of a file template or index-pattern updates to a bundled template", + long + )] template_overwrite: Option, } @@ -222,7 +228,21 @@ async fn main() -> ExitCode { } let discovery_options = DiscoveryOptions { symlinks, hidden }; - let discover_before_output = should_discover_input_before_output(batch_size, &inputs); + let bundled_template = preflight + .template + .as_ref() + .and_then(|path| path.to_str()) + .is_some_and(|value| value.starts_with('_')); + let multi_source_local = if bundled_template && batch_size.is_none() { + match Input::discover_is_multi_source_local(&inputs, discovery_options) { + Ok(multi_source_local) => multi_source_local, + Err(err) => return exit_with_error(err), + } + } else { + false + }; + let discover_before_output = + should_discover_input_before_output(batch_size, &inputs, bundled_template); let (mut input, mut output) = if discover_before_output { let input = match Input::try_new(inputs, content, split, generate_id, discovery_options).await { @@ -255,7 +275,7 @@ async fn main() -> ExitCode { log::debug!("output: {output}"); (input, output) } else { - let batch_size = effective_batch_size(batch_size, false); + let batch_size = effective_batch_size(batch_size, multi_source_local); let elasticsearch_config = match ElasticsearchOutputConfig::try_new(batch_size, max_requests) { Ok(config) => config, @@ -417,8 +437,11 @@ fn effective_batch_size(explicit: Option, multi_source_local: bool) -> us fn should_discover_input_before_output( explicit_batch_size: Option, inputs: &[UriRef], + bundled_template: bool, ) -> bool { - inputs.len() > 1 || (explicit_batch_size.is_none() && inputs.iter().all(is_local_file_input)) + !bundled_template + && (inputs.len() > 1 + || (explicit_batch_size.is_none() && inputs.iter().all(is_local_file_input))) } fn load_dotenv() -> eyre::Result<()> { @@ -511,16 +534,27 @@ mod tests { let second_remote = UriRef::parse("https://example.com/more.ndjson".to_string()).unwrap(); let stdin = UriRef::parse("-".to_string()).unwrap(); - assert!(should_discover_input_before_output(None, &[local.clone()])); - assert!(!should_discover_input_before_output(Some(750), &[local])); + assert!(should_discover_input_before_output( + None, + &[local.clone()], + false + )); + assert!(!should_discover_input_before_output( + Some(750), + &[local.clone()], + false + )); + assert!(!should_discover_input_before_output(None, &[local], true)); assert!(!should_discover_input_before_output( None, - &[remote.clone()] + &[remote.clone()], + false )); assert!(should_discover_input_before_output( Some(750), - &[remote, second_remote] + &[remote, second_remote], + false )); - assert!(!should_discover_input_before_output(None, &[stdin])); + assert!(!should_discover_input_before_output(None, &[stdin], false)); } } diff --git a/src/output/elasticsearch.rs b/src/output/elasticsearch.rs index ee618aa..e119971 100644 --- a/src/output/elasticsearch.rs +++ b/src/output/elasticsearch.rs @@ -1,4 +1,5 @@ mod bulk_response; +mod embedded_templates; use super::{BulkAction, Sender}; use crate::input::InputDocument; @@ -34,11 +35,17 @@ pub struct ElasticsearchOutputConfig { #[derive(Clone, Debug)] pub struct TemplateConfig { - path: PathBuf, + source: TemplateSource, name: Option, overwrite: bool, } +#[derive(Clone, Debug)] +enum TemplateSource { + File(PathBuf), + Bundled(String), +} + impl TemplateConfig { pub fn try_new( path: Option, @@ -55,14 +62,29 @@ impl TemplateConfig { return Ok(None); } + let path = path.expect("checked above"); + let source = match path.to_str() { + Some(selector) if selector.starts_with('_') => { + TemplateSource::Bundled(selector.to_string()) + } + _ => TemplateSource::File(path), + }; + Ok(Some(Self { - path: path.expect("checked above"), + source, name, overwrite: overwrite.unwrap_or(true), })) } } +pub(super) fn validate_bundled_template(path: &Path) -> Result<()> { + if let Some(selector) = path.to_str().filter(|value| value.starts_with('_')) { + embedded_templates::resolve(selector)?; + } + Ok(()) +} + impl ElasticsearchOutputConfig { pub const DEFAULT_BATCH_SIZE: usize = DEFAULT_BATCH_SIZE; pub const MULTI_SOURCE_DEFAULT_BATCH_SIZE: usize = MULTI_SOURCE_DEFAULT_BATCH_SIZE; @@ -148,6 +170,7 @@ struct ParsedTemplate { name: String, overwrite: bool, body: Value, + bundled: bool, } async fn install_template( @@ -155,8 +178,135 @@ async fn install_template( target_index: &str, parsed: &ParsedTemplate, ) -> Result<()> { + if parsed.bundled { + return install_bundled_template(client, target_index, parsed).await; + } + warn_for_index_patterns(&parsed.body, target_index); + write_template(client, parsed, &parsed.body).await +} + +async fn install_bundled_template( + client: &Elasticsearch, + target_index: &str, + parsed: &ParsedTemplate, +) -> Result<()> { + let path = format!("/_index_template/{}", parsed.name); + let response = client + .send( + Method::Get, + &path, + HeaderMap::new(), + Option::<&()>::None, + Option::>::None, + None, + ) + .await + .map_err(|err| eyre!("failed to look up index template '{}': {err}", parsed.name))?; + + match response.status_code() { + StatusCode::NOT_FOUND => { + let mut body = parsed.body.clone(); + append_exact_index(&mut body, target_index).map_err(|err| { + eyre!( + "bundled template '{}' cannot be installed: {err}", + parsed.name + ) + })?; + write_template(client, parsed, &body).await + } + status if status.is_success() => { + let response_body = response.json::().await.map_err(|err| { + eyre!( + "failed to parse index template lookup response for '{}': {err}", + parsed.name + ) + })?; + let mut stored = extract_stored_template(&response_body, &parsed.name)?; + if !append_exact_index(&mut stored, target_index)? { + return Ok(()); + } + if !parsed.overwrite { + return Err(eyre!( + "index template '{}' does not list target index '{target_index}'; appending it requires --template-overwrite=true", + parsed.name + )); + } + write_template(client, parsed, &stored).await + } + status => { + let details = response + .text() + .await + .unwrap_or_else(|err| format!("failed to read error body: {err}")); + Err(eyre!( + "failed to look up index template '{}': status {status}: {details}", + parsed.name + )) + } + } +} + +fn extract_stored_template(response: &Value, selected_name: &str) -> Result { + let entries = response + .get("index_templates") + .and_then(Value::as_array) + .ok_or_else(|| { + eyre!("index template lookup for '{selected_name}' has no index_templates array") + })?; + let matches = entries + .iter() + .filter(|entry| entry.get("name").and_then(Value::as_str) == Some(selected_name)) + .collect::>(); + if matches.len() != 1 { + return Err(eyre!( + "index template lookup for '{selected_name}' returned {} exact matches; expected one", + matches.len() + )); + } + let body = matches[0] + .get("index_template") + .filter(|body| body.is_object()) + .ok_or_else(|| { + eyre!("index template lookup for '{selected_name}' has no composable template body") + })? + .clone(); + validate_index_patterns_array(&body)?; + Ok(body) +} + +fn validate_index_patterns_array(template: &Value) -> Result<()> { + let patterns = template + .get("index_patterns") + .and_then(Value::as_array) + .ok_or_else(|| eyre!("index_patterns must be an array of strings"))?; + if patterns.iter().any(|pattern| !pattern.is_string()) { + return Err(eyre!("index_patterns must be an array of strings")); + } + Ok(()) +} + +fn append_exact_index(template: &mut Value, target_index: &str) -> Result { + validate_index_patterns_array(template)?; + let patterns = template + .get_mut("index_patterns") + .and_then(Value::as_array_mut) + .expect("validated above"); + if patterns + .iter() + .any(|pattern| pattern.as_str() == Some(target_index)) + { + return Ok(false); + } + patterns.push(Value::String(target_index.to_string())); + Ok(true) +} +async fn write_template( + client: &Elasticsearch, + parsed: &ParsedTemplate, + body: &Value, +) -> Result<()> { let mut headers = HeaderMap::new(); headers.insert("content-type", HeaderValue::from_static("application/json")); let path = format!("/_index_template/{}", parsed.name); @@ -170,7 +320,7 @@ async fn install_template( } else { Some(&[("create", "true")][..]) }; - let body = serde_json::to_vec(&parsed.body)?; + let body = serde_json::to_vec(body)?; let response = client .send(method, &path, headers, params, Some(body), None) .await @@ -191,12 +341,22 @@ async fn install_template( } fn parse_template(config: TemplateConfig) -> Result { - let body = std::fs::read_to_string(&config.path) - .map_err(|err| eyre!("failed to read template '{}': {err}", config.path.display()))?; - let value = parse_config_body("template", &config.path, &body)?; + let (body, default_name, bundled) = match config.source { + TemplateSource::File(path) => { + let contents = std::fs::read_to_string(&path) + .map_err(|err| eyre!("failed to read template '{}': {err}", path.display()))?; + let body = parse_config_body("template", &path, &contents)?; + let name = derive_template_name(&path)?; + (body, name, false) + } + TemplateSource::Bundled(selector) => { + let embedded = embedded_templates::resolve(&selector)?; + (embedded.body, embedded.default_name, true) + } + }; let name = match config.name { Some(name) => name, - None => derive_template_name(&config.path)?, + None => default_name, }; if name.is_empty() { return Err(eyre!("template name must be non-empty")); @@ -205,7 +365,8 @@ fn parse_template(config: TemplateConfig) -> Result { Ok(ParsedTemplate { name, overwrite: config.overwrite, - body: value, + body, + bundled, }) } @@ -818,9 +979,9 @@ fn extract_update_id(doc: &RawValue) -> Result<(String, Value)> { mod tests { use super::{ DEFAULT_BATCH_SIZE, DEFAULT_MAX_INFLIGHT_REQUESTS, ElasticsearchOutputConfig, - OutputPreflightConfig, PreparedPreflight, TemplateConfig, build_bulk_body, - extract_default_pipeline, extract_update_id, index_patterns_match, parse_template, - wildcard_match, + OutputPreflightConfig, PreparedPreflight, TemplateConfig, TemplateSource, + append_exact_index, build_bulk_body, extract_default_pipeline, extract_stored_template, + extract_update_id, index_patterns_match, parse_template, wildcard_match, }; use crate::input::InputDocument; use crate::output::BulkAction; @@ -964,7 +1125,7 @@ mod tests { std::fs::write(&path, r#"{"index_patterns":["logs-*"]}"#).unwrap(); let parsed = parse_template(TemplateConfig { - path, + source: TemplateSource::File(path), name: None, overwrite: true, }) @@ -981,7 +1142,7 @@ mod tests { std::fs::write(&path, r#"{"index_patterns":["logs-*"]}"#).unwrap(); let parsed = parse_template(TemplateConfig { - path, + source: TemplateSource::File(path), name: Some("custom-template".to_string()), overwrite: false, }) @@ -991,6 +1152,76 @@ mod tests { assert!(!parsed.overwrite); } + #[test] + fn bundled_template_uses_default_and_overridden_names() { + let default = parse_template( + TemplateConfig::try_new(Some(PathBuf::from("_okf")), None, None) + .unwrap() + .unwrap(), + ) + .unwrap(); + assert_eq!(default.name, "open-knowledge-format"); + assert!(default.bundled); + + let overridden = parse_template( + TemplateConfig::try_new( + Some(PathBuf::from("_okf")), + Some("team-okf".to_string()), + None, + ) + .unwrap() + .unwrap(), + ) + .unwrap(); + assert_eq!(overridden.name, "team-okf"); + assert!(overridden.bundled); + } + + #[test] + fn template_paths_with_underscores_outside_the_first_character_stay_files() { + let config = + TemplateConfig::try_new(Some(PathBuf::from("templates/_okf.json")), None, None) + .unwrap() + .unwrap(); + assert!(matches!(config.source, TemplateSource::File(_))); + } + + #[test] + fn stored_template_extraction_requires_one_exact_valid_body() { + let body = json!({ + "index_templates": [{ + "name": "open-knowledge-format", + "index_template": {"index_patterns": ["knowledge-a"], "priority": 7} + }] + }); + let stored = extract_stored_template(&body, "open-knowledge-format").unwrap(); + assert_eq!(stored["priority"], 7); + + assert!(extract_stored_template(&json!({}), "open-knowledge-format").is_err()); + assert!( + extract_stored_template( + &json!({"index_templates": [{"name": "other", "index_template": {"index_patterns": []}}]}), + "open-knowledge-format" + ) + .is_err() + ); + assert!( + extract_stored_template( + &json!({"index_templates": [{"name": "open-knowledge-format", "index_template": {"index_patterns": "knowledge-*"}}]}), + "open-knowledge-format" + ) + .is_err() + ); + } + + #[test] + fn exact_target_append_ignores_wildcard_coverage() { + let mut body = json!({"index_patterns": ["team-*"]}); + assert!(append_exact_index(&mut body, "team-knowledge").unwrap()); + assert_eq!(body["index_patterns"], json!(["team-*", "team-knowledge"])); + assert!(!append_exact_index(&mut body, "team-knowledge").unwrap()); + } + #[test] fn template_name_rejects_empty_override() { let dir = tempfile::tempdir().unwrap(); @@ -998,7 +1229,7 @@ mod tests { std::fs::write(&path, r#"{"index_patterns":["logs-*"]}"#).unwrap(); let err = parse_template(TemplateConfig { - path, + source: TemplateSource::File(path), name: Some(String::new()), overwrite: true, }) @@ -1014,7 +1245,7 @@ mod tests { std::fs::write(&path, r#"{"index_patterns":["logs-*"] /* no */}"#).unwrap(); let err = parse_template(TemplateConfig { - path: path.clone(), + source: TemplateSource::File(path.clone()), name: None, overwrite: true, }) @@ -1040,13 +1271,13 @@ mod tests { .unwrap(); let jsonc = parse_template(TemplateConfig { - path: jsonc_path, + source: TemplateSource::File(jsonc_path), name: None, overwrite: true, }) .unwrap(); let json5 = parse_template(TemplateConfig { - path: json5_path, + source: TemplateSource::File(json5_path), name: None, overwrite: true, }) @@ -1073,7 +1304,7 @@ template: .unwrap(); let parsed = parse_template(TemplateConfig { - path, + source: TemplateSource::File(path), name: None, overwrite: true, }) @@ -1091,7 +1322,7 @@ template: std::fs::write(&path, r#"{"index_patterns":["logs-*"]}"#).unwrap(); let parsed = parse_template(TemplateConfig { - path, + source: TemplateSource::File(path), name: None, overwrite: true, }) diff --git a/src/output/elasticsearch/embedded_templates.rs b/src/output/elasticsearch/embedded_templates.rs new file mode 100644 index 0000000..4377efa --- /dev/null +++ b/src/output/elasticsearch/embedded_templates.rs @@ -0,0 +1,184 @@ +use eyre::{Result, eyre}; +use rust_embed::RustEmbed; +use serde_json::Value; + +#[derive(RustEmbed)] +#[folder = "assets/templates/"] +struct TemplateAssets; + +#[derive(Debug)] +pub(super) struct EmbeddedTemplate { + pub(super) default_name: String, + pub(super) body: Value, +} + +pub(super) fn resolve(selector: &str) -> Result { + let asset_name = format!("{selector}.yaml"); + let asset = TemplateAssets::get(&asset_name).ok_or_else(|| { + let available = available().join(", "); + eyre!("unknown bundled template '{selector}'; available bundled templates: {available}") + })?; + let contents = std::str::from_utf8(asset.data.as_ref()) + .map_err(|err| eyre!("bundled template '{selector}' is not UTF-8: {err}"))?; + let body: Value = yaml_serde::from_str(contents) + .map_err(|err| eyre!("bundled template '{selector}' is invalid YAML: {err}"))?; + let default_name = body + .pointer("/_meta/espipe/default_template_name") + .and_then(Value::as_str) + .filter(|name| !name.is_empty()) + .ok_or_else(|| { + eyre!("bundled template '{selector}' has no _meta.espipe.default_template_name") + })? + .to_string(); + + Ok(EmbeddedTemplate { default_name, body }) +} + +pub(super) fn available() -> Vec { + let mut selectors = TemplateAssets::iter() + .filter_map(|name| name.strip_suffix(".yaml").map(str::to_string)) + .collect::>(); + selectors.sort(); + selectors +} + +#[cfg(test)] +mod tests { + use super::{available, resolve}; + use serde_json::Value; + + fn mapping_type(template: &Value, path: &str) -> Option { + let mut mapping = &template["template"]["mappings"]; + for segment in path.split('.') { + mapping = mapping.get("properties")?.get(segment)?; + } + mapping.get("type")?.as_str().map(str::to_string) + } + + fn assert_no_multifields(value: &Value) { + match value { + Value::Object(object) => { + assert!( + !object.contains_key("fields"), + "unexpected multifield: {value}" + ); + for child in object.values() { + assert_no_multifields(child); + } + } + Value::Array(array) => { + for child in array { + assert_no_multifields(child); + } + } + _ => {} + } + } + + #[test] + fn okf_is_compiled_into_the_catalog() { + let template = resolve("_okf").unwrap(); + assert_eq!(template.default_name, "open-knowledge-format"); + assert_eq!(template.body["_meta"]["okf_version"], "0.2"); + assert!(available().contains(&"_okf".to_string())); + } + + #[test] + fn unknown_selector_lists_available_templates() { + let error = resolve("_missing").unwrap_err().to_string(); + assert!(error.contains("unknown bundled template '_missing'")); + assert!(error.contains("_okf")); + } + + #[test] + fn okf_maps_every_official_v0_2_field() { + let template = resolve("_okf").unwrap().body; + let expected = [ + ("content.type", "keyword"), + ("content.title", "text"), + ("content.description", "text"), + ("content.resource", "keyword"), + ("content.tags", "keyword"), + ("content.okf_version", "keyword"), + ("content.status", "keyword"), + ("content.stale_after", "date"), + ("content.runtime", "keyword"), + ("content.computation", "keyword"), + ("content.body", "text"), + ("content.markdown", "text"), + ("content.sources", "nested"), + ("content.sources.id", "keyword"), + ("content.sources.resource", "keyword"), + ("content.sources.title", "text"), + ("content.sources.author", "keyword"), + ("content.sources.usage_count", "long"), + ("content.sources.last_modified", "date"), + ("content.sources.usage_window.from", "date"), + ("content.sources.usage_window.to", "date"), + ("content.usage_window.from", "date"), + ("content.usage_window.to", "date"), + ("content.generated.by", "keyword"), + ("content.generated.at", "date"), + ("content.verified", "nested"), + ("content.verified.by", "keyword"), + ("content.verified.at", "date"), + ("content.parameters", "nested"), + ("content.parameters.name", "keyword"), + ("content.parameters.type", "keyword"), + ("content.parameters.required", "boolean"), + ("content.executor.resource", "keyword"), + ("content.executor.receipt", "keyword"), + ("content.attester.resource", "keyword"), + ("origin.scheme", "keyword"), + ("origin.path", "keyword"), + ("origin.filename", "keyword"), + ]; + + for (path, expected_type) in expected { + assert_eq!( + mapping_type(&template, path).as_deref(), + Some(expected_type), + "mapping for {path}" + ); + } + } + + #[test] + fn okf_unknown_strings_are_one_bounded_keyword() { + let template = resolve("_okf").unwrap().body; + let mappings = &template["template"]["mappings"]; + assert_eq!(mappings["date_detection"], false); + assert_eq!( + mappings["dynamic_templates"][0]["unknown_strings"]["match_mapping_type"], + "string" + ); + assert_eq!( + mappings["dynamic_templates"][0]["unknown_strings"]["mapping"]["type"], + "keyword" + ); + assert_eq!( + mappings["dynamic_templates"][0]["unknown_strings"]["mapping"]["ignore_above"], + 2048 + ); + assert_no_multifields(mappings); + } + + #[test] + fn representative_fixture_covers_both_verified_shapes() { + let documents = include_str!("../../../tests/fixtures/okf_v0_2.ndjson") + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + assert_eq!(documents.len(), 2); + assert!(documents[0]["content"]["verified"].is_object()); + assert!(documents[1]["content"]["verified"].is_array()); + for document in documents { + assert!(document["content"]["sources"][0]["usage_window"].is_object()); + assert!(document["content"]["generated"].is_object()); + assert!(document["content"]["stale_after"].is_string()); + assert!(document["content"]["parameters"].is_array()); + assert!(document["content"]["executor"].is_object()); + assert!(document["content"]["attester"].is_object()); + } + } +} diff --git a/src/output/mod.rs b/src/output/mod.rs index e3ca02f..7dc4d7c 100644 --- a/src/output/mod.rs +++ b/src/output/mod.rs @@ -97,7 +97,12 @@ impl Output { ) -> Result<()> { match uri.scheme().map(|scheme| scheme.as_str()) { Some("file") | None => reject_elasticsearch_options(preflight), - _ => Ok(()), + _ => { + if let Some(template) = &preflight.template { + elasticsearch::validate_bundled_template(template)?; + } + Ok(()) + } } } diff --git a/tests/bundled_template.rs b/tests/bundled_template.rs new file mode 100644 index 0000000..93771ff --- /dev/null +++ b/tests/bundled_template.rs @@ -0,0 +1,444 @@ +use serde_json::{Value, json}; +use std::{ + collections::VecDeque, + fs, + io::{Read, Write}, + net::{TcpListener, TcpStream}, + path::Path, + process::{Command, Output}, + sync::{Arc, Mutex}, + thread, +}; + +#[derive(Clone, Debug)] +struct RecordedRequest { + method: String, + path: String, + body: String, +} + +#[derive(Debug)] +struct MockResponse { + status: u16, + body: String, +} + +fn response(status: u16, body: Value) -> MockResponse { + MockResponse { + status, + body: body.to_string(), + } +} + +fn spawn_server(responses: Vec) -> (String, Arc>>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let thread_requests = Arc::clone(&requests); + let responses = Arc::new(Mutex::new(VecDeque::from(responses))); + + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { + break; + }; + handle_connection(stream, Arc::clone(&thread_requests), Arc::clone(&responses)); + } + }); + + (format!("http://{address}"), requests) +} + +fn handle_connection( + mut stream: TcpStream, + requests: Arc>>, + responses: Arc>>, +) { + let mut buffer = Vec::new(); + let mut chunk = [0_u8; 4096]; + let header_end = loop { + let count = stream.read(&mut chunk).unwrap(); + if count == 0 { + return; + } + buffer.extend_from_slice(&chunk[..count]); + if let Some(index) = buffer.windows(4).position(|window| window == b"\r\n\r\n") { + break index; + } + }; + let headers = String::from_utf8_lossy(&buffer[..header_end]).to_string(); + let content_length = headers + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length: ") + .and_then(|value| value.trim().parse::().ok()) + }) + .unwrap_or(0); + let body_start = header_end + 4; + while buffer.len() < body_start + content_length { + let count = stream.read(&mut chunk).unwrap(); + if count == 0 { + break; + } + buffer.extend_from_slice(&chunk[..count]); + } + let request_line = headers.lines().next().unwrap(); + let mut request_parts = request_line.split_whitespace(); + let method = request_parts.next().unwrap().to_string(); + let path = request_parts.next().unwrap().to_string(); + let body = + String::from_utf8_lossy(&buffer[body_start..body_start + content_length]).to_string(); + requests.lock().unwrap().push(RecordedRequest { + method, + path: path.clone(), + body: body.clone(), + }); + + let response = if path.contains("/_bulk") { + let item_count = body.lines().count() / 2; + let items = (0..item_count) + .map(|_| json!({"index":{"_index":"knowledge","_id":"1","status":201}})) + .collect::>(); + MockResponse { + status: 200, + body: json!({"errors": false, "items": items}).to_string(), + } + } else { + responses + .lock() + .unwrap() + .pop_front() + .unwrap_or_else(|| MockResponse { + status: 500, + body: json!({"error": "unexpected request"}).to_string(), + }) + }; + let reason = match response.status { + 200 => "OK", + 201 => "Created", + 400 => "Bad Request", + 401 => "Unauthorized", + 404 => "Not Found", + 409 => "Conflict", + _ => "Internal Server Error", + }; + let wire = format!( + "HTTP/1.1 {} {reason}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + response.status, + response.body.len(), + response.body + ); + stream.write_all(wire.as_bytes()).unwrap(); +} + +fn run_espipe(current_dir: &Path, input: &Path, output: &str, extra: &[&str]) -> Output { + let mut command = Command::new(env!("CARGO_BIN_EXE_espipe")); + command + .current_dir(current_dir) + .arg(input) + .arg(output) + .arg("--template") + .arg("_okf") + .arg("--uncompressed") + .args(extra); + command.output().expect("run espipe") +} + +fn input_file(directory: &Path) -> std::path::PathBuf { + let path = directory.join("input.ndjson"); + fs::write(&path, "{\"message\":\"hello\"}\n").unwrap(); + path +} + +#[test] +fn bundled_okf_creates_default_template_outside_source_tree() { + let directory = tempfile::tempdir().unwrap(); + let input = input_file(directory.path()); + let (base_url, requests) = spawn_server(vec![ + response(404, json!({"status": 404})), + response(200, json!({"acknowledged": true})), + ]); + + let output = run_espipe( + directory.path(), + &input, + &format!("{base_url}/team-knowledge"), + &[], + ); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let requests = requests.lock().unwrap(); + assert_eq!(requests[0].method, "GET"); + assert_eq!(requests[0].path, "/_index_template/open-knowledge-format"); + assert_eq!(requests[1].method, "PUT"); + assert_eq!(requests[1].path, "/_index_template/open-knowledge-format"); + let body: Value = serde_json::from_str(&requests[1].body).unwrap(); + assert_eq!(body["index_patterns"], json!(["team-knowledge"])); + assert_eq!(body["_meta"]["okf_version"], "0.2"); + assert_eq!(body["template"]["mappings"]["date_detection"], false); + assert_eq!(requests[2].path, "/team-knowledge/_bulk"); +} + +#[test] +fn bundled_okf_override_uses_only_the_selected_name_and_skips_exact_target() { + let directory = tempfile::tempdir().unwrap(); + let input = input_file(directory.path()); + let stored = json!({ + "index_templates": [{ + "name": "team-okf", + "index_template": {"index_patterns": ["team-knowledge"]} + }] + }); + let (base_url, requests) = spawn_server(vec![response(200, stored)]); + + let output = run_espipe( + directory.path(), + &input, + &format!("{base_url}/team-knowledge"), + &["--template-name", "team-okf"], + ); + + assert!(output.status.success()); + let requests = requests.lock().unwrap(); + assert_eq!(requests[0].path, "/_index_template/team-okf"); + assert!( + requests + .iter() + .all(|request| !request.path.contains("open-knowledge-format")) + ); + assert!(requests.iter().all(|request| request.method != "PUT")); + assert_eq!(requests[1].path, "/team-knowledge/_bulk"); +} + +#[test] +fn bundled_okf_appends_exact_target_and_preserves_stored_fields() { + let directory = tempfile::tempdir().unwrap(); + let input = input_file(directory.path()); + let stored_body = json!({ + "index_patterns": ["team-*"], + "composed_of": ["cluster-component"], + "priority": 42, + "version": 9, + "_meta": {"owner": "search-team"}, + "template": { + "settings": {"number_of_shards": 3}, + "mappings": {"properties": {"cluster_only": {"type": "long"}}}, + "aliases": {"knowledge-read": {}} + } + }); + let stored = json!({ + "index_templates": [{ + "name": "open-knowledge-format", + "index_template": stored_body + }] + }); + let (base_url, requests) = spawn_server(vec![ + response(200, stored), + response(200, json!({"acknowledged": true})), + ]); + + let output = run_espipe( + directory.path(), + &input, + &format!("{base_url}/team-knowledge"), + &[], + ); + + assert!(output.status.success()); + let requests = requests.lock().unwrap(); + let updated: Value = serde_json::from_str(&requests[1].body).unwrap(); + assert_eq!( + updated["index_patterns"], + json!(["team-*", "team-knowledge"]) + ); + assert_eq!(updated["composed_of"], json!(["cluster-component"])); + assert_eq!(updated["priority"], 42); + assert_eq!(updated["version"], 9); + assert_eq!(updated["_meta"]["owner"], "search-team"); + assert_eq!(updated["template"]["settings"]["number_of_shards"], 3); + assert_eq!( + updated["template"]["mappings"]["properties"]["cluster_only"]["type"], + "long" + ); + assert!( + updated["template"]["aliases"] + .get("knowledge-read") + .is_some() + ); +} + +#[test] +fn bundled_okf_create_only_handles_missing_existing_and_new_target_branches() { + let directory = tempfile::tempdir().unwrap(); + let input = input_file(directory.path()); + let (base_url, requests) = spawn_server(vec![ + response(404, json!({"status": 404})), + response(200, json!({"acknowledged": true})), + ]); + let output = run_espipe( + directory.path(), + &input, + &format!("{base_url}/team-knowledge"), + &["--template-overwrite=false"], + ); + assert!(output.status.success()); + assert_eq!( + requests.lock().unwrap()[1].path, + "/_index_template/open-knowledge-format?create=true" + ); + + let existing = json!({ + "index_templates": [{ + "name": "open-knowledge-format", + "index_template": {"index_patterns": ["other-knowledge"]} + }] + }); + let (base_url, requests) = spawn_server(vec![response(200, existing)]); + let output = run_espipe( + directory.path(), + &input, + &format!("{base_url}/team-knowledge"), + &["--template-overwrite=false"], + ); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("--template-overwrite=true")); + assert_eq!(requests.lock().unwrap().len(), 1); + + let existing = json!({ + "index_templates": [{ + "name": "open-knowledge-format", + "index_template": {"index_patterns": ["team-knowledge"]} + }] + }); + let (base_url, requests) = spawn_server(vec![response(200, existing)]); + let output = run_espipe( + directory.path(), + &input, + &format!("{base_url}/team-knowledge"), + &["--template-overwrite=false"], + ); + assert!(output.status.success()); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].method, "GET"); + assert_eq!(requests[1].path, "/team-knowledge/_bulk"); + assert!( + requests[1..] + .iter() + .all(|request| !request.path.contains("_index_template")) + ); +} + +#[test] +fn bundled_okf_rejects_lookup_failures_and_malformed_stored_templates() { + let directory = tempfile::tempdir().unwrap(); + let input = input_file(directory.path()); + let (base_url, requests) = spawn_server(vec![response( + 401, + json!({"error": {"type": "security_exception"}}), + )]); + let output = run_espipe( + directory.path(), + &input, + &format!("{base_url}/team-knowledge"), + &[], + ); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("failed to look up index template")); + assert_eq!(requests.lock().unwrap().len(), 1); + + let malformed = json!({ + "index_templates": [{ + "name": "open-knowledge-format", + "index_template": {"index_patterns": "team-*"} + }] + }); + let (base_url, requests) = spawn_server(vec![response(200, malformed)]); + let output = run_espipe( + directory.path(), + &input, + &format!("{base_url}/team-knowledge"), + &[], + ); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("index_patterns must be an array")); + assert_eq!(requests.lock().unwrap().len(), 1); +} + +#[test] +fn bundled_preflight_runs_before_input_construction() { + let directory = tempfile::tempdir().unwrap(); + let input = input_file(directory.path()); + let (base_url, requests) = spawn_server(vec![response( + 401, + json!({"error": {"type": "security_exception"}}), + )]); + + let output = run_espipe( + directory.path(), + &input, + &format!("{base_url}/team-knowledge"), + &["--content", "invalid.field"], + ); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("failed to look up index template")); + assert!(!stderr.contains("--content value")); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].method, "GET"); +} + +#[test] +fn unknown_bundled_selector_fails_before_input_access() { + let directory = tempfile::tempdir().unwrap(); + let missing_input = directory.path().join("missing.ndjson"); + let output = Command::new(env!("CARGO_BIN_EXE_espipe")) + .current_dir(directory.path()) + .arg(&missing_input) + .arg("http://127.0.0.1:9/team-knowledge") + .arg("--template") + .arg("_missing") + .output() + .unwrap(); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("unknown bundled template '_missing'")); + assert!(stderr.contains("_okf")); + assert!(!stderr.contains("failed to open")); +} + +#[test] +fn underscore_inside_file_template_path_keeps_file_behavior() { + let directory = tempfile::tempdir().unwrap(); + let input = input_file(directory.path()); + let templates = directory.path().join("templates"); + fs::create_dir(&templates).unwrap(); + fs::write( + templates.join("_okf.json"), + r#"{"index_patterns":["team-*"]}"#, + ) + .unwrap(); + let (base_url, requests) = spawn_server(vec![response(200, json!({"acknowledged": true}))]); + let output = Command::new(env!("CARGO_BIN_EXE_espipe")) + .current_dir(directory.path()) + .arg(&input) + .arg(format!("{base_url}/team-knowledge")) + .arg("--template") + .arg("templates/_okf.json") + .arg("--uncompressed") + .output() + .unwrap(); + + assert!(output.status.success()); + let requests = requests.lock().unwrap(); + assert_eq!(requests[0].method, "PUT"); + assert_eq!(requests[0].path, "/_index_template/_okf"); + assert!(requests.iter().all(|request| request.method != "GET")); +} diff --git a/tests/fixtures/okf_v0_2.ndjson b/tests/fixtures/okf_v0_2.ndjson new file mode 100644 index 0000000..c842236 --- /dev/null +++ b/tests/fixtures/okf_v0_2.ndjson @@ -0,0 +1,2 @@ +{"content":{"type":"concept","title":"Single verifier","description":"An OKF concept using the verifier shorthand.","resource":"https://example.test/concepts/single","tags":["okf","example"],"okf_version":"0.2","status":"active","stale_after":"2027-01-01T00:00:00Z","runtime":"wasm32-wasi","computation":"sha256:111","body":"Searchable body","markdown":"# Searchable markdown","sources":[{"id":"source-1","resource":"https://example.test/source/1","title":"Source one","author":"Example Author","usage_count":2,"last_modified":"2026-08-01T00:00:00Z","usage_window":{"from":"2026-01-01T00:00:00Z","to":"2026-08-01T00:00:00Z"}}],"usage_window":{"from":"2026-01-01T00:00:00Z","to":"2026-12-31T23:59:59Z"},"generated":{"by":"generator-1","at":"2026-08-20T00:00:00Z"},"verified":{"by":"verifier-1","at":"2026-08-21T00:00:00Z"},"parameters":[{"name":"query","type":"string","required":true}],"executor":{"resource":"https://example.test/executors/1","receipt":["sha256:aaa","sha256:bbb"]},"attester":{"resource":"https://example.test/attesters/1"}},"origin":{"scheme":"file","path":"knowledge/single.md","filename":"single.md"}} +{"content":{"type":"attested-computation","title":"Multiple verifiers","description":"An OKF computation with verifier history.","resource":"https://example.test/concepts/multiple","tags":["okf","computation"],"okf_version":"0.2","status":"verified","stale_after":"2027-02-01T00:00:00Z","runtime":"python3","computation":"sha256:222","body":"Another searchable body","markdown":"# Another document","sources":[{"id":"source-2","resource":"https://example.test/source/2","title":"Source two","author":"Another Author","usage_count":1,"last_modified":"2026-08-02T00:00:00Z","usage_window":{"from":"2026-02-01T00:00:00Z","to":"2026-08-02T00:00:00Z"}}],"usage_window":{"from":"2026-02-01T00:00:00Z","to":"2026-12-31T23:59:59Z"},"generated":{"by":"generator-2","at":"2026-08-22T00:00:00Z"},"verified":[{"by":"verifier-1","at":"2026-08-23T00:00:00Z"},{"by":"verifier-2","at":"2026-08-24T00:00:00Z"}],"parameters":[{"name":"limit","type":"integer","required":false}],"executor":{"resource":"https://example.test/executors/2","receipt":"sha256:ccc"},"attester":{"resource":"https://example.test/attesters/2"}},"origin":{"scheme":"file","path":"knowledge/multiple.md","filename":"multiple.md"}} diff --git a/tests/index_template.rs b/tests/index_template.rs index 99fbda7..1a79c20 100644 --- a/tests/index_template.rs +++ b/tests/index_template.rs @@ -759,6 +759,27 @@ fn invalid_template_arguments_fail_before_input_access() { String::from_utf8_lossy(&output.stderr) .contains("template options require an Elasticsearch output") ); + + for target in [output_path.display().to_string(), "-".to_string()] { + let output = run_espipe(&[ + missing_input.display().to_string(), + target, + "--template".to_string(), + "_missing".to_string(), + ]); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("template options require an Elasticsearch output"), + "stderr: {stderr}" + ); + assert!( + !stderr.contains("unknown bundled template"), + "stderr: {stderr}" + ); + assert!(!stderr.contains("missing.ndjson"), "stderr: {stderr}"); + } } #[test]