diff --git a/CHANGELOG.md b/CHANGELOG.md index a27cb7c..7c76c91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,30 @@ Notable changes to Cloudfall. The format follows A missing playbook, inventory, or role directory exits 2 before Ansible starts. `playbook run --check` is now `--dry-run`; Ansible's play log streams to stderr. `cloudfall-engine manifest` describes every command +- **Breaking:** the last eight commands that change state move to treaty: + `observe`, `secrets render`, `import render`, `import render-api`, + `deploy`, `rollback`, `restart` and `data migrate` (treaty #183 lets + their camelCase keys through). Each keeps its `data` keys, `status` + included, and adds `effect`. `deploy`, `rollback`, `restart` and + `data migrate` still change nothing without `--yes`; their plan answers + `effect: noop` and writes the result keys as `null`, and the result + writes the plan keys (`wouldRun`, `instruction`) as `null`, since treaty + answers one object per command. `observe` exits 91 `INCOMPLETE` (was 1) + with the run in `data` when a server gave no snapshot. Engine steps, + sops and the Render API go through treaty: `doctor` checks sops, and + `import render-api` honours `--proxy` and marks its answer as content + from outside. Failures that were exit 2 are 4 `PRECONDITION` with the + old code in `error.context.code`. A key a command writes only sometimes + is `null` when absent, so `inventory show` writes `data.ansible` as + `null` for a project. Still on argparse: `changelog`, `operator run` + (treaty #175) and `dashboard serve` +- **Breaking:** `why` runs on treaty, `--format html` included (treaty + #179): the page is drawn from the answer's JSON document, as before. Its + verdict moves to `data.status`; a bad `--since` or `--until` is 2 + `ARG_ERROR` with `why_instant_invalid` in `error.context.code`, and a + record that cannot be read 86 `RECORD_INVALID`. treaty offers a custom + format on every command (#209), so `--format html` on any other treaty + command renders its `data` as a page of JSON - **Breaking:** seven commands that run Ansible move to treaty: `health`, `backup run`, `backup verify`, `operations propose`, `operations approve`, `operator approve` and `migrate`. Ansible runs through treaty in the @@ -116,24 +140,22 @@ Notable changes to Cloudfall. The format follows root `--help` and `--version` stay on argparse for now. Their answer is a treaty envelope (`ok`, `data`, `error`, `meta`, `warnings`): the verdict moves from the top-level `status` to `data.status`, and the other `data` - keys are unchanged. `inventory show` writes `data.ansible` as `{}` for a + keys are unchanged. `inventory show` writes `data.ansible` as `null` for a project. Failures are an envelope on stdout with their own exit codes: 79 `PROJECT_INVALID`, 80 `CONFIG_INVALID` (resources or evidence files), 85 `INVENTORY_UNREADABLE`, 86 `RECORD_INVALID`, 5 `NOT_FOUND` for an undeclared operation or a missing proposal, 2 `ARG_ERROR` for bad input; the old snake_case code is in `error.context.code`. `audit` exits 83 - `DRIFT` (was 1) or 84 `UNKNOWN` (was 3) with the report in `data`; until - treaty #181 is fixed, that report's lists come back sorted rather than in - the order the servers and checks were declared. Pick + `DRIFT` (was 1) or 84 `UNKNOWN` (was 3) with the report in `data`. Pick the format with `--format json`: `--output` after one of these commands is refused, since treaty's `--output` names a file. Relative paths still resolve against the project, without changing the working directory - `cloudfall-engine` runs Ansible and git through treaty's `ctx.run` - (treaty 1.0.0rc10), so `--timeout` and Ctrl-C stop the whole process + (treaty 1.0.0rc12), so `--timeout` and Ctrl-C stop the whole process group, and secrets are redacted from the play log. The play log streams - to stderr at a terminal; elsewhere, such as under an agent or in CI, it - shows only with `-v`, one JSON line per log line. A failed playbook - carries the last 4096 characters of the log in `error.context.output`. + to stderr as plain text, as before, terminal or not; `--quiet` silences + it. A failed playbook carries the last 4096 characters of the log in + `error.context.output`, marked as untrusted content from the hosts. `playbook run` reports `meta.dry_run: true` under `--dry-run`. A git step that runs past its 600-second limit now exits 10 `TIMEOUT`, not 81 with `artifact_git_timeout`. `cloudfall-engine doctor` checks for diff --git a/engine/README.md b/engine/README.md index 29a0e4a..c548657 100644 --- a/engine/README.md +++ b/engine/README.md @@ -56,10 +56,11 @@ uv run cloudfall-engine playbook run engine/ansible/playbooks/proxy.yml \ ``` `--dry-run` (Ansible's check mode), `--diff`, `--syntax-check`, `--limit`, -`--tags`, and `--extra-vars` pass through to `ansible-playbook`. At a terminal, -Ansible's play log streams to stderr as it runs; elsewhere, such as under an -agent or in CI, pass `-v` to see it. stdout carries one JSON document when the -run ends, and a failed run keeps the end of the log in `error.context.output`. +`--tags`, and `--extra-vars` pass through to `ansible-playbook`. Ansible's play +log streams to stderr as it runs, as plain text whether or not a terminal is +attached (`--quiet` silences it). stdout carries one JSON document when the run +ends, and a failed run keeps the end of the log in `error.context.output`, +marked as untrusted content from the hosts. `cloudfall-engine doctor` checks that ansible-playbook and git are installed. A project uses this instead of setting `ANSIBLE_CONFIG` against a checkout; the `task` targets in this diff --git a/engine/src/cloudfall_engine/cli.py b/engine/src/cloudfall_engine/cli.py index 15f499b..6a75734 100644 --- a/engine/src/cloudfall_engine/cli.py +++ b/engine/src/cloudfall_engine/cli.py @@ -3,7 +3,7 @@ Built on treaty: every run answers one JSON envelope on stdout, a failure carries a declared exit code, and ``cloudfall-engine manifest`` describes each command. Ansible and git run through ``ctx.run``; Ansible's play log -streams to stderr on a terminal or under ``-v``. +streams to stderr as plain text. """ import json @@ -23,7 +23,18 @@ ) from cloudfall.resources import default_engine_directory, default_schema_directory from cloudfall.validation import ConfigValidationError, validate_config -from treaty import App, Arg, Ctx, Exit, Flag, Out, ParseError, Subprocess, Timeout +from treaty import ( + App, + Arg, + Ctx, + Exit, + External, + Flag, + Out, + ParseError, + Subprocess, + Timeout, +) from cloudfall_engine.ansible_inventory import render_ansible_inventory from cloudfall_engine.artifact import ( @@ -88,8 +99,8 @@ retryable=False, side_effects="partial", suggestion=( - "read the end of the Ansible log in error.context.output, or run " - "again with -v to stream all of it; fix the failing task, and run again" + "read the Ansible log on stderr, or its end in error.context.output; " + "fix the failing task, and run again" ), ) @@ -392,13 +403,14 @@ class Played: "run", description=( "Run one bundled or project playbook against the inventory; " - "the Ansible log streams to stderr at a terminal or under -v" + "the Ansible log streams to stderr" ), danger_level="mutating", exit_codes=["PRECONDITION", "PLAYBOOK_FAILED"], timeout=None, subprocess=Subprocess("ansible-playbook"), required_tools={"ansible-playbook": ANSIBLE_MINIMUM_VERSION}, + child_log=True, external=False, supports_raw_payload=True, examples=[ @@ -423,9 +435,10 @@ def run_playbook(args: RunArgs, ctx: Ctx) -> Played: error.detail, context={"code": error.code} ) from error # Stdout belongs to the envelope, so treaty streams Ansible's play log to - # stderr line by line, redacted, and keeps its tail for a failure. + # stderr as plain text, redacted, terminal or not (a timer's journal or a + # CI log keeps it), and keeps its tail for a failure. completed = ctx.run( - command.argv, env=command.environment, check=False, stream=True + command.argv, env=command.environment, check=False, stream="always" ) if completed.returncode != 0: returncode = completed.returncode @@ -435,7 +448,8 @@ def run_playbook(args: RunArgs, ctx: Ctx) -> Played: context={ "returncode": returncode, "playbook": str(run.playbook), - "output": completed.stdout, + # The hosts wrote this text: the agent gets it marked untrusted. + "output": External(completed.stdout), }, ) return Played(_effect(args), run.playbook, run.inventory_file) diff --git a/pyproject.toml b/pyproject.toml index 82e4ef4..b894dc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "ansible-core>=2.21,<2.22", "jsonschema[format]>=4.23,<5", "pyyaml>=6,<7", - "treaty==1.0.0rc10", + "treaty==1.0.0rc12", ] [project.urls] diff --git a/sdk/src/cloudfall/app.py b/sdk/src/cloudfall/app.py index 14af68f..330a462 100644 --- a/sdk/src/cloudfall/app.py +++ b/sdk/src/cloudfall/app.py @@ -15,9 +15,13 @@ from __future__ import annotations import errno +import json import os +import shutil from dataclasses import dataclass from functools import partial +from html import escape +from http import HTTPStatus from importlib.metadata import version from pathlib import Path from typing import TYPE_CHECKING, ClassVar, Self, cast @@ -83,24 +87,47 @@ ConnectionAddress, Hostname, LinuxUser, + ReleaseId, ResourceId, TcpPort, ) +from cloudfall.importer import ( + ImportTargets, + RenderImportError, + import_render_blueprint, +) from cloudfall.inventory import PlatformInventory from cloudfall.lifecycle import ( ERROR_EXECUTION_FAILED, STEP_TIMEOUT_SECONDS, + DeployOptions, EngineContext, ExecutionStep, LifecycleError, + LifecyclePreview, StepRun, StepRunner, backup_service, + deploy, health, + migrate_data, + preview_data_migration, + preview_deploy, + preview_restart, + preview_rollback, + restart, + rollback, verify_backup, ) from cloudfall.migrate import MigrateError, MigrateOptions, execute_migration from cloudfall.observation import load_observations +from cloudfall.observe import ( + ObservationRequest, + ObserveError, + PlaybookRun, + collect_observations, + team_configuration, +) from cloudfall.operations import FleetOperations, UtcTimestamp, build_operations_view from cloudfall.operator import ( ERROR_PROPOSAL_MISSING, @@ -130,8 +157,21 @@ resolve_installed_version, resolve_project_directory, ) +from cloudfall.render_api import ( + ERROR_API_UNREACHABLE, + HttpRenderApiClient, + import_render_api, + read_api_key, +) from cloudfall.resources import default_engine_directory, default_schema_directory -from cloudfall.secrets import load_environment_receipts +from cloudfall.secrets import ( + ERROR_DECRYPT_FAILED, + ERROR_SOPS_MISSING, + SecretsError, + SopsSecretProvider, + load_environment_receipts, + render_environment, +) from cloudfall.service_evidence import ( DeploymentReceiptSet, DomainObservationSet, @@ -147,9 +187,10 @@ ValidatedConfig, validate_config, ) +from cloudfall.why import WhyError, WhyQuery, answer, render_why_document if TYPE_CHECKING: - from collections.abc import Mapping, Sequence + from collections.abc import Callable, Mapping, Sequence app = App( "cloudfall", @@ -224,16 +265,31 @@ ) +def _html_page(data: object) -> str: + """Render any command's data as a page of its JSON, for ``--format html``. + + Only ``why`` has a page of its own; treaty offers a format on every + command once one has it (treaty #209), so the others answer this. + """ + body = escape(json.dumps(data, indent=2, sort_keys=True)) + return f'\n\n
{body}
\n' + + +app.format("html", render=_html_page, media_type="text/html") + + class Payload: """A command's flat payload, written as ``data`` under its declared keys. ``command`` names the command in the operating contract; its output shape is this payload's schema. A key the shape marks optional is - written as an empty object when the command has nothing for it, since + written as ``null`` when the command has nothing for it, since treaty writes every key of a schema on every answer. """ command: ClassVar[str] + writes: ClassVar[bool] = False + """The command changes state, so the payload carries ``effect`` (REQ-C-003).""" def __init__(self, body: Mapping[str, object]) -> None: """Keep the payload as the domain serialized it.""" @@ -247,12 +303,32 @@ def _shape_keys(command: str) -> tuple[tuple[str, bool], ...]: for entry in CLI_COMMANDS if entry.program == "cloudfall" and entry.name == command ) - (shape,) = contract.output - return tuple((key.name, key.optional) for key in shape.keys if key.name != "error") + # A command with two shapes (a plan without --yes, a result with it) + # writes one object: a key not in every shape is optional, so null. + names: list[str] = [] + for shape in contract.output: + names.extend( + key.name + for key in shape.keys + if key.name != "error" and key.name not in names + ) + return tuple( + ( + name, + any( + all(key.name != name for key in shape.keys) + or any(key.name == name and key.optional for key in shape.keys) + for shape in contract.output + ), + ) + for name in names + ) def _payload_schema(cls: type[Payload]) -> dict[str, object]: names = [name for name, _ in _shape_keys(cls.command)] + if cls.writes: + names.append("effect") return { "type": "object", # The domain writes each array in the order it means: a ranking, a @@ -267,7 +343,7 @@ def _payload_document(payload: Payload) -> dict[str, object]: document = dict(payload.body) for name, optional in _shape_keys(payload.command): if optional and name not in document: - document[name] = {} + document[name] = None return document @@ -334,6 +410,8 @@ class Fleet: directory: Path state: ValidatedConfig read: FleetRead | None = None + source: InventorySource | None = None + """The team's Ansible inventory the fleet was read from, if any.""" @classmethod def acquire(cls, args: ProjectArgs, _ctx: Ctx) -> Self: @@ -342,7 +420,7 @@ def acquire(cls, args: ProjectArgs, _ctx: Ctx) -> Self: source = _inventory_source(args) if source is not None: read = read_fleet(read_inventory(source), args.schemas) - return cls(Path.cwd(), read.config, read) + return cls(Path.cwd(), read.config, read, source) directory = resolve_project_directory( args.project, os.environ, Path.cwd() ) @@ -432,7 +510,7 @@ class InventoryPayload(Payload): "show", description=( "Show the typed, secret-free platform inventory; data.ansible says " - "how it was read from an Ansible inventory, empty for a project" + "how it was read from an Ansible inventory, null for a project" ), danger_level="safe", exit_codes=[ @@ -656,7 +734,7 @@ class ProposalPayload(Payload): "list", description="List proposal receipts, oldest first", danger_level="safe", - exit_codes=["PROJECT_INVALID", "CONFIG_INVALID", "RECORD_INVALID"], + exit_codes=["PROJECT_INVALID", "CONFIG_INVALID", "RECORD_INVALID", "NOT_FOUND"], examples=[("List the operator's proposals", "cloudfall operator list")], ) def operator_list(args: ProposalArgs, _ctx: Ctx, fleet: Fleet) -> ProposalsPayload: @@ -1092,6 +1170,7 @@ def _authoring_failed(error: AuthoringError) -> Exception: "CONFLICT", "PERMISSION_DENIED", "PRECONDITION", + "NOT_FOUND", ] @@ -1101,7 +1180,7 @@ def _authoring_failed(error: AuthoringError) -> Exception: danger_level="mutating", timeout=30, supports_raw_payload=True, - exit_codes=[*_ADD_EXIT_CODES, "NOT_FOUND"], + exit_codes=_ADD_EXIT_CODES, examples=[ ( "Declare your own key", @@ -1440,6 +1519,7 @@ class HealthPayload(Payload): "PROJECT_INVALID", "CONFIG_INVALID", "PRECONDITION", + "ENGINE_STEP_FAILED", "UNHEALTHY", ], timeout=None, @@ -2090,3 +2170,758 @@ def migrate(args: MigrateArgs, ctx: Ctx, fleet: Fleet) -> Migration: message = f"the migration failed at {migration.step}" raise Exit.ENGINE_STEP_FAILED(message, context=cause, data=migration) return migration + + +# why + + +@dataclass(frozen=True, slots=True, kw_only=True) +class WhyArgs(DecisionsArgs): + """Arguments of ``why``.""" + + host: str | None = Flag( + default=None, description="Only decisions whose record names this host" + ) + operation: ResourceId | None = Flag( + default=None, description="Only decisions of this declared operation" + ) + since: str | None = Flag( + default=None, + description=( + "Only decisions with a moment at or after this ISO 8601 time or date" + ), + ) + until: str | None = Flag( + default=None, + description=( + "Only decisions with a moment at or before this ISO 8601 time or date" + ), + ) + + def __post_init__(self) -> None: + """Refuse a time the question cannot be asked with.""" + DecisionsArgs.__post_init__(self) + try: + self.query() + except WhyError as error: + raise ParseError(error.detail, context={"code": error.code}) from error + + def query(self) -> WhyQuery: + """Return the question, as the record is filtered by it.""" + return WhyQuery.from_boundary( + host=self.host, + operation=self.operation.value if self.operation is not None else None, + since=self.since, + until=self.until, + ) + + +class WhyPayload(Payload): + """``why``: each decision the question is about, told from its record.""" + + command = "why" + + +@app.command( + "why", + description=( + "Answer why the agent did that, from the record, for a host, an operation " + "or a time window; --format html renders one page" + ), + danger_level="safe", + exit_codes=["RECORD_INVALID"], + renderers={"html": render_why_document}, + examples=[ + ("Ask about one host", "cloudfall why --host h1"), + ("One page for a person", "cloudfall why --since 2026-10-01 --format html"), + ], +) +def why(args: WhyArgs, _ctx: Ctx) -> WhyPayload: + """Answer from the record alone: no catalog, no fleet.""" + store = DecisionStore( + directory=args.root / args.decisions, catalog=SchemaCatalog(args.schemas) + ) + try: + result = answer(store, args.query()) + except (DecisionError, WhyError) as error: + raise Exit.RECORD_INVALID(error.detail, context={"code": error.code}) from error + return WhyPayload(result.as_dict()) + + +# observe, secrets render, import render, import render-api: payloads with +# camelCase keys, written through the adapter now that it may carry effect. + + +app.exit_code( + "INCOMPLETE", + 91, + description=( + "Some servers produced no snapshot or Ansible failed; data says which" + ), + retryable=True, + # The hosts are only read; a retry rewrites the snapshots it wrote. + side_effects="none", + suggestion="read data.missing and data.detail, fix the hosts, then observe again", +) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ObserveArgs(FleetArgs): + """Arguments of ``observe``.""" + + output_dir: Path = Flag( + default=Path("tmp/observed"), + description="Snapshot directory to write (default: tmp/observed)", + ) + limit: str | None = Flag( + default=None, description="Ansible host pattern to inspect a subset" + ) + engine: Path = Flag( + default=default_engine_directory(), + description="Engine directory holding the playbooks (default: bundled)", + ) + + def __post_init__(self) -> None: + """Keep a relative output path inside the project.""" + _inside_project(self.output_dir, "output-dir") + + +class ObservedPayload(Payload): + """``observe``: which servers produced a snapshot.""" + + command = "observe" + writes = True + + +def _playbook_runner( + ctx: Ctx, directory: Path +) -> Callable[[Sequence[str], Mapping[str, str]], PlaybookRun]: + """Run the inspection through ``ctx.run``, keeping its output for a failure.""" + + def run(argv: Sequence[str], environment: Mapping[str, str]) -> PlaybookRun: + done = ctx.run(list(argv), env=environment, cwd=directory, check=False) + return PlaybookRun(exit_code=done.returncode, output=done.stdout + done.stderr) + + return run + + +@app.command( + "observe", + description=( + "Collect read-only server snapshots from the fleet; servers without one " + "exit non-zero with the run in data" + ), + danger_level="mutating", + exit_codes=[ + "PROJECT_INVALID", + "CONFIG_INVALID", + "INVENTORY_UNREADABLE", + "PRECONDITION", + "INCOMPLETE", + ], + requires=[ONE_FLEET], + timeout=None, + subprocess=Subprocess("ansible-playbook"), + required_tools={"ansible-playbook": "2.21.0"}, + supports_raw_payload=True, + examples=[("Snapshot every server", "cloudfall observe")], +) +def observe(args: ObserveArgs, ctx: Ctx, fleet: Fleet) -> ObservedPayload: + """Inspect every declared server and write one snapshot each.""" + sources = ( + (fleet.source.value,) + if fleet.source is not None + else (fleet.directory / "tmp/ansible-inventory.json",) + ) + output = fleet.path(args.output_dir) + effect = "updated" if output.exists() else "created" + try: + request = ObservationRequest( + inventory_sources=sources, + output_directory=output.resolve(), + engine_directory=args.engine, + limit=args.limit, + configuration=( + team_configuration(Path.cwd()) if fleet.source is not None else None + ), + ) + result = collect_observations( + fleet.inventory, + request, + fleet.path(Path("tmp/cloudfall")), + _playbook_runner(ctx, fleet.directory), + ) + except ObserveError as error: + raise Exit.PRECONDITION(error.detail, context={"code": error.code}) from error + payload = ObservedPayload({**result.as_dict(), "effect": effect}) + if not result.complete: + message = f"{len(result.missing)} server(s) produced no snapshot" + raise Exit.INCOMPLETE(message, data=payload) + return payload + + +# secrets render + + +secrets = app.group("secrets", description="Resolve declared secret references") + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SecretsRenderArgs(EngineArgs): + """Arguments of ``secrets render``.""" + + component: ResourceId = Arg(description="Component whose references to render") + secrets_dir: Path = Flag( + default=Path("secrets"), + description="sops-encrypted secrets directory (default: secrets)", + secret=False, + ) + output_file: Path | None = Flag( + default=None, + description="Environment file to write (default: tmp/env/.env)", + ) + receipts: Path = Flag( + default=Path("tmp/env-receipts"), + description="Environment receipt directory (default: tmp/env-receipts)", + ) + + def __post_init__(self) -> None: + """Keep relative paths inside the project.""" + EngineArgs.__post_init__(self) + _inside_project(self.secrets_dir, "secrets-dir") + _inside_project(self.receipts, "receipts") + if self.output_file is not None: + _inside_project(self.output_file, "output-file") + + +class RenderedSecretsPayload(Payload): + """``secrets render``: names and hashes of what was written, never values.""" + + command = "secrets render" + writes = True + + +def _sops_decrypt(ctx: Ctx) -> Callable[[Path], str]: + """Decrypt one sops file through ``ctx.run``.""" + + def decrypt(source: Path) -> str: + binary = shutil.which("sops") + if binary is None: + message = ( + "the sops binary is not installed; install sops and age, and " + "set SOPS_AGE_KEY_FILE to your age key" + ) + raise SecretsError(ERROR_SOPS_MISSING, message) + done = ctx.run([binary, "--decrypt", str(source)], check=False) + if done.returncode != 0: + message = f"sops could not decrypt {source}: {done.stderr.strip()[:300]}" + raise SecretsError(ERROR_DECRYPT_FAILED, message) + return done.stdout + + return decrypt + + +@secrets.command( + "render", + description=( + "Render one component's secret references into its 0600 environment " + "file; the answer names keys and hashes, never values" + ), + danger_level="mutating", + exit_codes=["PROJECT_INVALID", "CONFIG_INVALID", "PRECONDITION"], + timeout=120, + subprocess=Subprocess("sops"), + required_tools={"sops": "3.8.0"}, + supports_raw_payload=True, + examples=[("Render one component", "cloudfall secrets render crm-backend")], +) +def secrets_render( + args: SecretsRenderArgs, ctx: Ctx, fleet: Fleet +) -> RenderedSecretsPayload: + """Decrypt the component's fragments and write its environment file.""" + output = fleet.path( + args.output_file + if args.output_file is not None + else Path("tmp/env") / f"{args.component}.env" + ) + effect = "updated" if output.exists() else "created" + provider = SopsSecretProvider( + secrets_directory=fleet.path(args.secrets_dir), decrypt=_sops_decrypt(ctx) + ) + try: + body = render_environment( + args.context(fleet, ctx), + args.component, + provider, + output, + receipt_directory=fleet.path(args.receipts), + ) + except SecretsError as error: + raise Exit.PRECONDITION(error.message, context={"code": error.code}) from error + return RenderedSecretsPayload({**body, "effect": effect}) + + +# import render, render-api + + +imports = app.group("import", description="Import external platform definitions") + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ImportArgs(ProjectArgs): + """Options both importers share.""" + + application: ResourceId = Flag( + description="Cloudfall application id (also the application's Linux user)" + ) + server: ResourceId = Flag( + description="Declared server id that receives every imported resource" + ) + output_dir: Path = Flag( + default=Path("tmp/import/config"), + description="Config fragment output directory (default: tmp/import/config)", + ) + env_dir: Path = Flag( + default=Path("tmp/import/env"), + description="Environment file output directory (default: tmp/import/env)", + ) + + def __post_init__(self) -> None: + """Keep relative output paths inside the project.""" + _inside_project(self.output_dir, "output-dir") + _inside_project(self.env_dir, "env-dir") + + def targets(self, project: ProjectDirectory) -> ImportTargets: + """Return where the imported resources land.""" + return ImportTargets( + application_id=self.application, + server_id=self.server, + project_directory=project.path / self.output_dir, + environment_directory=project.path / self.env_dir, + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ImportBlueprintArgs(ImportArgs): + """Arguments of ``import render``.""" + + blueprint: Path = Arg(description="render.yaml blueprint, relative to the project") + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ImportApiArgs(ImportArgs): + """Arguments of ``import render-api``.""" + + api_key_file: Path = Flag( + description="File containing only the Render API key", + # The flag names a file; the key itself never reaches argv. + secret=False, + ) + api_url: str = Flag( + default="https://api.render.com/v1", + description="Render API base URL (default: https://api.render.com/v1)", + ) + + +class ImportedPayload(Payload): + """``import render``: the resources written and what could not be mapped.""" + + command = "import render" + writes = True + + +class ImportedApiPayload(Payload): + """``import render-api``: the resources written from the live workspace.""" + + command = "import render-api" + writes = True + + +_IMPORT_EXIT_CODES = [ + "PROJECT_INVALID", + "CONFIG_INVALID", + "PRECONDITION", + "PERMISSION_DENIED", +] + + +@imports.command( + "render", + description="Map a render.yaml blueprint onto Cloudfall config fragments", + danger_level="mutating", + exit_codes=_IMPORT_EXIT_CODES, + timeout=60, + supports_raw_payload=True, + examples=[ + ( + "Import a blueprint for one server", + "cloudfall import render render.yaml --application crm --server h1", + ), + ], +) +def import_render( + args: ImportBlueprintArgs, _ctx: Ctx, project: ProjectDirectory +) -> ImportedPayload: + """Write config fragments and environment files from the blueprint.""" + try: + result = import_render_blueprint( + project.path / args.blueprint, args.targets(project), args.schemas + ) + except RenderImportError as error: + raise Exit.PRECONDITION(error.detail, context={"code": error.code}) from error + except ConfigValidationError as error: + raise _config_invalid(error) from error + except OSError as error: + if error.errno not in _NOT_WRITABLE: + raise + raise _not_writable(error) from error + return ImportedPayload({**result.as_dict(), "effect": "created"}) + + +def _render_transport(ctx: Ctx) -> Callable[[str, str], object]: + """Fetch one Render API page through ``ctx.http``.""" + + def get(url: str, api_key: str) -> object: + response = ctx.http.get( + url, + headers={ + "Authorization": f"Bearer {api_key}", + "Accept": "application/json", + }, + ) + if response.status != HTTPStatus.OK: + message = f"Render API answered {response.status} for {url}" + raise RenderImportError(ERROR_API_UNREACHABLE, message) + return response.json() + + return get + + +@imports.command( + "render-api", + description="Map a live Render workspace onto Cloudfall config via the API", + danger_level="mutating", + exit_codes=_IMPORT_EXIT_CODES, + timeout=300, + has_network_io=True, + # Service names, URLs and settings in the answer come from the workspace. + external=True, + supports_raw_payload=True, + examples=[ + ( + "Import a workspace", + "cloudfall import render-api --api-key-file render.key " + "--application crm --server h1", + ), + ], +) +def import_render_api_command( + args: ImportApiArgs, ctx: Ctx, project: ProjectDirectory +) -> ImportedApiPayload: + """Read the workspace page by page and write config fragments from it.""" + try: + client = HttpRenderApiClient( + api_key=read_api_key(project.path / args.api_key_file), + base_url=args.api_url, + transport=_render_transport(ctx), + ) + result = import_render_api(client, args.targets(project), args.schemas) + except RenderImportError as error: + raise Exit.PRECONDITION(error.detail, context={"code": error.code}) from error + except ConfigValidationError as error: + raise _config_invalid(error) from error + except OSError as error: + if error.errno not in _NOT_WRITABLE: + raise + raise _not_writable(error) from error + return ImportedApiPayload({**result.as_dict(), "effect": "created"}) + + +# deploy, rollback, restart, data migrate: --yes runs them; without it they +# validate the request and answer the plan. + + +app.scalar( + ReleaseId, + parse=ReleaseId.from_boundary, + pattern=r"[0-9]{8}T[0-9]{6}Z-[0-9a-f]{7,40}", +) + +_PLAN_INSTRUCTION = "review the plan and re-run with --yes to execute it" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ComponentArgs(EngineArgs): + """Options of a command that acts on one component's servers.""" + + component: ResourceId = Arg(description="Component to act on") + yes: bool = Flag( + default=False, + description=( + "Change the servers; without it the command validates the request " + "and only shows what it would do" + ), + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class DeployArgs(ComponentArgs): + """Arguments of ``deploy``.""" + + release: ReleaseId = Flag( + description="Release id produced by cloudfall-engine artifact build" + ) + artifacts: Path = Flag( + default=Path("tmp/artifacts"), + description="Artifact directory (default: tmp/artifacts)", + ) + env_file: Path | None = Flag( + default=None, + description="Optional controller-side environment file for the component", + ) + receipts: Path = Flag( + default=Path("tmp/releases"), + description="Release receipt directory (default: tmp/releases)", + ) + + def __post_init__(self) -> None: + """Keep relative paths inside the project.""" + ComponentArgs.__post_init__(self) + _inside_project(self.artifacts, "artifacts") + _inside_project(self.receipts, "receipts") + + +@dataclass(frozen=True, slots=True, kw_only=True) +class RollbackArgs(ComponentArgs): + """Arguments of ``rollback``.""" + + release: ReleaseId = Flag(description="Existing release id to activate") + + +@dataclass(frozen=True, slots=True, kw_only=True) +class DataMigrateArgs(EngineArgs): + """Arguments of ``data migrate``.""" + + service: ResourceId = Arg(description="Declared service to restore into") + database: str = Flag(description="Declared database name inside the service") + source_url_file: Path = Flag( + description=( + "Controller-side file whose only content is the source database " + "connection URL" + ), + # The flag names a file; the URL with its password never reaches argv. + secret=False, + ) + receipts: Path = Flag( + default=Path("tmp/data-migrations"), + description="Migration receipt directory (default: tmp/data-migrations)", + ) + yes: bool = Flag( + default=False, + description=( + "Change the servers; without it the command validates the request " + "and only shows what it would do" + ), + ) + + def __post_init__(self) -> None: + """Keep relative paths inside the project.""" + EngineArgs.__post_init__(self) + _inside_project(self.receipts, "receipts") + + +class DeployPayload(Payload): + """``deploy``: the plan without --yes, the deployed release with it.""" + + command = "deploy" + writes = True + + +class RollbackPayload(Payload): + """``rollback``: the plan without --yes, the release switched to with it.""" + + command = "rollback" + writes = True + + +class RestartPayload(Payload): + """``restart``: the plan without --yes, the health-gated restart with it.""" + + command = "restart" + writes = True + + +class DataMigratePayload(Payload): + """``data migrate``: the plan without --yes, the verified restore with it.""" + + command = "data migrate" + writes = True + + +def _plan(preview: LifecyclePreview) -> dict[str, object]: + # Nothing ran: noop, as treaty reads would_* only from its own dry run + # (treaty #197). + return {**preview.as_dict(), "instruction": _PLAN_INSTRUCTION, "effect": "noop"} + + +_LIFECYCLE_EXIT_CODES = [ + "PROJECT_INVALID", + "CONFIG_INVALID", + "PRECONDITION", + "ENGINE_STEP_FAILED", +] + + +@app.command( + "deploy", + description=( + "Deploy one built component release behind its health gate; without " + "--yes it shows the plan" + ), + danger_level="mutating", + exit_codes=_LIFECYCLE_EXIT_CODES, + timeout=None, + subprocess=Subprocess("ansible-playbook"), + required_tools={"ansible-playbook": "2.21.0"}, + supports_raw_payload=True, + examples=[ + ( + "Show the plan", + "cloudfall deploy crm-backend --release 20260101T000000Z-abcdef0", + ), + ( + "Deploy it", + "cloudfall deploy crm-backend --release 20260101T000000Z-abcdef0 --yes", + ), + ], +) +def deploy_command(args: DeployArgs, ctx: Ctx, fleet: Fleet) -> DeployPayload: + """Activate the release on the component's servers, or show the plan.""" + context = args.context(fleet, ctx) + artifacts = fleet.path(args.artifacts) + try: + if not args.yes: + return DeployPayload( + _plan(preview_deploy(context, args.component, args.release, artifacts)) + ) + result = deploy( + context, + args.component, + args.release, + artifacts, + DeployOptions( + environment_file=( + fleet.path(args.env_file) if args.env_file is not None else None + ), + receipt_directory=fleet.path(args.receipts), + ), + ) + except LifecycleError as error: + raise _lifecycle_failed(error) from error + return DeployPayload({**result.as_dict(), "effect": "updated"}) + + +@app.command( + "rollback", + description=( + "Switch one component back to an existing release; without --yes it " + "shows the plan" + ), + danger_level="mutating", + exit_codes=_LIFECYCLE_EXIT_CODES, + timeout=None, + subprocess=Subprocess("ansible-playbook"), + required_tools={"ansible-playbook": "2.21.0"}, + supports_raw_payload=True, + examples=[ + ( + "Show the plan", + "cloudfall rollback crm-backend --release 20260101T000000Z-abcdef0", + ), + ], +) +def rollback_command(args: RollbackArgs, ctx: Ctx, fleet: Fleet) -> RollbackPayload: + """Switch the component back to the release, or show the plan.""" + context = args.context(fleet, ctx) + try: + if not args.yes: + return RollbackPayload( + _plan(preview_rollback(context, args.component, args.release)) + ) + result = rollback(context, args.component, args.release) + except LifecycleError as error: + raise _lifecycle_failed(error) from error + return RollbackPayload({**result.as_dict(), "effect": "updated"}) + + +@app.command( + "restart", + description=( + "Restart one component behind its health check; without --yes it shows " + "the plan" + ), + danger_level="mutating", + exit_codes=_LIFECYCLE_EXIT_CODES, + timeout=None, + supports_raw_payload=True, + subprocess=Subprocess("ansible-playbook"), + required_tools={"ansible-playbook": "2.21.0"}, + examples=[ + ("Show the plan", "cloudfall restart crm-backend"), + ("Restart it", "cloudfall restart crm-backend --yes"), + ], +) +def restart_command(args: ComponentArgs, ctx: Ctx, fleet: Fleet) -> RestartPayload: + """Restart the component, or show the plan.""" + context = args.context(fleet, ctx) + try: + if not args.yes: + return RestartPayload(_plan(preview_restart(context, args.component))) + result = restart(context, args.component) + except LifecycleError as error: + raise _lifecycle_failed(error) from error + return RestartPayload({**result.as_dict(), "effect": "updated"}) + + +data = app.group("data", description="Migrate data into declared services") + + +@data.command( + "migrate", + description=( + "Dump an external PostgreSQL database and restore it into a declared " + "service with row-count verification; without --yes it shows the plan" + ), + danger_level="mutating", + exit_codes=_LIFECYCLE_EXIT_CODES, + timeout=None, + subprocess=Subprocess("ansible-playbook"), + required_tools={"ansible-playbook": "2.21.0"}, + supports_raw_payload=True, + examples=[ + ( + "Show the plan", + "cloudfall data migrate postgresql-main --database crm " + "--source-url-file source.url", + ), + ], +) +def data_migrate( + args: DataMigrateArgs, ctx: Ctx, fleet: Fleet +) -> DataMigratePayload: + """Restore the source database into the service, or show the plan.""" + context = args.context(fleet, ctx) + source = fleet.path(args.source_url_file) + try: + if not args.yes: + return DataMigratePayload( + _plan( + preview_data_migration(context, args.service, args.database, source) + ) + ) + body = migrate_data( + context, args.service, args.database, source, fleet.path(args.receipts) + ) + except LifecycleError as error: + raise _lifecycle_failed(error) from error + return DataMigratePayload({**body, "effect": "updated"}) diff --git a/sdk/src/cloudfall/cli.py b/sdk/src/cloudfall/cli.py index a2d2a58..405fb18 100644 --- a/sdk/src/cloudfall/cli.py +++ b/sdk/src/cloudfall/cli.py @@ -14,7 +14,6 @@ from argparse import Namespace from collections.abc import Callable, Mapping, Sequence - from cloudfall.operations import FleetOperations from cloudfall.operator import AlertFeed, OperatorProposal from cloudfall.ansible_api import ( @@ -46,36 +45,11 @@ from cloudfall.decision import ( DECISION_DIRECTORY, DecisionError, - DecisionStore, -) -from cloudfall.importer import ( - ImportTargets, - RenderImportError, - import_render_blueprint, ) from cloudfall.inventory import PlatformInventory from cloudfall.lifecycle import ( - DeployOptions, EngineContext, - LifecycleError, - LifecyclePreview, - deploy, - migrate_data, - preview_data_migration, - preview_deploy, - preview_restart, - preview_rollback, - restart, - rollback, -) -from cloudfall.observation import load_observations -from cloudfall.observe import ( - ObservationRequest, - ObserveError, - collect_observations, - team_configuration, ) -from cloudfall.operations import UtcTimestamp, build_operations_view from cloudfall.operator import ( OperatorError, ProposalStore, @@ -106,30 +80,14 @@ is_project, project_context, ) -from cloudfall.render_api import ( - HttpRenderApiClient, - import_render_api, - read_api_key, -) from cloudfall.resources import default_engine_directory, default_schema_directory -from cloudfall.secrets import ( - SecretsError, - SopsSecretProvider, - render_environment, -) -from cloudfall.service_evidence import ( - DeploymentReceiptSet, - DomainObservationSet, - load_deployment_receipts, - load_domain_observations, -) from cloudfall.validation import ( ConfigValidationError, SchemaCatalog, ValidatedConfig, validate_config, ) -from cloudfall.why import WhyError, WhyQuery, answer, render_why_html +from cloudfall.why import WhyError def _add_config_parsers( @@ -1268,9 +1226,6 @@ def _main(argv: Sequence[str] | None) -> int: return _dispatch_from_inventory(arguments, source) with project_context(arguments.project, os.environ) as project_directory: arguments.project_directory = project_directory - without_state = _run_without_validated_fleet(arguments) - if without_state is not None: - return without_state schema_directory = Path(arguments.schemas) state = validate_config(project_directory, schema_directory) return _dispatch(arguments, state, schema_directory) @@ -1285,18 +1240,6 @@ def _main(argv: Sequence[str] | None) -> int: _ERROR_SOURCE_AMBIGUOUS = "project_source_ambiguous" -_RENDERED_INVENTORY = Path("tmp/ansible-inventory.json") -_OVERLAY_DIRECTORY = "tmp/cloudfall" - - -def _run_without_validated_fleet(arguments: Namespace) -> int | None: - """Run the commands that read no fleet, or return ``None`` for the rest. - - `import` writes the fleet rather than reads it (`add` runs on treaty). - """ - if arguments.command == "import": - return _run_import_render(arguments) - return None def _inventory_source( @@ -1338,40 +1281,12 @@ def _dispatch_from_inventory(arguments: Namespace, source: InventorySource) -> i return _dispatch(arguments, read.config, schema_directory) -def _run_import_render(arguments: Namespace) -> int: - try: - targets = ImportTargets( - application_id=arguments.application, - server_id=arguments.server, - project_directory=Path(arguments.output_dir), - environment_directory=Path(arguments.env_dir), - ) - if arguments.import_command == "render-api": - api_key = read_api_key(Path(arguments.api_key_file)) - client = HttpRenderApiClient(api_key=api_key, base_url=arguments.api_url) - result = import_render_api(client, targets, Path(arguments.schemas)) - else: - result = import_render_blueprint( - Path(arguments.blueprint), targets, Path(arguments.schemas) - ) - except (RenderImportError, ConfigValidationError) as error: - return write_error(error.as_dict(), 2) - write_result(result.as_dict()) - return 0 - - def _dispatch( arguments: Namespace, state: ValidatedConfig, schema_directory: Path ) -> int: handlers: dict[str, Callable[[Namespace, ValidatedConfig, Path], int]] = { - "secrets:render": _run_secrets_render, - "data:migrate": _run_data_migrate, "dashboard:serve": _run_dashboard_serve, - "deploy": _run_deploy, - "observe": _run_observe, "operator:run": _run_operator_run, - "restart": _run_restart, - "rollback": _run_rollback, } key = _command_key(arguments) try: @@ -1398,39 +1313,6 @@ def _command_key(arguments: Namespace) -> str: return f"{arguments.command}:{subcommand}" -def _run_why(arguments: Namespace) -> int: - """Answer the question from the record alone: no catalog, no fleet. - - The record outlives both, so the question must be answerable from a - checkout that holds nothing but the decisions directory. - """ - repository = ( - Path(arguments.repository) if arguments.repository is not None else Path.cwd() - ) - query = WhyQuery.from_boundary( - host=arguments.host, - operation=( - arguments.operation.value if arguments.operation is not None else None - ), - since=arguments.since, - until=arguments.until, - ) - result = answer(_decision_store(arguments, repository), query) - if arguments.format == "html": - sys.stdout.write(render_why_html(result)) - sys.stdout.flush() - return 0 - write_result(result.as_dict()) - return 0 - - -def _decision_store(arguments: Namespace, repository: Path) -> DecisionStore: - return DecisionStore( - directory=repository / Path(arguments.decisions), - catalog=SchemaCatalog(Path(arguments.schemas)), - ) - - def _run_changelog(arguments: Namespace) -> int: write_result( { @@ -1449,65 +1331,10 @@ def _run_changelog(arguments: Namespace) -> int: _RECORD_COMMANDS: Mapping[str, Callable[[Namespace], int]] = { - "why": _run_why, } """The commands that read the repository and nothing else: no fleet, no project.""" -def _run_observe( - arguments: Namespace, state: ValidatedConfig, _schema_directory: Path -) -> int: - """Inspect every declared server and write one snapshot each.""" - source = getattr(arguments, "inventory_source", None) - sources = ( - (source.value,) - if source is not None - else (Path(arguments.project_directory) / _RENDERED_INVENTORY,) - ) - try: - request = ObservationRequest( - inventory_sources=sources, - output_directory=Path(arguments.output_dir).resolve(), - engine_directory=Path(arguments.engine), - limit=arguments.limit, - configuration=( - team_configuration(Path.cwd()) if source is not None else None - ), - ) - result = collect_observations( - PlatformInventory.from_state(state), - request, - Path(_OVERLAY_DIRECTORY), - ) - except ObserveError as error: - return write_error(error.as_dict(), 2) - code = 0 if result.complete else 1 - write_result(result.as_dict(), exit_code=code) - return code - - -def _run_secrets_render( - arguments: Namespace, _state: ValidatedConfig, schema_directory: Path -) -> int: - output = ( - Path(arguments.output_file) - if arguments.output_file is not None - else Path("tmp/env") / f"{arguments.component}.env" - ) - try: - result = render_environment( - _engine_context(arguments, schema_directory), - arguments.component, - SopsSecretProvider(secrets_directory=Path(arguments.secrets_dir)), - output, - receipt_directory=Path(arguments.receipts), - ) - except SecretsError as error: - return write_error(error.as_dict(), 2) - write_result(result) - return 0 - - def _operator_store(arguments: Namespace, schema_directory: Path) -> ProposalStore: return ProposalStore( directory=Path(arguments.proposals), @@ -1625,25 +1452,6 @@ def _run_dashboard_serve( return 0 -def _service_operations( - arguments: Namespace, state: ValidatedConfig, schema_directory: Path -) -> FleetOperations: - observations = load_observations(Path(arguments.observed), schema_directory) - deployment_receipts = _optional_deployment_receipts( - Path(arguments.deployments), schema_directory - ) - domain_observations = _optional_domain_observations( - Path(arguments.service_observed), schema_directory - ) - return build_operations_view( - PlatformInventory.from_state(state), - observations, - deployment_receipts, - domain_observations, - generated_at=UtcTimestamp.now(), - ) - - def _engine_context(arguments: Namespace, schema_directory: Path) -> EngineContext: return EngineContext( project_directory=Path(arguments.project_directory), @@ -1653,134 +1461,8 @@ def _engine_context(arguments: Namespace, schema_directory: Path) -> EngineConte ) -def _lifecycle_exit(error: LifecycleError) -> int: - code = 1 if error.code == "lifecycle_execution_failed" else 2 - return write_error(error.as_dict(), code) - - -def _run_data_migrate( - arguments: Namespace, _state: ValidatedConfig, schema_directory: Path -) -> int: - context = _engine_context(arguments, schema_directory) - try: - if not arguments.yes: - return _write_plan( - preview_data_migration( - context, - arguments.service, - str(arguments.database), - Path(arguments.source_url_file), - ) - ) - result = migrate_data( - context, - arguments.service, - str(arguments.database), - Path(arguments.source_url_file), - Path(arguments.receipts), - ) - except LifecycleError as error: - return _lifecycle_exit(error) - write_result(result) - return 0 - - -def _run_deploy( - arguments: Namespace, _state: ValidatedConfig, schema_directory: Path -) -> int: - context = _engine_context(arguments, schema_directory) - try: - if not arguments.yes: - return _write_plan( - preview_deploy( - context, - arguments.component, - arguments.release, - Path(arguments.artifacts), - ) - ) - result = deploy( - context, - arguments.component, - arguments.release, - Path(arguments.artifacts), - DeployOptions( - environment_file=( - Path(arguments.env_file) if arguments.env_file is not None else None - ), - receipt_directory=Path(arguments.receipts), - ), - ) - except LifecycleError as error: - return _lifecycle_exit(error) - write_result(result.as_dict()) - return 0 - - -def _run_rollback( - arguments: Namespace, _state: ValidatedConfig, schema_directory: Path -) -> int: - context = _engine_context(arguments, schema_directory) - try: - if not arguments.yes: - return _write_plan( - preview_rollback(context, arguments.component, arguments.release) - ) - result = rollback( - context, - arguments.component, - arguments.release, - ) - except LifecycleError as error: - return _lifecycle_exit(error) - write_result(result.as_dict()) - return 0 - - -def _run_restart( - arguments: Namespace, _state: ValidatedConfig, schema_directory: Path -) -> int: - context = _engine_context(arguments, schema_directory) - try: - if not arguments.yes: - return _write_plan(preview_restart(context, arguments.component)) - result = restart(context, arguments.component) - except LifecycleError as error: - return _lifecycle_exit(error) - write_result(result.as_dict()) - return 0 - - -def _write_plan(preview: LifecyclePreview) -> int: - write_result( - { - **preview.as_dict(), - "instruction": "review the plan and re-run with --yes to execute it", - } - ) - return 0 - - def run() -> None: """Installed console-script entry point.""" raise SystemExit(main()) -def _optional_deployment_receipts( - directory: Path, schema_directory: Path -) -> DeploymentReceiptSet: - return ( - load_deployment_receipts(directory, schema_directory) - if directory.is_dir() - else DeploymentReceiptSet.empty() - ) - - -def _optional_domain_observations( - directory: Path, schema_directory: Path -) -> DomainObservationSet: - return ( - load_domain_observations(directory, schema_directory) - if directory.is_dir() - else DomainObservationSet.empty() - ) diff --git a/sdk/src/cloudfall/commands.py b/sdk/src/cloudfall/commands.py index 2700be2..d2ae47b 100644 --- a/sdk/src/cloudfall/commands.py +++ b/sdk/src/cloudfall/commands.py @@ -113,7 +113,7 @@ def treaty_schema(self) -> dict[str, object]: """Return the JSON Schema of this shape in treaty's envelope. ``status`` moves into ``data`` beside the other keys, every key is - written on every answer (an optional one as an empty object), and a + written on every answer (an optional one as ``null``), and a failed run keeps ``data`` only for a verdict such as drift. A command treaty runs as mutating also writes ``effect`` (REQ-C-003). """ @@ -220,14 +220,23 @@ def invocation(self) -> str: def output_schema(self) -> dict[str, object]: """Return the JSON Schema of the command's stdout documents.""" - schemas = [ - shape.treaty_schema() if self.treaty else shape.json_schema() - for shape in self.output - ] + if self.treaty: + return self._treaty_shape().treaty_schema() + schemas = [shape.json_schema() for shape in self.output] if len(schemas) == 1: return schemas[0] return {"oneOf": schemas} + def _treaty_shape(self) -> OutputShape: + """Merge the shapes: treaty writes one object, a missing key as null.""" + keys: dict[str, OutputKey] = {} + for shape in self.output: + for key in shape.keys: + keys.setdefault(key.name, key) + if len(self.output) == 1: + return self.output[0] + return OutputShape(keys=tuple(keys.values())) + _YES = "`--yes`; without it the command validates and prints the plan only" @@ -385,6 +394,7 @@ def _plan(subject: str, *extra: str) -> OutputShape: when="`--format json`, the default; `--format html` is not JSON", ), ), + treaty=True, ), CommandContract( "cloudfall", @@ -403,6 +413,7 @@ def _plan(subject: str, *extra: str) -> OutputShape: when="`detail`, the end of Ansible's output, when the playbook failed", ), ), + treaty=True, ), CommandContract( "cloudfall", @@ -467,6 +478,7 @@ def _plan(subject: str, *extra: str) -> OutputShape: CommandEffect.READ, "map a `render.yaml` blueprint onto config fragments under `tmp/`", output=_IMPORTED, + treaty=True, ), CommandContract( "cloudfall", @@ -474,6 +486,7 @@ def _plan(subject: str, *extra: str) -> OutputShape: CommandEffect.READ, "map a live Render workspace onto config fragments under `tmp/`", output=_IMPORTED, + treaty=True, ), CommandContract( "cloudfall", @@ -492,6 +505,7 @@ def _plan(subject: str, *extra: str) -> OutputShape: "receipt", ), ), + treaty=True, ), CommandContract( "cloudfall", @@ -542,6 +556,7 @@ def _plan(subject: str, *extra: str) -> OutputShape: _plan("component", "release"), _shape(*_EXECUTED, "release", "receipt", when="with `--yes`"), ), + treaty=True, ), CommandContract( "cloudfall", @@ -553,6 +568,7 @@ def _plan(subject: str, *extra: str) -> OutputShape: _plan("component", "release"), _shape(*_EXECUTED, "release", when="with `--yes`"), ), + treaty=True, ), CommandContract( "cloudfall", @@ -561,6 +577,7 @@ def _plan(subject: str, *extra: str) -> OutputShape: "restart one component behind its health check", gate=_YES, output=(_plan("component"), _shape(*_EXECUTED, when="with `--yes`")), + treaty=True, ), CommandContract( "cloudfall", @@ -581,6 +598,7 @@ def _plan(subject: str, *extra: str) -> OutputShape: when="with `--yes`", ), ), + treaty=True, ), CommandContract( "cloudfall", diff --git a/sdk/src/cloudfall/render_api.py b/sdk/src/cloudfall/render_api.py index deac253..9b5ede4 100644 --- a/sdk/src/cloudfall/render_api.py +++ b/sdk/src/cloudfall/render_api.py @@ -26,7 +26,7 @@ from cloudfall.importer import ImportTargets, RenderImportResult -_ERROR_API_UNREACHABLE = "render_api_unreachable" +ERROR_API_UNREACHABLE = "render_api_unreachable" _ERROR_API_INVALID = "render_api_invalid" _PAGE_LIMIT = 100 _SERVICE_TYPES: Mapping[str, str] = { @@ -137,7 +137,7 @@ def _get(self, path: str) -> object: body = response.read() except OSError as error: message = f"Render API unreachable: {url}: {error}" - raise RenderImportError(_ERROR_API_UNREACHABLE, message) from error + raise RenderImportError(ERROR_API_UNREACHABLE, message) from error try: return cast("object", json.loads(body.decode("utf-8"))) except json.JSONDecodeError as error: diff --git a/sdk/src/cloudfall/secrets.py b/sdk/src/cloudfall/secrets.py index 2bef85f..e37ca62 100644 --- a/sdk/src/cloudfall/secrets.py +++ b/sdk/src/cloudfall/secrets.py @@ -30,11 +30,11 @@ from cloudfall.inventory import ComponentInventory, SecretReference from cloudfall.lifecycle import EngineContext -_ERROR_SOPS_MISSING = "secrets_sops_missing" +ERROR_SOPS_MISSING = "secrets_sops_missing" _ERROR_SOURCE_MISSING = "secrets_source_missing" _ERROR_SOURCE_INVALID = "secrets_source_invalid" _ERROR_COMPONENT_UNKNOWN = "secrets_component_unknown" -_ERROR_DECRYPT_FAILED = "secrets_decrypt_failed" +ERROR_DECRYPT_FAILED = "secrets_decrypt_failed" _ERROR_KEY_CONFLICT = "secrets_key_conflict" _RECEIPT_SCHEMA = "environment-receipt.schema.json" _KEY_PATTERN = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") @@ -230,7 +230,7 @@ def _sops_decrypt(source: Path) -> str: "the sops binary is not installed; install sops and age, and " "set SOPS_AGE_KEY_FILE to your age key" ) - raise SecretsError(_ERROR_SOPS_MISSING, message) + raise SecretsError(ERROR_SOPS_MISSING, message) completed = subprocess.run( # noqa: S603 - fixed binary, declared file [binary, "--decrypt", str(source)], capture_output=True, @@ -242,7 +242,7 @@ def _sops_decrypt(source: Path) -> str: f"sops could not decrypt {source}: " f"{completed.stderr.strip()[:300]}" ) - raise SecretsError(_ERROR_DECRYPT_FAILED, message) + raise SecretsError(ERROR_DECRYPT_FAILED, message) return completed.stdout diff --git a/sdk/src/cloudfall/why.py b/sdk/src/cloudfall/why.py index b7b0525..f1da1a9 100644 --- a/sdk/src/cloudfall/why.py +++ b/sdk/src/cloudfall/why.py @@ -18,13 +18,13 @@ from dataclasses import dataclass from datetime import UTC, datetime from html import escape -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from cloudfall.decision import DecisionStatus, Requirement from cloudfall.domain import Hostname, ResourceId if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Iterable, Mapping, Sequence from cloudfall.decision import Decision, DecisionStore, RunRecord @@ -234,17 +234,28 @@ def moments_of(decision: Decision) -> tuple[Instant, ...]: def render_why_html(result: WhyAnswer) -> str: """Render the answer as one page a person reads without the JSON.""" - cards = "".join(_render_card(entry) for entry in result.explanations) + return render_why_document(result.as_dict()) + + +def render_why_document(document: Mapping[str, object]) -> str: + """Render the answer's JSON document as one page, as ``--format html`` does. + + The page is drawn from the document alone, so treaty's renderer, which + receives the answer's ``data``, draws the same page as the object would. + """ + answers = cast("Sequence[Mapping[str, object]]", document["answers"]) + query = cast("Mapping[str, object]", document["query"]) + cards = "".join(_render_card(entry) for entry in answers) if not cards: cards = ( '

The record holds no decision this question is about.

' ) filters = ", ".join( f"{escape(key)} = {escape(str(value))}" - for key, value in result.query.as_dict().items() + for key, value in query.items() ) asked = escape(filters) if filters else "Every decision the record holds" - count = len(result.explanations) + count = len(answers) counted = f"{count} decision{'' if count == 1 else 's'}" return f""" @@ -285,7 +296,7 @@ def render_why_html(result: WhyAnswer) -> str:

Why did the agent do that

{asked} · {counted} - · from {escape(result.directory)}

+ · from {escape(str(document["directory"]))}

{cards}