diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..e494300 --- /dev/null +++ b/.env.example @@ -0,0 +1,19 @@ +# --------------------------------------------------------------------------- +# cb-testing Docker image overrides +# --------------------------------------------------------------------------- +# Copy this file to .env and customize for your local images. +# All values have defaults in generate_kurtosis_configs.py, so this file +# is entirely optional — only needed when using custom Docker images. +# --------------------------------------------------------------------------- + +# Helix relay image (your custom relay build) +# Default: helix-relay:kurtosis +# HELIX_RELAY_IMAGE=helix-relay:kurtosis + +# mev-boost relay image (used in multi-relay scenarios) +# Default: ethpandaops/mev-boost-relay:main +# MEV_RELAY_IMAGE=ethpandaops/mev-boost-relay:main + +# Commit-Boost PBS image (sub latest for your local build) +# Default: commit-boost/commit-boost:latest +# MEV_BOOST_IMAGE=commit-boost/commit-boost:latest diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6b843d9..4cac47b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,20 +1,19 @@ name: Integration Test on: + schedule: + # Runs every day at 00:00 UTC + - cron: '0 0 * * *' workflow_dispatch: inputs: - cb_ref: - description: 'Commit-Boost branch/commit to test' - required: false - default: 'main' target_epoch: description: 'Target epoch for verification' required: false - default: '7' + default: '2' min_epochs: description: 'Observation window in epochs' required: false - default: '2' + default: '1' env: ENCLAVE: cb-ci-${{ github.run_id }} @@ -28,6 +27,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + submodules: recursive - name: Setup Rust uses: dtolnay/rust-toolchain@master @@ -45,63 +46,25 @@ jobs: - name: Build cb-verify run: just build-release - - name: Checkout Commit-Boost - uses: actions/checkout@v6 - with: - repository: commit-boost/commit-boost-client - ref: ${{ github.event.inputs.cb_ref }} - path: commit-boost-client - - - name: Build CB Docker image - working-directory: commit-boost-client - run: | - docker build \ - -t commit-boost/pbs:kurtosis \ - -f Dockerfile \ - --build-arg BIN=pbs \ - . - - - name: Generate CI config - run: | - PLAYBOOK_URL="https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/assertoor/cb-mev-pipeline.yaml" - sed "s|http://host.docker.internal:8888/assertoor/cb-mev-pipeline.yaml|${PLAYBOOK_URL}|" \ - configs/assertoor-pbs.yml > configs/assertoor-ci.yml - echo "Playbook URL: $PLAYBOOK_URL" - - name: Install Kurtosis + shell: bash run: | - echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | \ + echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" | \ sudo tee /etc/apt/sources.list.d/kurtosis.list sudo apt update sudo apt install -y kurtosis-cli + kurtosis analytics disable + echo "$(dirname $(which kurtosis))" >> $GITHUB_PATH - - name: Launch devnet + - name: Launch devnet + verify timeout-minutes: 60 run: | - kurtosis run github.com/ethpandaops/ethereum-package \ - --enclave "$ENCLAVE" \ - --args-file configs/assertoor-ci.yml \ - --image-download always \ - --non-blocking-tasks 2>&1 | tee kurtosis-run.log - - - name: Verify - timeout-minutes: 30 - run: | - cargo run --release -- \ - --enclave "$ENCLAVE" \ - --target-epoch "${{ github.event.inputs.target_epoch }}" \ + ./scripts/run-and-verify.sh \ + --config configs/example-kurtosis-config.yml \ + --json \ + --live-metrics \ --min-epochs "${{ github.event.inputs.min_epochs }}" \ - --timeout 3600 \ - --json > verify-report.json 2>verify-err.log - echo "exit_code=$?" >> "$GITHUB_OUTPUT" - - - name: Report - if: always() - run: | - echo "=== Verification Report ===" - if [ -f verify-report.json ]; then - python3 -m json.tool verify-report.json || cat verify-report.json - fi + --target-epoch "${{ github.event.inputs.target_epoch }}" \ - name: Upload artifacts if: always() @@ -110,7 +73,6 @@ jobs: name: verify-${{ github.run_id }} path: | verify-report.json - verify-err.log kurtosis-run.log - name: Teardown diff --git a/.gitignore b/.gitignore index a29b902..4bc7c38 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,8 @@ *.swp *.swo .DS_Store +configs/generated +kurtosis-configs/ +scripts/kurtosis-configs/ +.env +*/__pycache__ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..bea06ae --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ethereum-package"] + path = ethereum-package + url = https://github.com/JasonVranek/ethereum-package.git diff --git a/Cargo.lock b/Cargo.lock index 3ae42ec..24d22d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -684,6 +684,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "serde_yaml", "tokio", "tracing", "tracing-subscriber", @@ -2701,6 +2702,19 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "serdect" version = "0.2.0" @@ -3238,6 +3252,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 2c9d600..87b31ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "cb-testnet-verifier" +default-run = "cb-verify" version = "0.1.0" edition = "2024" rust-version = "1.91" @@ -10,6 +11,19 @@ license = "MIT" name = "cb-verify" path = "src/main.rs" +[[bin]] +name = "cb-orchestrator" +path = "src/orchestrator.rs" + +[[bin]] +name = "test-mux" +path = "src/bin/test_mux.rs" + +[[bin]] +name = "test-relay" +path = "src/bin/test_relay.rs" + + [dependencies] alloy-primitives = "^1.3.1" alloy-rpc-types-beacon = "^1.0.35" @@ -25,6 +39,7 @@ serde_json = "1" tokio = { version = "1", features = ["full"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +serde_yaml = "0.9.34" [profile.release] strip = true diff --git a/README.md b/README.md index 4a9d532..2e0379b 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,6 @@ Automated verification for [Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) Kurtosis devnets. Spins up a local Ethereum testnet with Commit-Boost as the MEV sidecar, waits for the MEV pipeline to stabilize, verifies each stage of the pipeline, and reports pass/fail. -## What is this? - -When Commit-Boost proxies MEV between validators and relays, several things need to work in sequence: validators register with the relay, the builder submits blocks, the relay serves headers through CB, and the delivered payloads land on chain. If any stage breaks, the rest falls apart silently. - -This tool verifies each stage independently and tells you exactly what broke. - -``` -Validator -> CB (get_header) -> Relay -> Builder -Validator -> CB (submit_block) -> Relay -> Payload on chain - ^ - we verify this whole flow -``` - ## Prerequisites - [Kurtosis CLI](https://docs.kurtosis.com/install) (>= 0.90) @@ -24,39 +11,83 @@ Validator -> CB (submit_block) -> Relay -> Payload on chain If you're testing a local CB build, you also need: - The [commit-boost-client](https://github.com/Commit-Boost/commit-boost-client) repo cloned -## Quick start +## Docker image configuration + +The generated configs embed Docker images for the relay, PBS sidecar, builder CL, +and builder EL. These are hardcoded by default but can be overridden via `.env`: ```bash -# Run with the default config (2 Lighthouse/Geth nodes) -# (builds automatically on first run via cargo) -./scripts/run-and-verify.sh - -# Takes ~55 minutes. You'll see: -# [PASS] chain_finality - Finalized epoch: 7 -# [PASS] relay.payloads_delivered_multi - Delivered 64 payload(s) -# [PASS] payload_hash_match - 64 matched, 0 mismatched -# Result: PASS (8 passed, 0 failed, 0 warnings) +# From the cb-testing/ directory: +cp .env.example .env +# Edit .env to point at your local images ``` -### Testing a local CB build +| Variable | Default | Purpose | +|---|---|---| +| `HELIX_RELAY_IMAGE` | `helix-relay:kurtosis` | Custom Helix relay image | +| `MEV_RELAY_IMAGE` | `ethpandaops/mev-boost-relay:main` | mev-boost relay (multi-relay scenarios) | +| `MEV_BOOST_IMAGE` | `commit-boost/pbs:kurtosis` | Commit-Boost PBS image | +| `BUILDER_CL_IMAGE` | `sigp/lighthouse:latest` | Builder consensus client | +| `BUILDER_EL_IMAGE` | `ethpandaops/reth-rbuilder:develop` | Builder execution client | + +The `.env` file is read automatically by `generate_kurtosis_configs.py`. +It is gitignored — do not commit it. Use `.env.example` as the reference. -```bash -# In the commit-boost-client repo, build the PBS Docker image: -just build-pbs kurtosis +## Kurtosis setup / gotchas + +The repo contains a forked `ethereum-package` as a submodule: -# Then run the verifier (it uses the commit-boost/pbs:kurtosis image): -./scripts/run-and-verify.sh --config configs/basic-pbs.yml +```bash +git submodule update --init ``` -### Testing against a specific ethereum-package +The fork generalizes hardcoded patterns from upstream, enabling configs like commit-boost + helix that weren't possible before. Once [this PR](https://github.com/ethpandaops/ethereum-package/pull/1384) merges we can deprecate the fork. + +### Kurtosis configs -If you have a local checkout of the [ethereum-package](https://github.com/ethpandaops/ethereum-package) (e.g., with the [custom CB config PR](https://github.com/ethpandaops/ethereum-package/pull/1355)): +Kurtosis uses a default Commit-Boost config that can be overridden by inlining it into the kurtosis config — see `configs/example-kurtosis-config.yml`. Every generated test config uses this pattern. + +`generate_kurtosis_configs.py` generates test scenarios from `.env`: ```bash -./scripts/run-and-verify.sh --package ../ethereum-package --keep -v +just generate-configs ``` -The `--keep` flag leaves the enclave running so you can inspect it after. The `-v` flag enables debug logging. +Six scenarios are generated: + +| Config | What it tests | +|---|---| +| `cb-basic.yml` | Single relay (helix), default CB config | +| `cb-multiple-relays.yml` | Two relays (helix + flashbots), aggregated bidding | +| `cb-mux.yml` | Mux routing — 128 validators to helix, 128 to flashbots | +| `cb-skip-sigverify.yml` | Fast path with BLS signature verification disabled | +| `cb-timing-games.yml` | Aggressive per-relay timing overrides for late bidding | +| `cb-extra-validation.yml` | Extra get_header validation via local EL RPC | + +## Quick start + +```bash +# Generate configs from .env +just generate-configs + +# Launch a testnet + verify (attached mode) +just testnet configs/generated/cb-mux.yml + +# Verify a running enclave (standalone, no observation window) +just verify-now CB-Testnet + +# Verify with mux routing checks +just verify-with-config configs/generated/cb-mux.yml CB-Testnet + +# Show raw CB PBS logs for debugging +just show-logs CB-Testnet + +# Quick mux routing diagnostic +just test-mux CB-Testnet configs/generated/cb-mux.yml + +# Test relay API endpoints +cargo run --release --bin test-relay -- http://127.0.0.1:PORT 128 160 +``` ## What it checks @@ -67,8 +98,9 @@ The `--keep` flag leaves the enclave running so you can inspect it after. The `- | `chain_finality` | Beacon chain has finalized (epoch >= 2) | | `sync_status` | Beacon node is not syncing | | `cb_running` | Commit-Boost services are running in the enclave | -| `relay.payloads_delivered_multi` | The relay delivered payloads to proposers | -| `payload_hash_match` | Every delivered payload's block_hash matches on chain | +| `relay.payloads_delivered_multi` | Relays delivered payloads to proposers | +| `payload_hash_match` | Delivered payload block_hashes match on-chain | +| `mux.routing` | Validator pubkeys routed to correct relay (config-dependent) | ### Tier 2: Quality metrics (should pass) @@ -77,206 +109,109 @@ The `--keep` flag leaves the enclave running so you can inspect it after. The `- | `missed_slots` | Missed slot rate in observation window | < 10% | | `relay.builder_blocks_received` | Builder submitted blocks to relay | > 0 | | `relay.mev_delivery_rate` | Slots using relay-built blocks vs local | >= 30% | -| `cb_relay_latency` | Mean get_header latency | < 500ms | -| `cb_relay_errors` | Relay 5xx error count | == 0 | -| `cb_header_values` | Relay header bid values | > 0 | -| `cb_get_header_success` | Successful get_header responses | > 0 | - -Tier 2 metric checks (latency, errors, header values, get_header success) require CB metrics. See [Metrics limitations](#metrics-limitations) below. +| `relay.validator_registrations` | All validators registered on relay | 100% | -### Tier 3: Extended checks (config-dependent) +### Tier 3: CB metrics -| Check | When it runs | +| Check | What it verifies | |---|---| -| `relay.validator_registrations` | When validator pubkeys are provided | - -Mux routing, SSZ encoding, and signer health checks are planned but not yet implemented. - -## Config presets - -The `configs/` directory contains ready-to-use Kurtosis config files: - -| Config | Nodes | What it exercises | -|---|---|---| -| [`basic-pbs.yml`](configs/basic-pbs.yml) | 2x Lighthouse/Geth | Default PBS pipeline. Header selection, relay fan-out. Start here. | -| [`pbs-metrics.yml`](configs/pbs-metrics.yml) | 2x Lighthouse/Geth | PBS + `[metrics]` enabled + Prometheus. Exercises Tier 2 checks. | -| [`pbs-validation-modes.yml`](configs/pbs-validation-modes.yml) | 3x Lighthouse/Geth | Mux with different `header_validation_mode` settings (None vs Standard). | -| [`assertoor-pbs.yml`](configs/assertoor-pbs.yml) | 2x Lighthouse/Geth | CI integration. Same checks run inside assertoor via `run_shell`. | - -Most presets use `commit_boost_config` to inject a full CB config as inline TOML (`basic-pbs.yml` uses the default template). Template variables (`{{ .Network }}`, `{{ .Port }}`, `{{ .Relays }}`, `{{ .Timestamp }}`) are injected by the ethereum-package at launch. - -Additional reference configs for SSZ testing, client matrix testing, and mux experiments are in `configs/reference/`. - -### Writing your own config - -Start from `basic-pbs.yml` and modify the `commit_boost_config` block. The config is standard CB TOML with template variable injection. Key things to know: - -- `{{ .Port }}` becomes the PBS listen port (18550) -- `{{ .Relays }}` is the list of relay URLs discovered by the ethereum-package -- `{{ .Network }}` is the path to the network config inside the container -- `{{ .Timestamp }}` is the genesis time -- `spamoor` in `additional_services` is required (generates transactions so the builder has something to build) +| `cb_get_header_matrix` | get_header status codes from relay vs beacon side | +| `cb_register_validator_matrix` | register_validator status codes | +| `cb_submit_blinded_block_matrix` | submit_blinded_block status codes | +| `cb_status_matrix` | status check responses | +| `cb_v2_fallback` | v1→v2 fallback behavior | +| `cb_relay_latency` | get_header latency histogram (p95) | ## CLI reference ### cb-verify ``` -cargo run --release -- --enclave CB-Testnet [OPTIONS] +cb-verify [OPTIONS] Options: - --enclave NAME Kurtosis enclave name (required) - --min-epochs N Observation window in epochs (default: 2) - --target-epoch N Wait until this epoch before checks (default: 7) - --timeout SECS Readiness timeout in seconds (default: 1500) - --mev-threshold PCT Min MEV delivery rate, 0.0-1.0 (default: 0.30) - --json Output JSON report - -v, --verbose Debug logging + --enclave Kurtosis enclave name (required) + --config Kurtosis YAML config (for mux verification) + --min-epochs Observation window in epochs [default: 2] + --target-epoch Wait before starting checks [default: 7] + --timeout Readiness timeout [default: 3600] + --mev-threshold Min MEV delivery rate [default: 0.30] + --json Output JSON report + --verbose Debug logging + --strict Promote WARN to FAIL + --live-metrics Poll :9090/metrics during observation + --show-logs Print raw CB PBS logs, no checks + --output-dir Save JSON reports (requires --json) ``` -### run-and-verify.sh +### test-mux ``` -./scripts/run-and-verify.sh [OPTIONS] - -Options: - --config FILE Kurtosis config (default: configs/basic-pbs.yml) - --enclave NAME Enclave name (default: CB-Testnet) - --package PATH ethereum-package: local path or GitHub ref - --keep Don't tear down the enclave on exit - --json JSON output - --timeout SECS Readiness timeout (default: 1500) - --min-epochs N Observation window (default: 2) - -v, --verbose Debug logging -``` - -## How it works - -1. **Discovery**: Finds beacon, relay, and CB services via `kurtosis enclave inspect` -2. **Readiness**: Polls beacon API until synced, finalizing, and past epoch 7 (no hardcoded sleeps) -3. **Observation**: Watches 2 more epochs of steady-state activity -4. **Checks**: Queries beacon API, relay data API, and (optionally) CB metrics -5. **Report**: Colored terminal output or JSON (`--json`). Exit code 0/1/2. - -### Exit codes - -- `0` All Tier 1 checks passed -- `1` One or more Tier 1 checks failed -- `2` Setup failure (enclave not found, timeout, etc.) - -### Timing - -With the default `seconds_per_slot: 12` (matching mainnet), expect ~55 minutes total: -- ~45 min for the devnet to reach epoch 7 and finalize -- ~8 min for the 2-epoch observation window - -## Metrics limitations - -CB's metrics server only starts when the `CB_METRICS_PORT` environment variable is set. This env var is normally injected by `cb docker init` when generating docker-compose files, but upstream `ethpandaops/ethereum-package` doesn't set it. The TOML `[metrics]` block in `commit_boost_config` alone is ignored by PBS standalone -- see `crates/metrics/src/provider.rs` in commit-boost-client. - -### Fixed in local ethereum-package fork - -The `JasonVranek/ethereum-package` fork patches this in `src/mev/commit-boost/mev_boost/mev_boost_launcher.star`: -- Adds `metrics: 9090/tcp` to `USED_PORTS` -- Adds `CB_METRICS_PORT: "9090"` to `env_vars` +test-mux -Run against the local fork to enable metrics: - -```bash -./scripts/run-and-verify.sh --package ../ethereum-package --config configs/assertoor-pbs.yml +Fetches CB PBS logs, parses mux events, verifies routing against config. +No observation window. Completes in seconds. ``` -With the fork, the `cb_*_matrix` checks populate with per-endpoint, per-relay status code counts, and the `cb_relay_latency` p95 histogram check activates. Using upstream ethereum-package, these tier 2 checks SKIP. All tier 1 checks (chain health, relay pipeline, payload matching) work either way. - -## Assertoor integration (CI) - -For CI pipelines, the verification checks run inside [assertoor](https://github.com/ethpandaops/assertoor) as an in-enclave service. This avoids the need for a separate verification container or external polling. - -The test definition lives at [`assertoor/cb-mev-pipeline.yaml`](assertoor/cb-mev-pipeline.yaml). It uses assertoor's native tasks for chain readiness (finality, sync) and `run_shell` tasks with `curl`+`jq` for relay and payload checks. The assertoor container has both tools installed. +### test-relay -### How it works - -1. The kurtosis config (`configs/assertoor-pbs.yml`) adds assertoor to `additional_services` -2. `assertoor_params.tests` references the test YAML via raw GitHub URL -3. Assertoor fetches the test at startup and runs it inside the enclave -4. The [kurtosis-assertoor-github-action](https://github.com/ethpandaops/kurtosis-assertoor-github-action) polls assertoor's API and reports pass/fail - -### GitHub Actions example - -```yaml -jobs: - cb-mev-pipeline: - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - uses: ethpandaops/kurtosis-assertoor-github-action@v1 - with: - ethereum_package_args: configs/assertoor-pbs.yml ``` +test-relay [pubkey] -### Service discovery inside the enclave - -The test YAML defaults to standard kurtosis DNS names: -- Beacon: `http://cl-1-lighthouse-geth:4000` -- Relay: `http://mev-relay-api:9062` - -If your enclave uses different CL/EL types, override via `testConfig` in `assertoor_params.tests`: - -```yaml -assertoor_params: - tests: - - file: "https://raw.githubusercontent.com/jvranek/cb-testing/main/assertoor/cb-mev-pipeline.yaml" - testConfig: - beaconUrl: "http://cl-1-prysm-geth:3500" +Tests relay data API endpoints with slot filtering. +Verifies delivered payloads, builder blocks, validator registration. ``` -### Local assertoor testing - -To test the assertoor flow locally without pushing to GitHub: - -```bash -# Serve the test YAML locally -cd assertoor && python3 -m http.server 8888 & - -# Point assertoor at the local URL (edit assertoor-pbs.yml temporarily) -# Change the test URL to: http://host.docker.internal:8888/cb-mev-pipeline.yaml -kurtosis run github.com/ethpandaops/ethereum-package \ - --enclave CB-Testnet --args-file configs/assertoor-pbs.yml - -# Check assertoor status -ASSERTOOR=$(kurtosis port print CB-Testnet assertoor http) -curl -s $ASSERTOOR/api/v1/test_runs | jq '.[0].status' -``` +## How it works -For iterating on checks locally, the standalone verifier (`./scripts/run-and-verify.sh`) is faster. +**Attached mode** (`just testnet`): +1. `run-and-verify.sh` launches Kurtosis enclave with the chosen config +2. `cb-verify` waits for chain readiness (target epoch, finalization) +3. Observes for `min_epochs` while polling health +4. Runs all tier checks, outputs report + +**Standalone mode** (`just verify-now`): +1. Discovers services in running enclave via `kurtosis enclave inspect` +2. Skips observation window (`--min-epochs 0`) +3. Runs checks against current state + +**Mux verification** (`test-mux` or `--config`): +1. Parses Kurtosis YAML → extracts `commit_boost_config` → finds `[[mux]]` sections +2. Fetches CB PBS logs via `kurtosis service logs` +3. Parses log lines (ANSI-aware) for `using mux config` events +4. Cross-references proposer pubkeys against mux mapping +5. FAIL if any pubkey appears on wrong relay + +**Metrics** (tier 3): +1. Discovers metrics URL from `kurtosis port print` (port 9090) +2. Falls back to `kurtosis exec` into CB container +3. Parses Prometheus text format +4. Checks status code matrices, latency histograms ## Project layout ``` -src/ - main.rs CLI entry point and orchestrator (clap) - beacon.rs Beacon API client (alloy types) - relay.rs Relay Data API client (alloy types) - discovery.rs Kurtosis service/port discovery - metrics.rs Prometheus metrics fetching - report.rs Terminal (ANSI) and JSON output formatting - checks/ - mod.rs CheckResult, CheckStatus types - chain_health.rs Finality, sync, missed slots, CB service status - relay_pipeline.rs Builder blocks, delivered payloads, MEV delivery rate - payload_matching.rs Cross-ref relay payloads with on-chain blocks - cb_metrics.rs CB Prometheus metric assertions -assertoor/ - cb-mev-pipeline.yaml Assertoor test definition for CI -configs/ - basic-pbs.yml Default PBS preset - pbs-metrics.yml PBS + metrics preset - pbs-validation-modes.yml Mux validation mode preset - assertoor-pbs.yml CI preset with assertoor - reference/ Additional configs for SSZ, client matrix, etc. -scripts/ - run-and-verify.sh Full lifecycle: launch, verify, tear down -Cargo.toml Rust project config -PLAN.md Design doc and roadmap +cb-testing/ + Cargo.toml # Workspace: cb-verify, test-mux, test-relay + justfile # Build/test/launch commands + README.md + .env.example # Docker image overrides + scripts/ + run-and-verify.sh # Attached mode launcher + generate_kurtosis_configs.py # Config generator + configs/ + generated/ # Pre-generated test scenarios + example-kurtosis-config.yml + src/ + main.rs # cb-verify binary + checks/ + chain_health.rs # Finality, sync, missed slots + relay_pipeline.rs # Delivery, registration, MEV rate + payload_matching.rs # Hash matching + mux_routing.rs # Mux config parsing, log analysis + cb_metrics.rs # Prometheus metrics checks + bin/ + test_mux.rs # Mux diagnostic binary + test_relay.rs # Relay API diagnostic binary + ethereum-package/ # Forked Kurtosis package (submodule) ``` diff --git a/configs/assertoor-pbs.yml b/configs/assertoor-pbs.yml deleted file mode 100644 index 5647388..0000000 --- a/configs/assertoor-pbs.yml +++ /dev/null @@ -1,91 +0,0 @@ -# cb-testnet-verifier preset: Assertoor PBS Pipeline + Metrics -# -# Runs CB MEV pipeline checks inside assertoor for CI integration, AND -# enables CB's Prometheus metrics endpoint so cb-verify's status-code -# matrix checks populate when run against the same enclave. -# -# Two verifiers, one enclave, same invariants. Run assertoor for CI-blessed -# pass/fail, then run cb-verify against the same enclave to get the full -# status-code matrix + latency histograms. -# -# Usage (local): -# kurtosis run github.com/ethpandaops/ethereum-package \ -# --enclave CB-Testnet --args-file configs/assertoor-pbs.yml -# -# # Check assertoor status: -# ASSERTOOR=$(kurtosis port print CB-Testnet assertoor http) -# curl -s $ASSERTOOR/api/v1/test_runs | jq '.[0].status' -# -# # Run cb-verify against the same enclave (metrics now populated): -# ./scripts/run-and-verify.sh --config configs/assertoor-pbs.yml -# -# Usage (CI with GitHub Action): -# uses: ethpandaops/kurtosis-assertoor-github-action@v1 -# with: -# ethereum_package_args: configs/assertoor-pbs.yml - -participants: - - el_type: geth - cl_type: lighthouse - count: 2 - -additional_services: - - spamoor - - dora - - assertoor - - prometheus - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - # Enable CB metrics so cb-verify's status-code matrix populates. - # The kurtosis ethereum-package doesn't publish port 9090 to the host - # by default, so cb-verify falls back to `kurtosis service exec` + curl - # inside the container to scrape. See checks/cb_metrics.rs. - commit_boost_config: | - chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } - - [pbs] - host = "0.0.0.0" - port = {{ .Port }} - timeout_get_header_ms = 950 - timeout_get_payload_ms = 4000 - late_in_slot_time_ms = 2000 - - {{ range $index, $relay := .Relays }} - [[relays]] - id = "mev_relay_{{$index}}" - url = "{{ $relay }}" - {{- end }} - - [metrics] - host = "0.0.0.0" - port = 9090 - - [logs.stdout] - level = "debug" - - [logs.file] - enabled = false - -assertoor_params: - run_stability_check: false - run_block_proposal_check: false - tests: - # For CI: use the raw GitHub URL (uncomment and update branch/tag): - # - file: "https://raw.githubusercontent.com/jvranek/cb-testing/main/assertoor/cb-mev-pipeline.yaml" - # For local testing: serve the file and use host.docker.internal: - - file: "http://host.docker.internal:8888/assertoor/cb-mev-pipeline.yaml" - -network_params: - network: kurtosis - seconds_per_slot: 12 - num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: - "giant issue aisle success illegal bike spike - question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy - very lucky have athlete" diff --git a/configs/basic-pbs.yml b/configs/basic-pbs.yml deleted file mode 100644 index 8b4537e..0000000 --- a/configs/basic-pbs.yml +++ /dev/null @@ -1,102 +0,0 @@ -# cb-testnet-verifier preset: Basic PBS Pipeline -# -# Tests the core MEV pipeline through Commit-Boost. -# -# Image pinning rationale (from the CB repo's working kurtosis config): -# - mev_boost_image = commit-boost/pbs:kurtosis -# CB-internal kurtosis build. Must be built locally from the -# commit-boost-client repo (e.g. `just kurtosis-image` or equivalent) -# before running. The ghcr :latest tag has historically been unstable. -# - mev_relay_image and mev_builder_image pinned to the pair known to -# co-operate. Rolling tags drift; this pair is what the CB repo runs. -# -# Usage: -# ./scripts/run-and-verify.sh --config configs/basic-pbs.yml - -participants: - - el_type: geth - cl_type: lighthouse - -additional_services: - - dora - - spamoor -mev_type: commit-boost - -mev_params: - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_boost_image: commit-boost/pbs:kurtosis - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - -network_params: - network: kurtosis - network_id: "3151908" - deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" - seconds_per_slot: 12 - slot_duration_ms: 12000 - num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: - "giant issue aisle success illegal bike spike - question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy - very lucky have athlete" - preregistered_validator_count: 0 - additional_mnemonics: [] - genesis_delay: 20 - genesis_time: 0 - genesis_gaslimit: 60000000 - max_per_epoch_activation_churn_limit: 8 - churn_limit_quotient: 65536 - confirmation_byzantine_threshold: 25 - ejection_balance: 16000000000 - eth1_follow_distance: 2048 - min_validator_withdrawability_delay: 256 - shard_committee_period: 256 - attestation_due_bps_gloas: 2500 - aggregate_due_bps_gloas: 5000 - sync_message_due_bps_gloas: 2500 - contribution_due_bps_gloas: 5000 - payload_attestation_due_bps: 7500 - view_freeze_cutoff_bps: 7500 - inclusion_list_submission_due_bps: 6667 - proposer_inclusion_list_cutoff_bps: 9167 - deneb_fork_epoch: 0 - electra_fork_epoch: 0 - fulu_fork_epoch: 0 - gloas_fork_epoch: 18446744073709551615 - network_sync_base_url: https://snapshots.ethpandaops.io/ - force_snapshot_sync: false - samples_per_slot: 8 - custody_requirement: 4 - max_blobs_per_block_electra: 9 - max_request_blocks_deneb: 128 - target_blobs_per_block_electra: 6 - base_fee_update_fraction_electra: 5007716 - additional_preloaded_contracts: {} - devnet_repo: ethpandaops - prefunded_accounts: {} - bpo_1_epoch: 0 - bpo_1_max_blobs: 15 - bpo_1_target_blobs: 10 - bpo_1_base_fee_update_fraction: 8346193 - bpo_2_epoch: 18446744073709551615 - bpo_2_max_blobs: 21 - bpo_2_target_blobs: 14 - bpo_2_base_fee_update_fraction: 11684671 - bpo_3_epoch: 18446744073709551615 - bpo_3_max_blobs: 0 - bpo_3_target_blobs: 0 - bpo_3_base_fee_update_fraction: 0 - bpo_4_epoch: 18446744073709551615 - bpo_4_max_blobs: 0 - bpo_4_target_blobs: 0 - bpo_4_base_fee_update_fraction: 0 - bpo_5_epoch: 18446744073709551615 - bpo_5_max_blobs: 0 - bpo_5_target_blobs: 0 - bpo_5_base_fee_update_fraction: 0 - withdrawal_type: "0x00" - withdrawal_address: "0x8943545177806ED17B9F23F0a21ee5948eCaa776" - validator_balance: 32 - min_epochs_for_data_column_sidecars_requests: 4096 - builder_count: 0 - builder_balance: 100 diff --git a/configs/pbs-metrics.yml b/configs/example-kurtosis-config.yml similarity index 62% rename from configs/pbs-metrics.yml rename to configs/example-kurtosis-config.yml index 748e103..7244098 100644 --- a/configs/pbs-metrics.yml +++ b/configs/example-kurtosis-config.yml @@ -1,32 +1,32 @@ -# cb-testnet-verifier preset: PBS + Metrics -# -# Same as basic-pbs but with CB metrics enabled and Prometheus in the enclave. -# Adds [metrics] block to the CB config via commit_boost_config inline TOML. -# -# NOTE: Tier 2 metric checks will still SKIP until the CB metrics env var -# issue is resolved (CB needs CB_METRICS_PORT env var to start the metrics -# server, which kurtosis doesn't set). See PLAN.md for details. + +# cb-basic: Single relay (helix) with default Commit-Boost config. # -# Usage: -# ./scripts/run-and-verify.sh --config configs/pbs-metrics.yml +# Tests the core MEV pipeline through Commit-Boost with a single Helix +# relay as the only relay endpoint. participants: - el_type: geth cl_type: lighthouse - count: 2 additional_services: - - spamoor - dora + - spamoor - prometheus -mev_type: commit-boost +mev_type: custom mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main + mev_boost_image: ghcr.io/commit-boost/pbs:latest mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + mev_builder_subsidy: 1 + + commit_boost_config: | chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } @@ -37,16 +37,16 @@ mev_params: timeout_get_payload_ms = 4000 late_in_slot_time_ms = 2000 + [metrics] + host = "0.0.0.0" + port = 9090 + {{ range $index, $relay := .Relays }} [[relays]] id = "mev_relay_{{$index}}" url = "{{ $relay }}" {{- end }} - [metrics] - host = "0.0.0.0" - port = 9090 - [logs.stdout] level = "debug" @@ -55,9 +55,13 @@ mev_params: network_params: network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" seconds_per_slot: 12 + slot_duration_ms: 12000 num_validator_keys_per_node: 128 preregistered_validator_keys_mnemonic: "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232e7efc77e1c79": {"balance": "100000ETH"}}' diff --git a/configs/pbs-validation-modes.yml b/configs/pbs-validation-modes.yml deleted file mode 100644 index c6e4fb4..0000000 --- a/configs/pbs-validation-modes.yml +++ /dev/null @@ -1,81 +0,0 @@ -# cb-testnet-verifier preset: Header Validation Modes -# -# Tests Commit-Boost's header validation modes using mux routing. -# The default path uses "standard" mode (full decode + signature verification). -# A mux route overrides to "none" mode (light path, minimal decoding). -# -# To use this config effectively, you need to populate the mux's -# validator_pubkeys with real pubkeys from the devnet. Steps: -# -# 1. Start the enclave with this config (mux pubkeys empty = no routing) -# 2. Query validator pubkeys: -# curl -s http:///eth/v1/beacon/states/head/validators \ -# | jq '.data[:5] | .[].validator.pubkey' -# 3. Paste those pubkeys into the [[mux]] validator_pubkeys list -# 4. Restart the enclave with the updated config -# -# Usage: -# ./scripts/run-and-verify.sh --config configs/pbs-validation-modes.yml --keep - -participants: - - el_type: geth - cl_type: lighthouse - count: 3 - -additional_services: - - spamoor - - dora - - prometheus - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - commit_boost_config: | - chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } - - [pbs] - host = "0.0.0.0" - port = {{ .Port }} - timeout_get_header_ms = 950 - timeout_get_payload_ms = 4000 - header_validation_mode = "standard" - - {{ range $index, $relay := .Relays }} - [[relays]] - id = "relay_{{$index}}" - url = "{{ $relay }}" - {{- end }} - - # Mux: route specific validators through "none" mode (light path). - # Replace the placeholder pubkey with real validator pubkeys from your devnet. - [[mux]] - id = "light_path" - validator_pubkeys = [] - timeout_get_header_ms = 500 - header_validation_mode = "none" - [[mux.relays]] - id = "mux_relay_0" - url = "{{ index .Relays 0 }}" - - [metrics] - host = "0.0.0.0" - port = 9090 - - [logs.stdout] - level = "debug" - - [logs.file] - enabled = false - -network_params: - network: kurtosis - seconds_per_slot: 12 - num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: - "giant issue aisle success illegal bike spike - question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy - very lucky have athlete" diff --git a/configs/reference/kurtosis-config.yml b/configs/reference/kurtosis-config.yml deleted file mode 100644 index 9658f7e..0000000 --- a/configs/reference/kurtosis-config.yml +++ /dev/null @@ -1,101 +0,0 @@ -# ELs: geth, nethermind, erigon, besu, reth, ethrex -# CLs: nimbus, lighthouse, lodestar, teku, prysm, and grandine -participants: - - el_type: geth - cl_type: nimbus - - - el_type: nethermind - cl_type: lighthouse - - - el_type: erigon - cl_type: lodestar - - - el_type: besu - cl_type: teku - - - el_type: reth - cl_type: prysm - - - el_type: ethrex - cl_type: grandine - -additional_services: - - dora - - spamoor -mev_type: commit-boost - -mev_params: - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_boost_image: commit-boost/pbs:kurtosis - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - -network_params: - network: kurtosis - network_id: "3151908" - deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" - seconds_per_slot: 12 - slot_duration_ms: 12000 - num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: - "giant issue aisle success illegal bike spike - question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy - very lucky have athlete" - preregistered_validator_count: 0 - additional_mnemonics: [] - genesis_delay: 20 - genesis_time: 0 - genesis_gaslimit: 60000000 - max_per_epoch_activation_churn_limit: 8 - churn_limit_quotient: 65536 - ejection_balance: 16000000000 - eth1_follow_distance: 2048 - min_validator_withdrawability_delay: 256 - shard_committee_period: 256 - attestation_due_bps_gloas: 2500 - aggregate_due_bps_gloas: 5000 - sync_message_due_bps_gloas: 2500 - contribution_due_bps_gloas: 5000 - payload_attestation_due_bps: 7500 - view_freeze_cutoff_bps: 7500 - inclusion_list_submission_due_bps: 6667 - proposer_inclusion_list_cutoff_bps: 9167 - deneb_fork_epoch: 0 - electra_fork_epoch: 0 - fulu_fork_epoch: 0 - gloas_fork_epoch: 18446744073709551615 - network_sync_base_url: https://snapshots.ethpandaops.io/ - force_snapshot_sync: false - samples_per_slot: 8 - custody_requirement: 4 - max_blobs_per_block_electra: 9 - max_request_blocks_deneb: 128 - target_blobs_per_block_electra: 6 - base_fee_update_fraction_electra: 5007716 - additional_preloaded_contracts: {} - devnet_repo: ethpandaops - prefunded_accounts: {} - bpo_1_epoch: 0 - bpo_1_max_blobs: 15 - bpo_1_target_blobs: 10 - bpo_1_base_fee_update_fraction: 8346193 - bpo_2_epoch: 18446744073709551615 - bpo_2_max_blobs: 21 - bpo_2_target_blobs: 14 - bpo_2_base_fee_update_fraction: 11684671 - bpo_3_epoch: 18446744073709551615 - bpo_3_max_blobs: 0 - bpo_3_target_blobs: 0 - bpo_3_base_fee_update_fraction: 0 - bpo_4_epoch: 18446744073709551615 - bpo_4_max_blobs: 0 - bpo_4_target_blobs: 0 - bpo_4_base_fee_update_fraction: 0 - bpo_5_epoch: 18446744073709551615 - bpo_5_max_blobs: 0 - bpo_5_target_blobs: 0 - bpo_5_base_fee_update_fraction: 0 - withdrawal_type: "0x00" - withdrawal_address: "0x8943545177806ED17B9F23F0a21ee5948eCaa776" - validator_balance: 32 - min_epochs_for_data_column_sidecars_requests: 4096 diff --git a/configs/reference/kurtosis-custom-config.yml b/configs/reference/kurtosis-custom-config.yml deleted file mode 100644 index 634d66e..0000000 --- a/configs/reference/kurtosis-custom-config.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Test config: exercises commit_boost_config inline override. -# Adds skip_sigverify and custom timeouts that the default template doesn't expose. -# -# Run against local ethereum-package with the feature branch: -# just kurtosis-restart /path/to/ethereum-package provisioning/kurtosis-custom-config.yml - -participants: - - el_type: geth - cl_type: lighthouse - count: 2 - -additional_services: - - dora - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - commit_boost_config: | - chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } - - [pbs] - host = "0.0.0.0" - port = {{ .Port }} - skip_sigverify = true - timeout_get_header_ms = 800 - timeout_get_payload_ms = 3000 - late_in_slot_time_ms = 1500 - - {{ range $index, $relay := .Relays }} - [[relays]] - id = "mev_relay_{{$index}}" - url = "{{ $relay }}" - {{- end }} - - [logs.stdout] - level = "debug" - - [logs.file] - enabled = false diff --git a/configs/reference/kurtosis-mux-test.yml b/configs/reference/kurtosis-mux-test.yml deleted file mode 100644 index 7543274..0000000 --- a/configs/reference/kurtosis-mux-test.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Test config: mux routing with single relay backing both entries -# Tests that CB parses [[mux]] correctly -# Both relay entries point at the same flashbots relay with different IDs. - -participants: - - el_type: geth - cl_type: lighthouse - count: 2 - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - commit_boost_config: | - chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } - - [pbs] - host = "0.0.0.0" - port = {{ .Port }} - skip_sigverify = true - timeout_get_header_ms = 950 - - {{ range $index, $relay := .Relays }} - [[relays]] - id = "relay_{{$index}}" - url = "{{ $relay }}" - {{- end }} - - [[mux]] - id = "fast_path" - validator_pubkeys = [ - "0x8000000000000000000000000000000000000000000000000000000000000000000000000000000000000000deadbeef", - ] - timeout_get_header_ms = 500 - late_in_slot_time_ms = 1000 - [[mux.relays]] - id = "mux_relay" - url = "{{ index .Relays 0 }}" - - [logs.stdout] - level = "debug" - - [logs.file] - enabled = false - -additional_services: - - dora diff --git a/configs/reference/kurtosis-ssz-lighthouse.yml b/configs/reference/kurtosis-ssz-lighthouse.yml deleted file mode 100644 index 21b9076..0000000 --- a/configs/reference/kurtosis-ssz-lighthouse.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Kurtosis config: SSZ get_header testing with Lighthouse -# Lighthouse sends Accept: application/octet-stream;q=1.0,application/json;q=0.9 by default. -# No extra flags needed. Use --builder-disable-ssz to force JSON-only. -# -# Prerequisites: -# just build-pbs kurtosis -# -# Run: -# kurtosis run github.com/ethpandaops/ethereum-package \ -# --enclave CB-Testnet \ -# --args-file provisioning/kurtosis-ssz-lighthouse.yml -# -# Verify SSZ is flowing: -# kurtosis service logs CB-Testnet commit-boost-001-lighthouse-geth 2>&1 | grep -i "content_type\|ssz\|encoding" - -participants: - - el_type: geth - cl_type: lighthouse - count: 2 - -additional_services: - - dora - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop diff --git a/configs/reference/kurtosis-ssz-matrix.yml b/configs/reference/kurtosis-ssz-matrix.yml deleted file mode 100644 index 50a1edd..0000000 --- a/configs/reference/kurtosis-ssz-matrix.yml +++ /dev/null @@ -1,60 +0,0 @@ -# Kurtosis config: SSZ encoding matrix across all CL clients -# Tests SSZ get_header behavior with every major CL implementation. -# -# SSZ builder API support by client (as of 2026-04): -# -# Lighthouse: SSZ preferred by default. --builder-disable-ssz to force JSON. -# Teku: SSZ preferred by default. No flag. Hardcoded. -# Nimbus: SSZ preferred by default. No flag. Hardcoded. -# Lodestar: SSZ preferred by default. No flag. Adaptive with 415 fallback. -# Prysm: JSON by default. Requires --enable-builder-ssz flag. -# Grandine: Unknown / not audited. -# -# All clients except Prysm send Accept: application/octet-stream with higher -# priority than application/json. Prysm is the only holdout requiring explicit opt-in. -# -# This config runs all SSZ-capable CLs to verify CB handles the full matrix. -# -# Prerequisites: -# just build-pbs kurtosis -# -# Run: -# kurtosis run github.com/ethpandaops/ethereum-package \ -# --enclave CB-Testnet \ -# --args-file provisioning/kurtosis-ssz-matrix.yml -# -# Verify per-client encoding: -# for svc in $(kurtosis enclave inspect CB-Testnet 2>&1 | grep commit-boost | awk '{print $1}'); do -# echo "=== $svc ==="; kurtosis service logs CB-Testnet $svc 2>&1 | grep "content_type" | tail -3 -# done - -participants: - # SSZ by default (no flags needed) - - el_type: geth - cl_type: lighthouse - - - el_type: nethermind - cl_type: teku - - - el_type: reth - cl_type: nimbus - - - el_type: erigon - cl_type: lodestar - - # SSZ requires explicit flag - - el_type: besu - cl_type: prysm - cl_extra_params: - - "--enable-builder-ssz" - -additional_services: - - dora - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop diff --git a/configs/reference/kurtosis-ssz-prysm.yml b/configs/reference/kurtosis-ssz-prysm.yml deleted file mode 100644 index 62f7e3a..0000000 --- a/configs/reference/kurtosis-ssz-prysm.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Kurtosis config: SSZ get_header testing with Prysm -# Prysm is the ONLY major CL that requires a flag for SSZ builder API support. -# Without --enable-builder-ssz, Prysm sends Accept: application/json only. -# -# Prerequisites: -# just build-pbs kurtosis -# -# Run: -# kurtosis run github.com/ethpandaops/ethereum-package \ -# --enclave CB-Testnet \ -# --args-file provisioning/kurtosis-ssz-prysm.yml -# -# Verify SSZ is flowing: -# kurtosis service logs CB-Testnet commit-boost-001-prysm-geth 2>&1 | grep -i "content_type\|ssz\|encoding" - -participants: - - el_type: geth - cl_type: prysm - cl_extra_params: - - "--enable-builder-ssz" - count: 2 - -additional_services: - - dora - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop diff --git a/ethereum-package b/ethereum-package new file mode 160000 index 0000000..4844f88 --- /dev/null +++ b/ethereum-package @@ -0,0 +1 @@ +Subproject commit 4844f884cb06daab30dd2cc1693328d55168720e diff --git a/justfile b/justfile index 8a7ead2..7efcaa5 100644 --- a/justfile +++ b/justfile @@ -32,9 +32,13 @@ ci: check test lint build-release: cargo build --release +# Build the orchestrator binary (concurrent multi-enclave test runner) +build-orchestrator: + cargo build --release --bin cb-orchestrator + # Run verifier against a running enclave verify enclave="CB-Testnet" target_epoch="7" min_epochs="2": - cargo run --release -- \ + cargo run --release --bin cb-verify -- \ --enclave {{enclave}} \ --target-epoch {{target_epoch}} \ --min-epochs {{min_epochs}} \ @@ -42,10 +46,83 @@ verify enclave="CB-Testnet" target_epoch="7" min_epochs="2": # Run verifier with live metrics and strict mode verify-strict enclave="CB-Testnet" target_epoch="7" min_epochs="2": - cargo run --release -- \ + cargo run --release --bin cb-verify -- \ --enclave {{enclave}} \ --target-epoch {{target_epoch}} \ --min-epochs {{min_epochs}} \ --timeout 3600 \ --live-metrics \ --strict + +# Standalone: quick health check (no observation window) +verify-now enclave="CB-Testnet": + cargo run --release --bin cb-verify -- \ + --enclave {{enclave}} \ + --min-epochs 0 \ + --timeout 60 + +# Standalone: verify with config (mux checks + health) +verify-with-config config enclave="CB-Testnet": + cargo run --release --bin cb-verify -- \ + --enclave {{enclave}} \ + --config {{config}} \ + --min-epochs 1 \ + --timeout 300 + +# Show raw CB PBS logs with parsing (for debugging) +show-logs enclave="CB-Testnet": + cargo run --release --bin cb-verify -- \ + --enclave {{enclave}} \ + --show-logs + +# Quick mux routing check (no observation window, just fetch logs and check) +test-mux enclave="CB-Testnet" config="configs/generated/cb-mux.yml": + cargo run --release --bin test-mux -- {{enclave}} {{config}} + +# Generate Kurtosis YAML configs from templates into configs/generated/ +# Loads optional .env for Docker image overrides (see .env.example). +generate-configs: + python3 scripts/generate_kurtosis_configs.py + +# Run kurtosis testnet with verification on target `config` +testnet config: + ./scripts/run-and-verify.sh \ + --config {{config}} \ + --json \ + --live-metrics \ + --min-epochs 1 \ + --target-epoch 2 \ + --keep \ + -v + +# Run a single config with verbose logging (for debugging) +testnet-verbose config: + ./scripts/run-and-verify.sh \ + --config {{config}} \ + --json \ + --live-metrics \ + --min-epochs 1 \ + --target-epoch 2 \ + --keep \ + -v + +# Run all generated configs concurrently and print a summary. +# +# Uses cb-orchestrator to run multiple enclaves in parallel (bounded by --jobs). +# Each config gets its own enclave. While one is observing, others can launch. +# For N configs with --jobs=4, expect roughly 4× throughput vs sequential. +# +# Usage: +# just test-all # default: 2 jobs, no results dir +# just test-all 4 /tmp/results # 4 jobs, save results to /tmp/results +# just test-all 2 /tmp/results --strict --keep +test-all jobs="2": + #!/usr/bin/env bash + set -euo pipefail + cargo run --release --bin cb-orchestrator -- --jobs {{jobs}} + +# Run a single config through the orchestrator (for debugging) +test-one config jobs="1": + cargo run --release --bin cb-orchestrator -- \ + --jobs {{jobs}} \ + {{config}} diff --git a/keys/README.md b/keys/README.md new file mode 100644 index 0000000..5e6921f --- /dev/null +++ b/keys/README.md @@ -0,0 +1,7 @@ +# Keys +These are the deterministically created public keys from mnemonic: +``` +"giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete" +``` + +There are 128 per file and they are used to configure the mux settings within the Commit-Boost configs. diff --git a/keys/node-0-pubkeys.json b/keys/node-0-pubkeys.json new file mode 100644 index 0000000..acc8669 --- /dev/null +++ b/keys/node-0-pubkeys.json @@ -0,0 +1 @@ +["0xaaf6c1251e73fb600624937760fef218aace5b253bf068ed45398aeb29d821e4d2899343ddcbbe37cb3f6cf500dff26c","0x8aa5bbee21e98c7b9e7a4c8ea45aa99f89e22992fa4fc2d73869d77da4cc8a05b25b61931ff521986677dd7f7159e8e6","0x996323af7e545fb6363ace53f1538c7ddc3eb0d985b2479da3ee4ace10cbc393b518bf02d1a2ddb2f5bdf09b473933ea","0xa1584dfe1573df8ec88c7b74d76726b4821bfe84bf886dd3c0e3f74c2ea18aa62ca44c871fb1c63971fccf6937e6501f","0xac69ae9e6c385a368df71d11ac68f45f05e005306df3c2bf98ed3577708256bd97f8c09d3f72115444077a9bb711d8d1","0xa54fe5c26059ed60b4f0b66ef7b0bf167580504525f83c169507dc812816df41b1da6128341c23977300dffd32a32f41","0xad9222dec71ff8ee6bc0426ffe7b5e66f96738225db281dd20027a1556d089fdebd040abfbc2041d6c1a0d8fdcfce183","0x87231421a08ed28e7d357e2b37a26a458155c8d822d829344bd1029e5d175b5edfaa78f16f784f724a2caef124944c4f","0xb72cb106b7bc1ecae219e0ae1830a509ed18a042b56a2779f4033419de69ba8ae8017090caed1f5377bfa68506157360","0xb27ad13afc8ff30e087797b344c8382bb0a84447549f1b0274059ddd652276e7b148ba8808a10cc45746762957d4efbe","0xaaddb0cb69ca18f14aed7054e98a24df0ff606aeff919d489f7884fd1bd183bcb46ea54bc363146e1a88db36dc20a7a4","0x996d10c3026b9344532b06c70a596f972a1e779a1f6106d3da9f6ba376bbf7ec82d2f52629e5dbf3f7d03b00f6b862af","0x91709ee06497b9ac049325853d64947290189a8c2322e3a500d91e23ea02dc158b6db63ae558b3b7670357a151cd6071","0xa03c2a82374e04b2e0594c4ce14fb3f225b46f13188f0d8002a523c7dcfb939ae4856053c2c9c695374d7c3685df1ca5","0xab72cbc6575c3179680a58c0ecd5de46d2678ccbafc016746348ee5688edcb21b4e15bd37c70c508e3ea73103c2d566b","0xafa10af166a0dbf3a25ff86cd6f8e44cccc818c5e70cd70e4e98e226b158f3563450b3fb184d2649adbb11e53080d1ca","0xabd12678c73463ecea5867a80caf256d5c5e6ba53ff188b143a4d5be83365ad257edf39eaa1ba8753c4cdf4c632ff99e","0xa35c6004f387430c3797ab0157af7b824c8fe106241c7cdeb897d900c0f9e4bb945ff2a6b88cbd10e35ec48aaa554ecb","0x8a8bb292bcc481070d3afdbbc8789e2ab4b29c9603936e6d85f5ff71e23fc5b6d61009f0fa636b5d5b2dc309d39e3d75","0xae940a07850cf904b44f31cbf0e44824bae5ec36dcfdb7fad858f2a39dba38de82ca12b0ae939a34fce7a02e4b9789f8","0xa75ca9447dca3a3745ada36731187ddd1f6a152cf15d7446b785eab381e5c8562c1202a6e7a24080bc6b619a161113db","0x84a687ffdf21a0ad754d0164d1e2c03035613ab76359e7f5cf51ea4a425a6ee026725ec0a0dbd336f7dab759596f0bf8","0x96947de9e6068c22a7716656a2755a9551b0b66c2d1a741bf84a088fe1e840e992dc39861bf8ba3e8d5b6d21e8f57e64","0xb570dde8ee80512e3d031caf22e775c60f7f5a6cbdeb3e52e24cf8c867d38569a53dd19cdc36a03a1bbb3a8d94b03670","0xae5302796cfeca685eaf37ffd5baeb32121f2f07415bee26cc0051ee513ff3932d2c365e3d9f87b0949a5980445cb64c","0x8de5a6200cebb09b2198e69fed84bcd512ec5cf317c5f1ee99aad03d2a9a8564bf3807c08da2664222268d59c34a06e4","0xa1d9840eda3036fbf63eeea40146e4548553e6e1b2a653ab349b376f31b367c40d71fb59ff8e94b91daa99c262ec8b52","0x8419cf00f2783c430dc861a710984d0429d3b3a7f6db849b4f5c05e0d87339704c5c7f5eede6adfc8776d666587b5932","0x8d46e9aa0c1986056e407efc7013b7f271027d3c98ce96667faa98074ab0588a61681faf78644c11819a459a95689dab","0x930743bfc7e18d3bd7351eaa74f477505268c1e4e1fd1ca3ccccdefb2595517343bbb8f5589c435c3c39323a4c0080f8","0x81ea9f74ef7d935b807474e38954ae3934856219a23e074954b2e860c5a3c400f9aedb42cd27cb4ceb697ca36d1e58cb","0xa804e4fa8d1391a9d078aa93985a12503b84ce4f6f1f9e70ab7fca421e1cf972538666299d4c1bfc39327b469b2db7a8","0xab40dc1cfe273ad0da700c64f8fc94f91db253ca3acf20e336d9bd09de67eec5c7d3506285d83c7bb6a08d64b77e5f2d","0x8dfa86c051edd28c3554a30e40531c898e5936ad3002711616ddd1b27054bc39caedd505a200c3d23a1c3f6b26c50ae9","0x81fa222737fe818b43f55f209f42adaee135b2801d02709617fc88c2871852358260ace97cf323e761b5cc18bc7325b3","0xa4ee6d37dc259cbb5237e4265429a9fd8ab5643af81628cc101e0d8b4a333ef2618a37df89ea3f92b5ea4333d8cda393","0xa759f6bcca8f35fcaadc406cc4b828c016c0ed23882987a79f52f2933b5cedefe24e31df6fd0d38e8a802dbafd750d01","0x8d028a021c5c31a1aa1e18eda74cfaf0fba1c454c17c2e0fc730dd07a19d0c77f7a905d54017292f3e800ca06b6977cd","0xa2e2d8384fc87a512ee34eb43405fd82572c9d7cd96e155a382cda284e8df9eb7189c25b7473d89c63ea4e6080e10ff8","0x81b676591b823270a3284ace7d81cbce2d6cdce55bb0e053874d7e3a08f729453009d3e662ec3130379f43c0f3210b6d","0xb5e898a1fc06d51c695712928f44646d15451340d1b3e480a40f03250160bc07d3b6691ec94361dd524d59d9df7f76d3","0x84dc37ca3cd621d3da0fbdd11ca84021e0cd81a73d772dd6fcf19775b72eb64af4e573213378ccee0915dde92ac83ba6","0x84d08d58c31bcd3cddf93e13d6f50203897384afa34644bff1135efe8e01c81c6a91ca6c234bb1e51ca32e41b828aaf9","0xb2225575d5e70da1257db7a0d1222c5041b52aac61cf161e8fc8126a3fdf5eb4f0867d98dfe272199c36cf8f02661b3d","0xa8fa3584a92b079c8c73ed1553e5e161a0b21325fc2fc4e24a892354a899c7fc0bfb436a97a7ed1fc71bccda438ea715","0x9918433b8f0bc5e126da3fdef8d7b71456492dae6d2d07f2e10c7a7f852046f84ed0ce6d3bfec42200670db27dcf3037","0xb24391aa97bfff29adc935d06a2b6d583433caf82f92de1980e0192d3b270323bdbf24b86dc61520a40c419dde3df4b3","0xa62c0205fb22df8535c0b70076486e69dfa908feddae79e4a94a9d47b97ed190d228e1c6217e84a59882bb992dacae30","0xb63f327df68581cdc02a66c1c65e906a06a1a3a8d7a6e38f7b6da944e8e6cc2db85fced5327d8c12945ceb33018272ca","0x8aec5129a518010912215e1887191da94be419b4e75904c2ea745e2d253d707c088fa5b2c46dade1d162affe9f7ab17b","0x8725b32751419f22a54485790f8187d1ba52d84a31ad45738a93777fcd1ccbec1652229923f82f37793ce0fc2763fb4c","0xab64f900c770e2b99de6b86b4390bbd1579bd48dccec55800adbcf52e006f22128e9971bbf3a92cc0105b0974849935a","0xa0485d71f1f5e177f7d5bc9d98c5248a6a2d0de4554c2eaf02abae48f5a3e273b2ee7765784cf2a4cb7df84f617177c9","0xb09cb155daf2022afd18114a352e506a84065c80573cb0c7c310cbe92e2706cdcf91f74bbd9e464f74e3d831386d5033","0x99d83a0ba33161d8c6bbe80929fd9046d4dfdac43477ff85fea5bae925e6c179ad28eb338375ee2417acbd6576ee670a","0x958c2692b86b4d20eaea3bb45e9447ebbc5b93ccaf8d21ef659d0cefedf5c4371b31b460ae40e8243682bde505abac1e","0x8d8985e5dd341c9035b37bf7391c5944c28131b47c7d5359d18fca598010ba9a63e27c55e6b421a807038c320564db17","0xaf89ab00a0eab1131645292a9cfba583a69a1e3ac58b210e262494853e67385aeb50d4af428bdd577b9399daa96d8b20","0x896a51e0b0de0f29029af38b796db1f1e6d0f9f9085ade40a313a60cb723fa3d58f6587175570086c4fbf0fe5331f1c8","0x9763dde1b8028136a3ffd6dafd1f450e2cafb2819c7fa901f7c6e9cde8f2897ee7e9a45da6947fde1ad0d3836188eab5","0x8fda66b8607af873f4c2c8218dd3ffc7940d411047eb199b5cd010156af4845d21dd2e65b0e44cfffb5e78271e9bb29d","0x86e014747c7922ccfc2b9d4bf6c1ecf0dc800197037858d0b85ab1944b4c3c14b95e0ed325bc42a6f467bc47ec27bc7b","0x8c0d15baa72bfcd317e9b9402ca9bb6e7ae1db35ffce7faccae0bd19b3c8e5de7d5524aef0377770b3a90626627a9304","0xaf61f263addfb41c46d66e60ecfb598a5942f648f58718b6b4e4c92019fdb12328efbff98703134bcf28e9c1fab4bb60","0x8de7ec501d574152f52a962bf588573df2fc3563fd0c6077651208ed20f24f3d8572425706b343117b48bdca56808416","0xb97ecbcfe8c52b9bcdca9e75da13c5650b751b037c570934ea6b6441ff32de6566c50dafc0557e63105b2ee7e8cbb39e","0xac30aacd9e91cb0727c34ca6b40fbfd4d255b998471e25c443cf6cf777d6bb823a58e162958f32c3c5ca80453387a5d2","0xb43ccb05317c2b666470ab251e987d6bf31f5ead6b5edac5fe007dd334ae6ce1a92e24c19e5ab387cd8fa253b63bb78c","0x966c488d807b3208bb1b10a1af422bac8d363c8015cda4e24d214549ced019cd3dd575545dd887461cae3f70d95cb061","0xb3faeebfbebd085b9123ae0e09af9cd15d3b1db6a25f3e82d8b48b68e53522b41b342a3a3c8b008897df356048862d98","0xb8c6663371dc37bf083134dea26a20115ccc52b7c15a662bcfa33435e4ade14c6bc9714a5cdee492530accf8a327b2aa","0xa39731d5cb52838d02d4ff897ab908c0f76a9ef837f9288c634ed3091a1f69d5347dc65cd2c8009a5207a369a4c6bdae","0xb97dbe4add8aefd96c575ae9de19d1ac590bb7d92f23a9e4e113f7271c2243cf689e7645879efbb546d58ec44f5263d6","0x95833097520df43a5cb013e97f80041a7a0b7d84a4ec79e2f16baeeb6edfbcf62ede97becfde73883831bb65e1415dc0","0xaee3fbc60f939c125877a4f4529517edcf114fdb83715f9f4041eccebb91323ce8c4784ff87815c38752517b3d2e2725","0x92b68717b3b88b77716884d492966713d902eb35196cf71faf1fb5625327ddbdebae94786c77dde207b5666ffa6dff98","0x91da49dbfb1a4a339ee8e1b902bf18302c3ca948da1dacfecfe9934d231013544dbd97689c2996f22b456f7e408a138c","0x93d491d7211af181ccab8353567ad10c065a6e991c9b70def7215c62fa3bf843a177e18b00580deead4b5678f46d4d39","0x8890da2859ccb2afd0742c1c791075104d7acc207b6fa478bb1f94fd6665ac75e5dff4d2cac0f81e4168448bc3a4c90d","0xb320e188ef282109ba8dc3d2573f9edc33831c1025a29844c86e7fc60a25627e507964725ec8dbeccfcbaa12e7fb5e1a","0xa90dab71ad924fec01a579727736fd9a19a147bf57ef471255527b55a4702cfd54bb0300623c506823e723e7c30cf4dd","0x95d7b5b39931578c673a6ffdcd5b3618ddf08b40909fd0ee04a74c70aae4e88c55a43d5b71cdf40c514195a7805019b3","0x978eb7a3aee4238207d1e78684a6769e3d71f1a3b9e42ac53c347c97e15eec3ac0082abf65ad9cfd126c687c51436cc1","0x98213294b82bc66ee39e95a678472fb41df846ec2863c5be53e1fd56b6ff0fe1bfd5b2bd8c534dd97acbe597ad119cc7","0xabd3280a86fbb8d736717d8bd950920873ff7bb8a68ac4d7c339bb3c783c6f4e6e29912f6a3528e04b13aef57d4290a1","0xb0b263c298eb4d09de14dd71005fc683c4405efdb230c6661b9697d3a0978e5ed7736e76729b2233341e75ea46ecff1f","0x85fc722fa6d2c54b9610307ef86f0752d1428bdec6dc7e46ae14318bd9203f32df1ca8d0a420c36973067a50930b7720","0x958b57a4b16322f680eb4eebf37c538f1b5bf96400e51f99df6a4a439b75b9acd8866caf4a091d02f25510d3bba1aa26","0x889dbdf3bd68af1f6fd84cb6173b1fa1f7c5e6ba63297dc1e2f45cd1a82bb6231ba832adc5228143c5cff3ef0b1caae2","0x99377a407f49949e88651b57160044764930ddd9fc404f4b610a581c5edf9e906f3edaad611364d240336a3eaf4dab82","0xb6cd22959866607d91e13122bd34b050a7da426dad5e9779bcbb99e4a2e7bd5d18ce39f266bc61ef7524a2e2adfaf765","0x8548d74ab33e8dd285e72b1ea3c15eb08b66d555493bcf39aa4355af06ccf0f4469e09f110544dc04b1f3d0e7880ac1d","0x8457b3220923283912ea67a58687eab0fd1747497c94a8f73c4f75147fcc548acbe662e7d2542f5fc3c94749f2f6bd6f","0x961a7d85c4e61428f07c4c4d41cc03115ec423bf40172b21f9db161edec282c950782ade73a6dfa915da1c39c716eb1f","0xb26fae53e4f7d2088de3fd67ce06fde88a84a7963af20ff8901613bd8be30e9660b244e5e0bbcc39c940c814f6f3f318","0x986a201d308cce68d381d103680725fbce0fd9f618de12d371fd5c909049ba3081de693de9e0560364c201505b2fabaa","0xb165f7ea9728a1e55dcebc59c89321586290bb3c6aad2114217e81990104863b779ffac9728ed736b6069a36a26aad47","0x8e8d4d32b9c20bfa05ae904524dd1fcffe5ef1cfa451d6f0289eb54844a6ba033da0c656c1b69a508a45bb572797e195","0xa9fdcd176de75a9a1ec07d553a267413ec5406ea5f46a65d924519da3fa4b3009e2bb350fed52a51bd8683fca5d6184a","0x889fd954b35f31ea3f3eca4496ba13b9678a527e4d3137c8aea484b4d741886bdfff59f99a89721165f5ba7c9f5a366b","0xb4147399957d387dd0ad99bc9fbd7ddd3ad85809c9658d121879099c602db62c4a4ce4d122915b6d8135bbd16c13ffbc","0xb9882d1217732ebc2e6e2eaf42a9fc606ccd61e6a28a01f77c0d6829e255b51e50cebb38cc86254b37bcab1fe817e7b5","0xb8f3af1b7f9ea13cb73348da2fa847c6e0b89bb415dae2b62ea29e294060ea5c73c51bf00c50d8b76af2df9b35ca8c45","0xac97a570a795d24af13ef32709d71a37c0fb90a49581e735d635721581c5553c10eb15f43531121b6026f55d603d73f7","0xb0d4372ed0f55fa767a5fefc734c155a33821ffb6e6be4f628955a6477cf1d4f12c1d0e82426a8ca14b9e92ef094472f","0xa11c83b69a43111201fe36f54212afe6f12cd3a8ad551b586061901775a8205b816cc8638956c57a339228d61b520aa7","0x867e89563df1501ac7dc5a369e6713cebab2aa1b676ea6d97fcb62802488866ae1223b4ed6c00718ee895d7e8e650cac","0xb4cb3c19c6974ff1ff486b97cfc43eccba5e61e288ff008349cc99d1adb317ad2e9e28adafc48057322998eef132ab26","0xb1f2588848bded71a0ad83d124b5ff2d1ab964068e4eab97edc678eddc9aaa6175b51f561c4ec96c818ad5f6c59aa936","0xb4c9e24cb284d23c40bccfe377f9963dcdc1d22d5daf0766a6291354eb4ff3b4d1c21913dadae80fad218bc02778f5f2","0xac2d58a25bea23bb5160dc19a9e3a936dcf276e81079326e2925704ac9ba43907d653b841ebbdbbfbabd0f1ea1fac717","0xaa080afda88d384d10c98431dfec91c4072d6c9a4b43f302e5918d6292fdca1969f37a26cb05af494b6e46b3d61eb053","0xa7ed4d8afeea6c020adddf320ed86a863ec5d048236c28e56f61f83b4603cb9e4f2f2bf3f6dc10864936c98c0784c038","0x899f89c23e08b2e89a5416a7c6c9e76c7e1a064c49cb2f0825699d4412784bd4e34799c06b3faf0539b6fb4ab10c104e","0xa878b6b608d51a556d4d599810a70dac104d7971a8d99ac72341ff2685fa3a75561ca1fc4e5ffb4b73b5dcfce372350c","0xa8d747a2b2602fe32095a7dda37a86d94f8a59cc771902be8c45de5bfd9d58fc1bf46fb0cf867054cd5eacf4f331dfd1","0x851cb7093041931bbda885ef6d5a0411353e41b2baf7713ff7688b70d505dc97561a6b6482571aca964486d8a9b76a47","0x8d9dc5d04f5b105ddd3e84529e16f33748969fea81729ea7dc087fee8f1e3e2faeaaee3ca7ee5dc264a22cbe0ff809b3","0xb8f404f4d6965ff42eb6b325a85570ad85c4e8bfa9953d10eda05dc5db0070b8b43242eef8c2fbbe3a549149419a6428","0xa8494e9a4ca0b6fb595347fda04978eb76c0520b07b760ebce609c5051d6a7e2b01dc68f2b2eab6aca5a8fdbbdcd9346","0xa3baebcb3b1f6364d27e5bb8cab4cdee8b8388f29bf5ea235251742563e171a6b7e02a854ed24691725c58b5a7a88987","0xb3a1fbfd06f4d68415d699ced0f6d3f10ab370214fe66e2e4843bde2821bb4b878de8b7c9b2e38b45de941d06a665117","0xaa8ec705d7394909c67619fe6ba077508290a2756f0bd39b46f53950da8965c8a2fe8d1f2ce19bca22a8aeb9efbdbff6","0xb967afc934f2efd001a6901490e770602f80df9ec0eb3293490ed8c55e14d08f5bebb7bd183f485de29a8423d859a501","0xae2dd5d89fc56a0368ebba3891020b24ecf7acbdced259277d10eddf8d812568542ee3df43fab157034cc3620f17b75b","0xaffdd643395f3ca4138646f729aa0f5d1bfdd085df8d2559e075f72cbcaa24d9310074491910572390fbca5478d7d369","0x81093820fe0770a18a816945494db8fd957f10f7693da18f782e0968ef28ea8f23afefb0dc203925262352925f2739df","0xb9e03b94bb696b0e4c7939bce96d9e4fb1938074233d87b290b20cf66d3e48a7f3d852d89969f45c075e3dca91945832"] \ No newline at end of file diff --git a/keys/node-1-pubkeys.json b/keys/node-1-pubkeys.json new file mode 100644 index 0000000..1bdcc96 --- /dev/null +++ b/keys/node-1-pubkeys.json @@ -0,0 +1 @@ +["0xb05cafec5912f22dbd6f15677f25f13d93ecd5ec6f957fddd7cf27d73521b34aaaf6a219f77b21128d18321c2c8d679b","0x870342ee85d1d3eda564de4126f20880d59164e2f88652d9dcf3dc93d0bf19e22ca3a11305f1cba1cadaf2d117028936","0xb0db46ced0115b365df2d2c1e29ef3333b0bd4ed297288f7a09ca9c1de5e702ef8f2cbeb89d0d70a584cfe991cf7bb65","0xa567c07d1f258a7dc4f685b9c45c3217e9e640d8cbb3fde3a875e31b0212df6d48985f8524922205aaf6917a5b577d89","0xb7f216edfaf073f84d71ed41b7376d8fe85c88b40297636a2870278d00b452fd37b41af5e357a2dbe1297b53fb027e9a","0x9188ee0eb50d3a88a27a2ce4334ea8cac1662593526f4c280cb3049bc91afd8381ddbda124b9fab871aacf378ab5380a","0x8bf6583d6de04a89b9ca61c69977de4ca440c4c9b13c7a1c65e205979849376e239d03cb46d8bcdf58afca96a8e0fcfa","0xa9f291de2f415ab3a4002206769493d82a094e1934b78d98b79c93f70ffff7389d8ea0962b34b6df04ba999442a0fda5","0xb7ec2f481129da715b78d3c6bc1ab2e04bedadb937812497196409c08d7837d133c8bf52aec70689b1b180d8eef2676c","0xb0862b7f0788739e315de558384d6f95531088016f18330ab572fe89f3600098284f7a08a1ea8e7e33666904b918b17c","0x947e056beba42f6d7fec6712de16458c85ae513391dc4d42120e5877ed21a0681820a6150cb45af4f0cbf98c1d25c8cc","0xa03f7aa96fd1dcb385cf9f4f29f9f3ef4c25a47efdb090c18763a51608e92267daa98695f765705a7630269561f9086c","0xa62b23a8a25355c20cf4ebf93bc43a7b4076ab247ac7bff133ddaa7cfb9588e598d1022285b0e2f2fdd0ee3fd51a6f39","0x84c1b3ec3752b9d7e3ddfe87c68c59443b5fca9d18d78e5441c80e4fb6c0ecec27f8074811c2e5f823364a71e0567394","0xb7c37e964efdbe3916e611daa4a1241bad3aeedf6a57a21c87c23bec872452c05816ee9179d021c3cf36843063c687d0","0xae2bb170260de9afb23f49d5770474a65b4d380d904aa41cab7c2852074ed8ad9fb94f6d7e5a5a1f6f71c0dce0bddd12","0x8faadcfded5c85beb36f1cdc234f0cfd8ebb0820a18899445accf3e6e35efc0ee34419eeb0cfb097dff5fd6462b9ed90","0x837d8e7320247799d20afbddb410fda1bb7fcd31e36ccc841012e4dac0d643e8e5e12d2467d8d3219aab33b64280eaea","0x859155dd5a22f116ae8f61b1516770f8ff41ec0ea24b8b745171b4cf34981bb7d235e7e1a739a0589e7c7ff69ede9b15","0xb869ba1794050a014193ad467452efa3a54bfe6c6d1689bf7de9576eac2d2c2dfd4383a219e2d450b00cd9a70fc5e2a7","0xb06cb2010c1167c72840b3149ba92b326799375b1b05a1c0ca38ace5e8f61cba48c1b350a7220938e9c2c0fe6b6c2881","0xadcb081ad4dd8f1acfdf1a71360c6d5655bdd58d9bb1f09e4de43b7ff8a6da60b61df9ce65f9ffc951740dfc69812667","0x8d1df9e9132058de96486f102ff4ef6e34c988b6dd42a9462954218b8728310bdc25a4251c092eec8128bfdde893049e","0x8257c261afa77e79086b503de88dad720443a1d135cbc14f8a6de408a03ac5b9c4263731d6693bf843f0a9657aa3c4e8","0x8afc5f7128e998c2a59b9e2dd1805ccda56220d1bb25ea94c13d4f9abb4ab55a522518aabface4f10d54f45d08dffa8d","0xadc1c39301fa1fe99678a7b7887e895c8df24e15546b13d2237ab2795cc7004e6b68e69724f2c0922f119d5af8819bc9","0x94ef22c9183e15da2e4ad8e05a75c5b9201d52e9ad7f66cb0061b0c68779ba5a1fd0f11b9c365c1721d00199d287923c","0x8c3f73416c86d93ae2dbc2a468b5d6cc39d42c2ea7e20cc215212d3dd3b8efbed324bcd4a28de941fe855c54a92c6973","0x914dc0bb6af111bc9021887ca4dfb27cc2063c9ebd2be133754d30ca18fa8698787005538f556bd94867cc1c5f7b817a","0x95e8e9e5c389be338759c40d2e408b1f0b78ebc0ffbafae360e33b683ada3638414338b83490e9ab1ff067425d25d785","0x88ce4d8fee80abef17438d6e1dee0da0087b3fde540c7cd157e169866c603af0af26fe3c3d2ba517dead10376d60df89","0xb956f93f164469f3bb6b2e95c7d3db7d979b03457be68c6b227a9bda9b4be639198e89c9a62452162640cf95a00ad339","0x902b0898a017b3e98d334cdf49d5411e507f6043fca0624e937945c7dfb1829a3a2e1d0bdba654aef7d2e3c14c76b48d","0xb8a434ac1fa9c6a2c2c37da065576261d981d870923222a50b225422097a3e59598f548be8c6bc0b4764c02546f11b7c","0x98eeb5417011d88a2924cbe7ff7ed616a6b7dfe273187360757c667f27349f525ac60665f9f7ba2d07d91d2f94566f1e","0xa83c93593c32b08e89f0089e1a0892dcf121a856199b751ec40959433b4e64ee7fea260a16f4929261c5e2ebc148042c","0x995c28e8767e8677ef93cf6aa49453a8ad6c279622820e321ff9d352ceafffaa7ec082713e4c85e50cf9f7e11439cc16","0xb83d8622fe3180d6e8fd95b59d5e5bc6b7c0451040ec3b14588b1405410bbb249c15d449dd9afa4d9b42650b26a58fcc","0xa757657d95e795d3460d5454b0f3987885f0a8138c4de92e08d2843709f808adb191c9d2b22399bb9445dbb94e190382","0x81f9ac9f60825d682d5ab33098117dbdcf3c5245116c8c03a8c0493a5d441ba578a0b3d069d745cfdff70122c65e421a","0x8d19ef6c96e7ea917640ea3ad0c6c6a9ac8320456c3ec046cebaa625d415476f71d854d186a1758c79843925b2d268ac","0xa1c646e753dd9d9811cd75f58c371e74ce83f05606bc076d64bdc77adb313b1bfd8ceceae38d8a8b7d4795f10ac68d18","0x8cb3b628e5ec89b1cae8fffb946a1277613deec1f3f0e7d75708fe3f6b15c5efb75a0ec8dce885812cf14f3d589b6f91","0x933d88e34601df31b7a66bae1d49079548a2ae2d72037c7c3b2fb8925631989c1ae353311c36fe325b2f1fdc1f648194","0x8ff3e5eb9c905e42b88cd8fd8593bde4d44e22cbf296d47917a4b0f144fbbe2ab69a4421b2c0ff3500141dcf88a0b007","0x9592c95f5c1574b8c510e546759f383779a0364fefa009f84dfcebb2efc4e86b909a5fffb90ed79b5f66980b420db7e4","0x9218096756d3ccec228caddd27979b9050b58d7304f2be4f7d4aafb6ae19fdab0bb9149ce29517a383f59facaacf6d39","0x8de14b70a834e78b74d6bd46bd0f5f92d878a1b19f9081808dae86391aeab2a048368f0ae5b93ab412561db70482dba0","0xa0833e9ca3181e33a9b51d7c31722cd07c8f0a18a34ba083eaeb091529c2a454c3950ec449ab7f6f237995bf1ec0a802","0x98249520dcdc8be36b8afed019336e2eb478d44a0f5ebb6d7c994710856abf32ed25313867e060f83b205ea5ebe9f0fc","0x8f14d19885ec7d1bf7b6c4668d505283d68d1af78588532a15aa80ecf96a2d8bbeb813f63c25c15cd8f04c10fca4c49f","0xa6094298efdad03170daefba1bf92ee8691aa86fb753afcc252090c74c5f97c7d6cdfbbe0017f57e25d406d4ccebdb0e","0xa94d36839c1557727aa55fcca24faaab0d1e5f0ecff1dd709e22f0c55408b8b74d22173bc312d27baaa8688ae1b4be2c","0xaa228e16f801dfbde52d5f57b44e7e9a613fef33f893fad979ce003eeed012c8f45e9050db814e241be87ae94c0c4011","0xa8f3c2f7371294ca58c85e7b1513c155a39e5bd76eedd90f50a4fb50d6ae55b40623d5ba0c5d85c40def89d6c67d5fc3","0x8fe444e4b5610d3583a667dcf23af26a9f686db05e7de9ebc03f8ea0a756cc96b4f55611f5788d61f26e5bda1df70f68","0xadfdf512e8ac8e3f01ab9a03733c6d0bd0a5403778eebf691d2906380bc01591e93b43cae86c94f657c92c29d5698a06","0xaaafd2bc633a130d0798c8f398ebd00f5c1b131c2b4d48cbaddde1c8ed59eae6af8290d27700852df2b3cf23d630d807","0x863fb35bdce0573031210c5a6d3521b5a3e11cc99356de9e8adccf3accb4c4d387f954b202d0db647e65c331b5019226","0xb1748deeb17775232e5b53f55bc2d1b08c494a80cb727a2f2361af61464afa85a1afcae978e5d802e6c75e3be60965da","0x999664783bb5eb59491a99142392f020fb4e3b607203c3f9bec7ccb3537e8f7856d7ddfbede9dc3332cf40bdd4334a64","0x84e8581cf13f7df6a96835ad1330593bf5d1f4e5bd6341f7a54063b0233e921250d0c09a48849155953e293cf635d7b6","0xa608c06384bc606f723cde2a6f7d64a29de9cd987c0626dabea2414bd5b653647b31953c1156803191484d5ecf4630f1","0x81ef1d058664e94bcdd4876d049fc40e9a2d55a104a2fbcd33b63774a55bf994d72e6c7dc641af29bfa57362413db705","0x944a35116eaa393571e02f6751f1a2820ae5c4075e9bcb9746143a320c5a6ef5c3f4f939629dd0a2753b27bf5c5317a0","0xa1c888d5d2c76f2b388d6a42b42005adeca9af4d454c6342b9b634bcc6944432398823d240b9bd1bc7f37f58b390405d","0xa2869de5721730e34b8b074d27e6e4c2c6d09f6bffdf892edc6b3590830362f496e9c41c601a3e38eb80dcaf6f0c65c3","0xa069b0ecd8be3a45dc395f87b4ea9ac575f2f39dd922a1201ff0a3b9a5ec551f64a325647e6fa58770d94af853d6d6e5","0x8eece2624f5aeefc66c93625462a310c55f8df3c0632543d590db8799df2642ae39a0e77a9842e5d1982701311d46e42","0x8b1748a729412116660036812e9196fce6b9c553b47ffd942dc40c60a018bd49ae026e2bf4a1999773521b31bc12e64a","0x88cee7a3c398f11dc6eaa039542e9467bc7cfcab47718d92533d7adf7e36021e7cbee54342320837aca410b4e1dd6a33","0xade9004ebdbf580c9c29ab6d34b548f489609c98961b95b9f0b1c6bbad66ea4941143398510bf4f1ae0a7c72555cb9e4","0x93641e8f9e440bd769fa86db3f72ecf076da7d2cdb01355415c7c54fc90075899e40fe005898d63d706f3f94bfe9373d","0xa85336ac11331940f64d8bcded1f4a3dde75808d847311b2ff79bad985af2e2665fa5bf58d67b9eedf001d5663c08030","0xa7372f1be3e7bf1aa20000103a787929d2b884c2081392571cfc7f5034a91b79127706c6a2acde2c0277009fbd9943d1","0x953ba2a21db4a6b4063eccf6b599003803c9b03d24009ca78cd139a4d2cf26c602c34fb186e106dd0649985a3e5718a3","0x8d1c890c3036ed8913e69c6fa9f37816425ff5c06d1821db0d32e338c5d9b8db47e9146dd1327fe2c8358911b4b2fa26","0xb2ad1574eaca33f1555308e24b27a095d24aed8f4af5302ea2c6ba2e50936d25ffea7047be94065eac630693c7f86757","0x9806dff63a4ea679c0283a72d42a38d10d7949e6e7b5964ec1eced6e7ff1add4f32d7ff92d4097e7b0773633cded5051","0xa826dbfe4cccb4198a2689e1b32e59be8b774af0abb56c720ee2fc7472552f3ddc7a1ad15ddb2c2bf4189ecd5ada3761","0x8eeeef43e0617ee768138e98103fa9aaadd9491966a3904a24b51e5c955f3ed007a5ae64de4b541533a19246b3357e3a","0xa4bacbd1f1195b0ab5253b7437e05d235d663f9e8109dded69ef45a71a5366f337c2787679717ea06f8321cd0982c22d","0x8188a5547ef304357dc50d0e7ff3505202b18ed0dacb2f856e0450bdab0be01c509d4659ec44cf518cc5fcade8b3111d","0x9320225d3ed15c0fd4c0d9d629a6c8bff13af7eead740f4941aa850918649f00e42e4fcba371ffcdb2864ada1c654c6e","0xb44a24c31c8330ab530f2d4066256d26a12fbd3163ebbfa8d3cfe091a3926d16b17a5d2f3b4f87ee5cbe91c2fdfc5f05","0xb3abb58d18a587ff17cdbd85be687622eb2add07e80ca5190745badb03a216f8969b59405c85bd5a188008bcdf5a58c5","0xa5c521377c52dfe16b14e171bf3cddfceb7a6dbd768a7ce9716447b342466ab23e5c0e77854649735c4e41dd5f981e74","0x8ca49f0c11f05aaf0d7f7277fce738c55cf907d9b3a03a7eb61ed31b4c6808397506624310000a6dba2441e96c7b9ce0","0x84b2c527a35b380c1d407f89a70f28e5241ab6ebc902f13e3f8dabde85c1c2a4fc5bcb9b16474ff0cbda39dbabcef906","0x92a6792fe1ffb91a4ae57c6f8b82fa12d9f5f8485d21232304b7a8a971c2f5c0759839b90833cf561b4be224ccc23018","0xa45e18dc474050ec2d4a16975e6d59a9714f7a06458ce995e7b4882e21dbd2f4f1094733f69da85a6a636254b2d228bb","0x879336a84d4871909368d36a4896036e4482ea11204aeb57ee703f4bd323307f4521f038f766a1776a0b648d795901cd","0xb8716a085bf7915e9fedb32b67c1daf91fa32638a06e2edf1b38966566e8ca10ca06deaa0faec734e3e03899c8c919fc","0xa11525182b424d0c561b7e08882e05717cb4c93eb3002927efd7e077f37f20bcfc6f2fbd3f0ca9533d36f499564b279e","0xa067c96285c9e48306065cc1b0217e2de81be782fc101b7cb0a1cc1fd25f10e7bd428df87593a04babc7cdb0ceff31aa","0xa01ba34e831653404dc9cddc23155e6085f631c713681fa56412ff7c6ed8f5702d384649f42791f6b36843522ca0b8de","0xa49bb35ada704f33f70d2d77a72498b3e2fa01fc9b7f8f02b2a6397a825b043b3eaf00e440c48ae522e42119095c88bd","0xabe99cd6c855608a1d0d34e724a6f4dea47cb8148544809169d7d323ad47efd55402adffe2d20f346fe17e3f60402e37","0xab22c3c322bc0e0594926d5ac48e2fad0eeaa24d7dc616fc9bb0af6652188242d1dc59072f35a1552b3a0bae0ad6b170","0x881b0b35d01247b4d7e5a1a3d2b071c40904d0c1892eed4f8b7f1c531b18feba1e5b81acf8c69cd6b6ad0cb31f658333","0xb586ce5eee6b78c9d65284be1496d8b6d32080ff573361ad2d69e26859772e8c2e0263b0f173489adf9c7b7cd98bf95f","0xb602e252aea22de9c5d5733b8ddcd2b4db8aaf24fe57f73b5584bbc9acb5f89af75717290ec7f7ea2a9fbc1ee7dd82d7","0x8a48989a82f473050e42cd9cb58d8538d285e069d775b034df1d3a703fd5821d8ae6e2bc5cc5ee3bc770cb28ba59f6bd","0x87c10f9555a9040368909a1c4670de4c2f725b4a485a77716e5e060e65c47c0edb926f28a422d2ec24868bdaf771dbd3","0xa38f647bb270cfd59a8567c5afe5e02463eafd2fc9d0379494c0b83b235594cf3283d5cf67ae6ddcca00f78bce7d78d2","0x97844029484238a8951e707cd4f46207722e0f9fd15b050fc2798362d87293af7021398188e1fc033423d75a604fd830","0xaf49927c935c201799ab1f4961786ce1bc01266bcd6d2adb81b193c3362b73f2dac32db526b597ec5baec292dea920d5","0xb79d0e7e5004f1679bd96e89fd94c5cc2ee5c368d9207b4c778b6983f288e5f4cee3bf84c7ff4a16935140da31615b9f","0xb7e03f23fac574fc7be4a455aad6807d0c71bf06fd74cd4a7d72cbf1c1db1be5a01f50dbd51457264a16d8874a6a1648","0xac1ada1e9db792ed12d314638c5b16e7820eeb0d178aeeeed1e986b6919eb886c7fda24e058768ec0c63e96c1b2c632f","0x8ff47ed158f7547c94a40492dd7996ebf2a5223f2d1313219b0e56ef41ad3a7676a59ed91f1870d7c77e0b398d1848fd","0x94ef5705ed74f1618a6918d903de2b950df9fa5273f27ce89481155c1f5b084370d59c9ac2df9ab2bacd6a9ff9fd1138","0xa25eef6fa1f3b3bd7abb301c28230897aaf57fad66ce0871c54241948328bf40092f343b5f092d89ffa79b462a0ad0f1","0x89108f74cef974fd3a3d6263831f54007aab5d272ae9c0e7193575406eb9556445c1bab5f1338a7fefe97d030ce2c0ad","0xb92e5259c50d22699af4fd397cdd32177154a01f26c3d2e687489af5a94e86aa7cf342213d0827e4c90801a8cfafaad3","0x815e815cf1a66a872ea0f171341d096afd5b026a6f6ef60d176f1598a6dca647379ddb82d00df0b28c0479d291a90b9a","0x9491aca6c6afd9319152796ab99fe3eedffcfcb959e7d7da0ba68e04754534d8576c394d80f407d84cfd3f25d90df4a0","0x84189eff6ff8fe060064c0bb0c9c50e8680ca4198c14878807ac781f62376662c13d6c8cf4c588428fba90541d12b35e","0x9995cd3fe60b1757fb734715aea9941479756e8e1a8912cfd4ab20e1f9e3c2496740a4bfe66c2633876606d5136f73ae","0xb9b76b28c192907820d1ea5e0fab9c545e8859cadf0a25e17bbeebeaaa8a01f36b0971124e844ef976fa076689ed03a8","0x876df2629991ad5014a9552fe4bef866ad93db67c291fed081254316441b0dc974d11b02d318c2342e2379293d88036e","0x84cc6208ab0086c1a439a4ae3787a46a133847a5daeb9f7476814490be581f25c5bb515d43ad6092d4a8a01083917a6f","0xad8f20f15d45a0293e58e0188f19cc59e649daabc63e03e66da3ef58306ecfe8f2501d6e0ea3e3017734585ec398507d","0xb0d6f1ceefb752039bd31dfd2c26dc4f96a5a0d8f0772e08737438aa6e324f557d09c643891dfc5358b23b25ba3fd310","0xae4dc089d92c027f4cc3141c309edb5e23f0bc0935146bf3db2592317479e7757fbc65624ef9c1ecfb54d21602079d06","0x91edd2ba701744a581d1dad1b67b862d08e0dad9558f8c31826692cb9eb11a97f25ea32859113300e69136b9b01b4ac2","0xb9b781554f467aea192418c18dae0bdc986d447f6a020224d5062143cb96c1bde68d8365d1108c11162492a79238fd96","0xa09202a971426cc12fc4609b8c63cf8b1e11e2ba59db9238da5d7eaff9b8df01d7ed6cd927f82685839edd3a621112ed"] \ No newline at end of file diff --git a/scripts/generate_kurtosis_configs.py b/scripts/generate_kurtosis_configs.py new file mode 100644 index 0000000..6e65e3b --- /dev/null +++ b/scripts/generate_kurtosis_configs.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python3 +"""Generate Kurtosis YAML configs for Commit-Boost testing scenarios. + +Reads optional .env file from the project root for Docker image overrides. +See .env.example for all available variables and defaults. +""" + +import argparse +import json +import os +import sys + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +PROJECT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..")) +KEYS_CONFIGS_DIR = os.path.join(PROJECT_DIR, "keys") + +# --------------------------------------------------------------------------- +# Load .env overrides from cb-testing/.env +# --------------------------------------------------------------------------- + +def load_env(): + """Load key=value pairs from .env file in the project root. + + Simple parser: no variable expansion, no quoting tricks. Just + strips comments and blank lines. Missing file = not an error. + Returns a dict of (key, value) pairs. + """ + env_path = os.path.join(PROJECT_DIR, ".env") + result = {} + if not os.path.isfile(env_path): + return result + with open(env_path) as f: + for line in f: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if "=" not in stripped: + continue + key, _, value = stripped.partition("=") + result[key.strip()] = value.strip() + return result + + +ENV = load_env() + +# Image defaults (overridable via .env) +HELIX_RELAY_IMAGE = ENV.get("HELIX_RELAY_IMAGE", "helix-relay:kurtosis") +MEV_RELAY_IMAGE = ENV.get("MEV_RELAY_IMAGE", "ethpandaops/mev-boost-relay:main") +MEV_BOOST_IMAGE = ENV.get("MEV_BOOST_IMAGE", "commit-boost/pbs:kurtosis") +BUILDER_CL_IMAGE = ENV.get("BUILDER_CL_IMAGE", "sigp/lighthouse:latest") +BUILDER_EL_IMAGE = ENV.get("BUILDER_EL_IMAGE", "ethpandaops/reth-rbuilder:develop") + +# --------------------------------------------------------------------------- +# Shared YAML fragments +# --------------------------------------------------------------------------- + +COMMON_PARTICIPANTS = """\ +participants: + - el_type: geth + cl_type: lighthouse""" + +COMMON_ADDITIONAL_SERVICES = """\ +additional_services: + - dora + - spamoor + - prometheus""" + +COMMON_NETWORK_PARAMS = ( + "network_params:\n" + ' network: kurtosis\n' + ' network_id: "3151908"\n' + ' deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa"\n' + " seconds_per_slot: 12\n" + " slot_duration_ms: 12000\n" + " num_validator_keys_per_node: 128\n" + " preregistered_validator_keys_mnemonic:\n" + ' "giant issue aisle success illegal bike spike\n' + " question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy\n" + ' very lucky have athlete"\n' + ' prefunded_accounts: \'{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}\'\n' +) + +MUX_NETWORK_PARAMS = ( + "network_params:\n" + ' network: kurtosis\n' + ' network_id: "3151908"\n' + ' deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa"\n' + " seconds_per_slot: 12\n" + " slot_duration_ms: 12000\n" + " num_validator_keys_per_node: 256\n" + " preregistered_validator_keys_mnemonic:\n" + ' "giant issue aisle success illegal bike spike\n' + " question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy\n" + ' very lucky have athlete"\n' + " prefunded_accounts: '{\"0xb9e79d19f651a941757b35830232E7EFC77E1c79\": {\"balance\": \"100000ETH\"}}'\n" +) +# --------------------------------------------------------------------------- + +def load_pubkeys(filename): + path = os.path.join(KEYS_CONFIGS_DIR, filename) + if not os.path.isfile(path): + print(f"Error: missing pubkey file {path}", file=sys.stderr) + sys.exit(1) + with open(path, "r") as f: + return json.load(f) + + +def format_pubkey_list(pubkeys): + """Return a multiline list literal with 4-space entry indentation. + + When placed inside a YAML literal block that is itself indented 4 spaces, + the entries end up at 8 spaces total — matching the ground truth. + """ + lines = ["["] + for i, pk in enumerate(pubkeys): + comma = "" if i == len(pubkeys) - 1 else "," + lines.append(f' "{pk}"{comma}') + lines.append("]") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# TOML builders (raw; get indented 4 spaces by build_mev_params) +# --------------------------------------------------------------------------- + +def build_cb_toml_basic(timeout_get_header_ms, timeout_get_payload_ms, + extra_pbs_lines=None, per_relay_lines=None): + lines = [ + 'chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" }', + "", + "[pbs]", + 'host = "0.0.0.0"', + "port = {{ .Port }}", + f"timeout_get_header_ms = {timeout_get_header_ms}", + f"timeout_get_payload_ms = {timeout_get_payload_ms}", + "late_in_slot_time_ms = 2000", + ] + + if extra_pbs_lines: + # Insert after port (idx 4), before timeouts (idx 5) + insert_idx = 5 + for line in extra_pbs_lines: + lines.insert(insert_idx, line) + insert_idx += 1 + + lines.append("") + lines.append("") + lines.append("[metrics]") + lines.append("enabled = true") + lines.append('host = "0.0.0.0"') + lines.append("start_port = 9090") + lines.append("") + lines.append("{{ range $index, $relay := .Relays }}") + lines.append("[[relays]]") + lines.append('id = "mev_relay_{{$index}}"') + lines.append('url = "{{ $relay }}"') + + if per_relay_lines: + for line in per_relay_lines: + lines.append(line) + + lines.append("{{- end }}") + lines.append("") + lines.append("[logs.stdout]") + lines.append('level = "debug"') + lines.append("") + lines.append("[logs.file]") + lines.append("enabled = false") + + return "\n".join(lines) + + +def build_cb_toml_mux(pubkeys_node0, pubkeys_node1): + node0_list = format_pubkey_list(pubkeys_node0) + node1_list = format_pubkey_list(pubkeys_node1) + + lines = [ + 'chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" }', + "", + "[pbs]", + 'host = "0.0.0.0"', + "port = {{ .Port }}", + "timeout_get_header_ms = 950", + "timeout_get_payload_ms = 4000", + "late_in_slot_time_ms = 2000", + "", + "{{ range $index, $relay := .Relays }}", + "[[relays]]", + 'id = "mev_relay_{{$index}}"', + 'url = "{{ $relay }}"', + "{{- end }}", + "", + "[metrics]", + "enabled = true", + 'host = "0.0.0.0"', + "start_port = 9090", + "", + "[[mux]]", + 'id = "node_0_to_helix"', + f"validator_pubkeys = {node0_list}", + "timeout_get_header_ms = 900", + "[[mux.relays]]", + 'id = "mux_helix"', + 'url = "{{ index .Relays 0 }}"', + "", + "[[mux]]", + 'id = "node_1_to_flashbots"', + f"validator_pubkeys = {node1_list}", + "timeout_get_header_ms = 900", + "[[mux.relays]]", + 'id = "mux_flashbots"', + 'url = "{{ index .Relays 1 }}"', + "", + + "[logs.stdout]", + 'level = "debug"', + "", + "[logs.file]", + "enabled = false", + ] + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# MEV params builder +# --------------------------------------------------------------------------- + +def build_mev_params(relays, images, toml_block): + lines = ["mev_params:"] + + if isinstance(relays, list): + lines.append(" mev_relay:") + for r in relays: + lines.append(f" - {r}") + else: + lines.append(f" mev_relay: {relays}") + + lines.append(" mev_sidecar: commit-boost") + lines.append(" mev_builder: flashbots") + lines.append("") + + for key, val in images.items(): + lines.append(f" {key}: {val}") + + lines.append("") + lines.append(" mev_builder_subsidy: 1") + lines.append("") + lines.append(" commit_boost_config: |") + # Indent every non-empty TOML line by 4 spaces; keep blanks truly empty + for line in toml_block.splitlines(): + if line.strip(): + lines.append(f" {line}") + else: + lines.append("") + # NB: no trailing empty line here — that is handled by the caller's join + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Scenario generators +# --------------------------------------------------------------------------- + +def generate_basic(): + comment = ( + "# cb-basic: Single relay (helix) with default Commit-Boost config.\n" + "#\n" + "# Tests the core MEV pipeline through Commit-Boost with a single Helix\n" + "# relay as the only relay endpoint." + ) + images = { + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, + } + toml = build_cb_toml_basic(950, 4000) + mev_params = build_mev_params("helix", images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + COMMON_NETWORK_PARAMS, + ]) + "\n" + + +def generate_multiple_relays(): + comment = ( + "# cb-multiple-relays: Two relays (helix + flashbots) behind a single\n" + "# Commit-Boost sidecar.\n" + "#\n" + "# Tests that CB correctly routes get_header requests to both relays,\n" + "# aggregating responses and selecting the best bid." + ) + images = { + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_relay_image": MEV_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, + } + toml = build_cb_toml_basic(950, 4000) + mev_params = build_mev_params(["helix", "flashbots"], images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + COMMON_NETWORK_PARAMS, + ]) + "\n" + + +def generate_skip_sigverify(): + comment = ( + "# cb-skip-sigverify: Signature verification disabled for header responses.\n" + "#\n" + "# Tests the CB fast path where BLS verification is skipped. This trades\n" + "# correctness for speed — useful to verify that the path exists and is\n" + "# reachable under load." + ) + images = { + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, + } + toml = build_cb_toml_basic(950, 4000, extra_pbs_lines=["skip_sigverify = true"]) + mev_params = build_mev_params("helix", images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + COMMON_NETWORK_PARAMS, + ]) + "\n" + + +def generate_timing_games(): + comment = ( + "# cb-timing-games: Aggressive timing game configuration.\n" + "#\n" + "# Tests CB's ability to orchestrate repeated get_header polls with\n" + "# short timeouts in order to arrive at the best bid as late as possible\n" + "# in the slot. Per-relay timing overrides are enabled for all relays." + ) + images = { + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_relay_image": MEV_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, + } + toml = build_cb_toml_basic( + 400, + 2000, + per_relay_lines=[ + "enable_timing_games = true", + "target_first_request_ms = 100", + "frequency_get_header_ms = 200", + ], + ) + mev_params = build_mev_params(["helix", "flashbots"], images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + COMMON_NETWORK_PARAMS, + ]) + "\n" + + +def generate_extra_validation(): + comment = ( + "# cb-extra-validation: Enable extra validation of get_header responses\n" + "# via a local execution layer client.\n" + "#\n" + "# Tests that CB will RPC-call the execution client to verify block\n" + "# parameters before returning a header to the beacon node." + ) + images = { + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, + } + toml = build_cb_toml_basic( + 950, + 4000, + extra_pbs_lines=[ + "extra_validation_enabled = true", + 'rpc_url = "http://el-1-geth-lighthouse:8545"', + ], + ) + mev_params = build_mev_params("helix", images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + COMMON_NETWORK_PARAMS, + ]) + "\n" + + +def generate_mux(pubkeys_node0, pubkeys_node1): + comment = ( + "# cb-mux: Multiplexed relay routing per validator node.\n" + "#\n" + "# Routes all 128 validators from node-0 exclusively to the Helix relay and\n" + "# all 128 validators from node-1 exclusively to the Flashbots relay.\n" + "# This tests CB's ability to partition the validator set and apply\n" + "# per-mux timeout and relay configurations." + ) + images = { + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_relay_image": MEV_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, + } + toml = build_cb_toml_mux(pubkeys_node0, pubkeys_node1) + mev_params = build_mev_params(["helix", "flashbots"], images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + MUX_NETWORK_PARAMS, + ]) + "\n" + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description="Generate Kurtosis YAML configs for Commit-Boost testing." + ) + parser.add_argument( + "--output-dir", + default="configs/generated", + help="Directory to write generated YAML configs (default: kurtosis-configs).", + ) + args = parser.parse_args() + + output_dir = os.path.abspath(args.output_dir) + os.makedirs(output_dir, exist_ok=True) + + pubkeys_node0 = load_pubkeys("node-0-pubkeys.json") + pubkeys_node1 = load_pubkeys("node-1-pubkeys.json") + + scenarios = { + "cb-basic.yml": generate_basic(), + "cb-multiple-relays.yml": generate_multiple_relays(), + "cb-skip-sigverify.yml": generate_skip_sigverify(), + "cb-timing-games.yml": generate_timing_games(), + "cb-extra-validation.yml": generate_extra_validation(), + "cb-mux.yml": generate_mux(pubkeys_node0, pubkeys_node1), + } + + for filename, content in scenarios.items(): + path = os.path.join(output_dir, filename) + with open(path, "w") as f: + f.write(content) + print(f"Generated {path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/run-and-verify.sh b/scripts/run-and-verify.sh index be471c0..1b459b4 100755 --- a/scripts/run-and-verify.sh +++ b/scripts/run-and-verify.sh @@ -12,19 +12,20 @@ set -euo pipefail # Defaults +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$(dirname "$SCRIPT_DIR")" ENCLAVE="CB-Testnet" CONFIG="" -PACKAGE="github.com/ethpandaops/ethereum-package" +PACKAGE="$REPO_DIR/ethereum-package" KEEP=false JSON_FLAG="" +JSON_DIR_FLAG="" STRICT_FLAG="" LIVE_METRICS_FLAG="" TIMEOUT=3600 MIN_EPOCHS=2 TARGET_EPOCH=7 VERBOSE="" -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_DIR="$(dirname "$SCRIPT_DIR")" usage() { echo "Usage: $0 [OPTIONS]" @@ -32,9 +33,10 @@ usage() { echo "Options:" echo " --config FILE Kurtosis config file (default: configs/basic-pbs.yml)" echo " --enclave NAME Enclave name (default: CB-Testnet)" - echo " --package PATH ethereum-package path or ref (default: ethpandaops/ethereum-package)" + echo " --package PATH ethereum-package path or ref (default: ./ethereum-package)" echo " --keep Don't tear down the enclave on exit" echo " --json Output JSON report" + echo " --json-dir DIR Save JSON report to DIR/{enclave}.json (implies --json)" echo " --strict Promote WARN to FAIL (zero bids, zero deliveries)" echo " --live-metrics Show counter deltas every 30s during observation" echo " --timeout SECS Readiness timeout (default: 1500)" @@ -52,6 +54,7 @@ while [[ $# -gt 0 ]]; do --package) PACKAGE="$2"; shift 2;; --keep) KEEP=true; shift;; --json) JSON_FLAG="--json"; shift;; + --json-dir) JSON_DIR_FLAG="--output-dir $2"; JSON_FLAG="--json"; shift 2;; --strict) STRICT_FLAG="--strict"; shift;; --live-metrics) LIVE_METRICS_FLAG="--live-metrics"; shift;; --timeout) TIMEOUT="$2"; shift 2;; @@ -68,6 +71,11 @@ if [[ -z "$CONFIG" ]]; then CONFIG="$REPO_DIR/configs/basic-pbs.yml" fi +# Default --json implies auto-save to repo root +if [[ -n "$JSON_FLAG" && -z "$JSON_DIR_FLAG" ]]; then + JSON_DIR_FLAG="--output-dir $REPO_DIR" +fi + # Resolve config path relative to CWD if [[ ! -f "$CONFIG" ]]; then echo "Config file not found: $CONFIG" @@ -110,12 +118,14 @@ echo "Enclave '$ENCLAVE' is up. Starting verification..." echo "" # Step 3: Run verification -cargo run --manifest-path "$REPO_DIR/Cargo.toml" --release -- \ +cargo run --bin cb-verify --manifest-path "$REPO_DIR/Cargo.toml" --release -- \ --enclave "$ENCLAVE" \ + --config "$CONFIG" \ --timeout "$TIMEOUT" \ --min-epochs "$MIN_EPOCHS" \ --target-epoch "$TARGET_EPOCH" \ $JSON_FLAG \ + $JSON_DIR_FLAG \ $STRICT_FLAG \ $LIVE_METRICS_FLAG \ $VERBOSE diff --git a/src/bin/test_mux.rs b/src/bin/test_mux.rs new file mode 100644 index 0000000..e22120d --- /dev/null +++ b/src/bin/test_mux.rs @@ -0,0 +1,475 @@ +//! Quick mux routing diagnostic. +//! +//! Fetches CB PBS logs from a running enclave, parses them, and checks +//! mux routing against the provided config. No observation window, no +//! epoch waiting. Just: fetch → parse → check. +//! +//! Usage: +//! cargo run --release --bin test_mux -- +//! +//! Example: +//! cargo run --release --bin test_mux -- CB-Testnet configs/generated/cb-mux.yml + +use std::process::Command; + +fn main() { + let args: Vec = std::env::args().collect(); + if args.len() != 3 { + eprintln!("Usage: {} ", args[0]); + eprintln!("Example: {} CB-Testnet configs/generated/cb-mux.yml", args[0]); + std::process::exit(1); + } + + let enclave = &args[1]; + let config_path = &args[2]; + + // Step 1: Parse mux config + println!("=== Parsing mux config: {config_path} ==="); + let entries = match parse_mux_config(config_path) { + Ok(Some(e)) => { + println!("Found {} mux entries:", e.len()); + for entry in &e { + println!(" {} → relay={} pubkeys={}", entry.id, entry.relay_identity, entry.validator_pubkeys.len()); + } + e + } + Ok(None) => { + println!("No [[mux]] sections found in config. Nothing to check."); + std::process::exit(0); + } + Err(e) => { + eprintln!("ERROR parsing config: {e}"); + std::process::exit(1); + } + }; + + // Step 2: Discover CB PBS services + println!("\n=== Discovering CB PBS services in enclave: {enclave} ==="); + let services = match discover_services(enclave) { + Ok(s) => s, + Err(e) => { + eprintln!("ERROR discovering services: {e}"); + std::process::exit(1); + } + }; + println!("Found {} CB PBS service(s): {:?}", services.len(), services); + + if services.is_empty() { + eprintln!("ERROR: No CB PBS services found"); + std::process::exit(1); + } + + // Step 3: Fetch and parse logs from each service + println!("\n=== Fetching logs ==="); + let mut all_events: Vec = Vec::new(); + let log_file = format!("/tmp/test_mux_{}.log", enclave); + + for service in &services { + println!("\n--- {service} ---"); + match fetch_logs(enclave, service) { + Ok(logs) => { + if logs.is_empty() { + println!(" (no relevant log lines found)"); + continue; + } + // Write raw logs to file for debugging + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(&log_file) { + use std::io::Write; + let _ = writeln!(f, "=== {service} ==="); + let _ = writeln!(f, "{}", logs); + } + + let mut parsed = 0; + let mut failed = 0; + for line in logs.lines() { + match parse_line(line) { + Some(event) => { + parsed += 1; + all_events.push(event); + } + None => { + failed += 1; + if failed <= 3 { + println!(" PARSE FAIL: {}", &line[..line.len().min(120)]); + } + } + } + } + println!(" Parsed: {parsed} lines, Failed: {failed} lines"); + } + Err(e) => { + println!(" ERROR: {e}"); + } + } + } + println!("\nRaw logs written to: {log_file}"); + + // Step 4: Print sample of ALL parsed events and collect unique messages + let mut all_messages: Vec = all_events.iter().map(|e| e.message.clone()).collect(); + all_messages.sort(); + all_messages.dedup(); + println!("\n=== Unique messages found ({} total) ===", all_messages.len()); + for msg in &all_messages { + let count = all_events.iter().filter(|e| &e.message == msg).count(); + println!(" ({}) {}", count, msg); + } + + println!("\n=== Sample of all parsed events (first 10) ==="); + for (i, event) in all_events.iter().take(10).enumerate() { + let pk_short = event.validator.as_ref().map(|v| if v.len() > 16 { &v[..16] } else { v }); + println!(" #{} msg={:?} slot={:?} mux={:?} relay={:?} val={:?}", + i, event.message, event.slot, event.mux_id, event.relay_id, pk_short); + } + if all_events.len() > 10 { + println!(" ... and {} more", all_events.len() - 10); + } + + // Step 5: Filter to mux events + println!("\n=== Mux Events ==="); + let mux_events: Vec<&CbEvent> = all_events + .iter() + .filter(|e| { + e.message.starts_with("using mux") + || e.message.starts_with("received new header") + || e.message.starts_with("auction winner") + }) + .collect(); + + println!("Total mux events: {}", mux_events.len()); + for event in &mux_events { + let pk_short = event.validator.as_ref().map(|v| if v.len() > 20 { &v[..20] } else { v }); + println!( + " [{}] slot={:?} mux={:?} relay={:?} val={:?}", + event.message, + event.slot, + event.mux_id, + event.relay_id, + pk_short + ); + } + + // Step 5: Check mux routing + println!("\n=== Mux Routing Check ==="); + let mut violations = 0; + let mut checked = 0; + + for event in &mux_events { + if let Some(ref pk) = event.validator { + let pk_norm = pk.to_lowercase(); + for entry in &entries { + if entry.validator_pubkeys.iter().any(|e| e.to_lowercase() == pk_norm) { + checked += 1; + if let Some(ref actual_mux) = event.mux_id { + if actual_mux != &entry.id { + violations += 1; + println!( + " VIOLATION: pubkey {} should route to '{}' but routed to '{}'", + &pk[..20.min(pk.len())], + entry.id, + actual_mux + ); + } + } + } + } + } + } + + println!("\n=== Result ==="); + if violations > 0 { + println!("FAIL: {violations} routing violation(s) out of {checked} checked"); + std::process::exit(1); + } else if checked == 0 { + println!("WARN: No mux events matched to config pubkeys. Events may not have proposer_pubkey fields."); + std::process::exit(0); + } else { + println!("PASS: All {checked} mux routing decisions are correct"); + std::process::exit(0); + } +} + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +struct MuxEntry { + id: String, + relay_identity: String, + validator_pubkeys: Vec, +} + +struct CbEvent { + message: String, + slot: Option, + validator: Option, + relay_id: Option, + mux_id: Option, +} + +// --------------------------------------------------------------------------- +// Config parsing +// --------------------------------------------------------------------------- + +fn parse_mux_config(path: &str) -> Result>, Box> { + let raw = std::fs::read_to_string(path)?; + + let template = if path.ends_with(".yml") || path.ends_with(".yaml") { + let parsed: serde_yaml::Value = serde_yaml::from_str(&raw)?; + parsed + .get("mev_params") + .and_then(|p| p.get("commit_boost_config")) + .and_then(|c| c.as_str()) + .ok_or("No mev_params.commit_boost_config found")? + .to_string() + } else { + raw + }; + + if !template.contains("[[mux]]") { + return Ok(None); + } + + let mut entries = Vec::new(); + let mut lines = template.lines().peekable(); + + while let Some(line) = lines.next() { + if line.trim() == "[[mux]]" { + let entry = parse_mux_section(&mut lines); + entries.push(entry); + } + } + + if entries.is_empty() { + return Ok(None); + } + + Ok(Some(entries)) +} + +fn parse_mux_section<'a>(lines: &mut std::iter::Peekable>) -> MuxEntry { + let mut id = None; + let mut pubkeys = None; + + loop { + let is_header = lines.peek().map(|l| l.trim().starts_with("[[")).unwrap_or(false); + if is_header { + let header = lines.peek().unwrap().trim().to_string(); + if header.starts_with("[[mux.relays]]") { + let _ = lines.next(); + continue; + } + break; + } + + let Some(line) = lines.next() else { break }; + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + if let Some((key, val)) = trimmed.split_once('=') { + let key = key.trim(); + let val = val.trim(); + match key { + "id" => id = Some(val.trim_matches('"').to_string()), + "validator_pubkeys" => pubkeys = Some(parse_pubkey_array(val, lines)), + _ => {} + } + } + } + + let id = id.unwrap_or_default(); + let relay_identity = if let Some(pos) = id.rfind("to_") { + let ident = id[pos + 3..].trim().to_string(); + if !ident.is_empty() { ident } else { id.clone() } + } else { + id.clone() + }; + let pubkeys = pubkeys.unwrap_or_default(); + + MuxEntry { id, relay_identity, validator_pubkeys: pubkeys } +} + +fn parse_pubkey_array(rest: &str, lines: &mut std::iter::Peekable) -> Vec { + let mut accum = rest.to_string(); + if !accum.trim_end().ends_with(']') { + loop { + let Some(next) = lines.next() else { break }; + accum.push('\n'); + accum.push_str(next); + if next.trim().ends_with(']') { break; } + } + } + let raw = accum.trim(); + let start = raw.find('[').unwrap_or(0); + let end = raw.rfind(']').unwrap_or(raw.len()); + raw[start + 1..end] + .split(',') + .map(|s| s.trim().trim_matches('"').to_string()) + .filter(|s| !s.is_empty()) + .collect() +} + +// --------------------------------------------------------------------------- +// Service discovery +// --------------------------------------------------------------------------- + +fn discover_services(enclave: &str) -> Result, Box> { + let output = Command::new("kurtosis") + .args(["enclave", "inspect", "--full-uuids", enclave]) + .output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("kurtosis enclave inspect failed: {stderr}").into()); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let mut services = Vec::new(); + + for line in stdout.lines() { + let lower = line.to_lowercase(); + if lower.contains("commit-boost") && lower.contains("running") { + // Extract service name (first column) + if let Some(name) = line.split_whitespace().next() { + services.push(name.to_string()); + } + } + } + + Ok(services) +} + +// --------------------------------------------------------------------------- +// Log fetching +// --------------------------------------------------------------------------- + +fn fetch_logs(enclave: &str, service: &str) -> Result> { + let output = Command::new("kurtosis") + .args(["service", "logs", enclave, service, "-n", "200000"]) + .output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("kurtosis service logs failed: {stderr}").into()); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + + // Filter to relevant lines + let result: String = stdout + .lines() + .filter(|line| { + line.contains("using mux config") + || line.contains("received new header") + || line.contains("auction winner") + || line.contains("received unblinded block") + }) + .collect::>() + .join("\n"); + + Ok(result) +} + +// --------------------------------------------------------------------------- +// Log parsing +// --------------------------------------------------------------------------- + +fn strip_ansi_codes(s: &str) -> String { + let mut result = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\x1b' { + if chars.peek() == Some(&'[') { + chars.next(); + while let Some(&ch) = chars.peek() { + chars.next(); + if ch.is_ascii_alphabetic() { break; } + } + } + } else { + result.push(c); + } + } + result +} + +fn parse_line(line: &str) -> Option { + let line = line.trim(); + if line.is_empty() { return None; } + + // Strip kurtosis prefix: "[service-name] rest" + let line = if line.starts_with('[') { + if let Some(pos) = line.find(']') { + line[pos + 1..].trim_start() + } else { line } + } else { line }; + + // Strip ANSI escape codes + let line = strip_ansi_codes(&line); + + // Find message after "LEVEL : " or "LEVEL " + let after_level: String = if let Some(pos) = line.find(" : ") { + line[pos + 3..].to_string() + } else { + let levels = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR"]; + let mut found = line.clone(); + for lvl in &levels { + if let Some(pos) = line.find(&format!(" {} ", lvl)) { + found = line[pos + lvl.len() + 2..].to_string(); + break; + } + if line.starts_with(lvl) { + found = line[lvl.len()..].trim_start().to_string(); + break; + } + } + found + }; + + // Find message/key boundary: first " key=" where key is a valid identifier + let mut message_end = after_level.len(); + let bytes = after_level.as_bytes(); + for i in 0..bytes.len() { + if bytes[i] == b' ' { + let rest = &after_level[i + 1..]; + if rest.is_empty() { continue; } + let first = rest.as_bytes()[0]; + if first.is_ascii_alphabetic() || first == b'_' { + if let Some(eq_pos) = rest.find('=') { + let key = &rest[..eq_pos]; + if key.chars().all(|c| c.is_alphanumeric() || c == '_') { + let after_eq = &rest[eq_pos + 1..]; + if !after_eq.is_empty() { + message_end = i; + break; + } + } + } + } + } + } + + let message = after_level[..message_end].trim().to_string(); + let kv_part = &after_level[message_end..]; + + let mut slot = None; + let mut validator = None; + let mut relay_id = None; + let mut mux_id = None; + + for kv in kv_part.split_whitespace() { + if let Some((key, val)) = kv.split_once('=') { + let val = val.trim_matches('"'); + match key { + "slot" => { slot = val.parse().ok(); } + "validator" | "pubkey" => { validator = Some(val.to_string()); } + "relay_id" => { relay_id = Some(val.to_string()); } + "mux_id" => { mux_id = Some(val.to_string()); } + _ => {} + } + } + } + + Some(CbEvent { message, slot, validator, relay_id, mux_id }) +} diff --git a/src/bin/test_relay.rs b/src/bin/test_relay.rs new file mode 100644 index 0000000..17e6876 --- /dev/null +++ b/src/bin/test_relay.rs @@ -0,0 +1,226 @@ +//! Quick relay API diagnostic. +//! +//! Tests all relay data API endpoints with proper slot filtering. +//! No observation window needed — just query the relay directly. +//! +//! Usage: +//! cargo run --release --bin test_relay -- [pubkey] +//! +//! Examples: +//! cargo run --release --bin test_relay -- http://127.0.0.1:59945 128 160 +//! cargo run --release --bin test_relay -- http://127.0.0.1:59945 128 160 0x889dbdf3bd68af1f6fd84cb6173b1fa1f7c5e6ba63297dc1e2f45cd1a82bb6231ba832adc5228143c5cff3ef0b1caae2 + +use std::time::Duration; + +#[tokio::main] +async fn main() { + let args: Vec = std::env::args().collect(); + if args.len() < 4 { + eprintln!("Usage: {} [pubkey]", args[0]); + eprintln!("Example: {} http://127.0.0.1:59945 128 160", args[0]); + std::process::exit(1); + } + + let relay_url = &args[1]; + let start_slot: u64 = args[2].parse().expect("invalid start_slot"); + let end_slot: u64 = args[3].parse().expect("invalid end_slot"); + let pubkey = args.get(4).map(|s| s.as_str()); + + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .expect("failed to build HTTP client"); + + let base = relay_url.trim_end_matches('/'); + + // === 0. Check if relay has ANY delivered payloads at all === + println!("=== 0. Latest delivered payloads (no slot filter) ==="); + { + let url = format!("{base}/relay/v1/data/bidtraces/proposer_payload_delivered"); + let req = client.get(&url).query(&[("limit", "5")]); + match send::>(req).await { + Ok(payloads) => { + println!(" Total payloads available: {}", payloads.len()); + for p in &payloads { + println!(" slot={} value={} proposer={}...", p.slot, p.value, &p.proposer_pubkey[..20.min(p.proposer_pubkey.len())]); + } + } + Err(e) => println!(" FAIL: {e}"), + } + } + + // === 1. Delivered payloads filtered by slot === + println!("=== 1. Delivered payloads (slot {start_slot}..={end_slot}) ==="); + { + let url = format!("{base}/relay/v1/data/bidtraces/proposer_payload_delivered"); + let mut all = Vec::new(); + let mut cursor: Option = None; + let mut page = 0; + let limit_str = String::from("200"); + let start_slot_str = start_slot.to_string(); + + loop { + page += 1; + let mut params: Vec<(&str, String)> = vec![ + ("limit", limit_str.clone()), + ("slot", start_slot_str.clone()), + ]; + if let Some(ref c) = cursor { + params.push(("cursor", c.clone())); + } + + let req = client.get(&url).query(¶ms); + match send::>(req).await { + Ok(payloads) => { + if payloads.is_empty() { + println!(" Page {page}: empty, done"); + break; + } + let min_slot = payloads.iter().filter_map(|p| p.slot.parse::().ok()).min().unwrap_or(0); + let max_slot = payloads.iter().filter_map(|p| p.slot.parse::().ok()).max().unwrap_or(0); + let page_len = payloads.len(); + let in_range: Vec<_> = payloads.into_iter().filter(|p| { + let s: u64 = p.slot.parse().unwrap_or(0); + s >= start_slot && s <= end_slot + }).collect(); + let count = in_range.len(); + all.extend(in_range); + println!(" Page {page}: {page_len} payloads (slots {min_slot}..={max_slot}), {count} in range, {} total", + all.len()); + if min_slot < start_slot { break; } + cursor = all.last().map(|p: &PayloadDelivered| p.block_number.clone()); + } + Err(e) => { + println!(" FAIL: {e}"); + break; + } + } + if page >= 50 { break; } + } + println!(" Total delivered in range: {}", all.len()); + for p in all.iter().take(5) { + let pk_short = if p.proposer_pubkey.len() > 20 { &p.proposer_pubkey[..20] } else { &p.proposer_pubkey }; + println!(" slot={} value={} proposer={}...", p.slot, p.value, pk_short); + } + } + + // === 2. Builder blocks received filtered by slot === + println!("\n=== 2. Builder blocks received (slot {start_slot}..={end_slot}) ==="); + { + let url = format!("{base}/relay/v1/data/bidtraces/builder_blocks_received"); + let mut all = Vec::new(); + + // Query each slot individually (the API supports slot filter) + for slot in start_slot..=end_slot { + let slot_str = slot.to_string(); + let limit_str = "200".to_string(); + let req = client.get(&url).query(&[("slot", &slot_str), ("limit", &limit_str)]); + match send::>(req).await { + Ok(blocks) => { + if !blocks.is_empty() { + println!(" Slot {slot}: {} blocks", blocks.len()); + for b in &blocks { + let val_short = if b.value.len() > 12 { &b.value[..12] } else { &b.value }; + println!(" builder={}... value={val_short}", &b.builder_pubkey[..20.min(b.builder_pubkey.len())]); + } + all.extend(blocks); + } + } + Err(e) => { + println!(" Slot {slot}: FAIL: {e}"); + } + } + } + println!(" Total builder blocks in range: {}", all.len()); + } + + // === 3. Validator registration check === + println!("\n=== 3. Validator registration ==="); + if let Some(pk) = pubkey { + let url = format!("{base}/relay/v1/data/validator_registration"); + let req = client.get(&url).query(&[("pubkey", pk)]); + match send::(req).await { + Ok(reg) => { + println!(" Registered: YES"); + if let Some(msg) = reg.get("message") { + println!(" Fee recipient: {}", msg.get("fee_recipient").and_then(|v| v.as_str()).unwrap_or("?")); + println!(" Gas limit: {}", msg.get("gas_limit").and_then(|v| v.as_str()).unwrap_or("?")); + println!(" Timestamp: {}", msg.get("timestamp").and_then(|v| v.as_str()).unwrap_or("?")); + } + } + Err(e) => { + println!(" Registered: NO ({e})"); + } + } + } else { + println!(" (skipped — no pubkey provided)"); + } + + // === 4. Summary === + println!("\n=== Summary ==="); + println!("Relay: {base}"); + println!("Slot range: {start_slot}..={end_slot} ({} slots)", end_slot - start_slot + 1); + println!("All queries completed successfully"); +} + +async fn send(req: reqwest::RequestBuilder) -> Result { + let resp = req.send().await.map_err(|e| format!("HTTP error: {e}"))?; + let status = resp.status(); + if !status.is_success() { + let body = resp.text().await.unwrap_or_default(); + return Err(format!("HTTP {}: {}", status, body)); + } + resp.json::().await.map_err(|e| format!("JSON error: {e}")) +} + +#[derive(serde::Deserialize, Debug, Clone)] +struct PayloadDelivered { + slot: String, + block_hash: String, + value: String, + #[serde(default)] + proposer_pubkey: String, + #[serde(default)] + builder_pubkey: String, + #[serde(default)] + block_number: String, + #[serde(default)] + parent_hash: String, + #[serde(default)] + proposer_fee_recipient: String, + #[serde(default)] + gas_limit: String, + #[serde(default)] + gas_used: String, + #[serde(default)] + num_tx: String, +} + +#[derive(serde::Deserialize, Debug, Clone)] +struct BuilderBlock { + slot: String, + block_hash: String, + value: String, + #[serde(default)] + builder_pubkey: String, + #[serde(default)] + proposer_pubkey: String, + #[serde(default)] + block_number: String, + #[serde(default)] + parent_hash: String, + #[serde(default)] + proposer_fee_recipient: String, + #[serde(default)] + gas_limit: String, + #[serde(default)] + gas_used: String, + #[serde(default)] + num_tx: String, + #[serde(default)] + timestamp: String, + #[serde(default)] + timestamp_ms: String, +} + +// Add this as test 0 at the start of main(), before the slot range tests diff --git a/src/checks/chain_health.rs b/src/checks/chain_health.rs index 56478a7..41a83ac 100644 --- a/src/checks/chain_health.rs +++ b/src/checks/chain_health.rs @@ -29,10 +29,18 @@ pub async fn check_missed_slots( end_slot: u64, threshold: f64, ) -> CheckResult { - let total = end_slot.saturating_sub(start_slot); - if total == 0 { + if start_slot > end_slot { return CheckResult::fail("missed_slots", 2, "Invalid slot range"); } + // Single-slot window: nothing meaningful to check + if start_slot == end_slot { + return CheckResult::skip( + "missed_slots", + 2, + format!("Single-slot window (slot {}), skipping missed slot check", start_slot), + ); + } + let total = end_slot - start_slot; let mut missed = 0u64; for slot in start_slot..end_slot { diff --git a/src/checks/mod.rs b/src/checks/mod.rs index e8e04be..b08bd54 100644 --- a/src/checks/mod.rs +++ b/src/checks/mod.rs @@ -87,5 +87,6 @@ impl CheckResult { pub mod cb_metrics; pub mod chain_health; +pub mod mux_routing; pub mod payload_matching; pub mod relay_pipeline; diff --git a/src/checks/mux_routing.rs b/src/checks/mux_routing.rs new file mode 100644 index 0000000..c71e526 --- /dev/null +++ b/src/checks/mux_routing.rs @@ -0,0 +1,977 @@ +//! MUX routing verification check. +//! +//! Extracts `[[mux]]` sections from a Commit-Boost config TOML file. +//! If the config contains mux rules, verifies that CB's PBS service +//! correctly routes getHeader requests according to the mux config. +//! +//! Verification works by fetching CB PBS container logs and parsing +//! structured INFO lines. Each line records key=value pairs that are +//! cross-referenced against the mux configuration. +//! +//! Relevant CB log lines (from commit-boost-client crates): +//! "using mux config" — mux_id, relays, pubkey (DEBUG) +//! "received new header" — relay_id, slot, validator, value_eth, block_hash (INFO) +//! "auction winner" — relay_id, value_eth, block_hash (INFO) +//! "new request" (submit_blinded_blocks) — slot, validator (INFO) +//! "received unblinded block (v1/v2)" — (INFO) +//! "CRITICAL: no payload received" — block_hash (ERROR) +//! +//! If no mux sections are found, the check is skipped. + +use std::collections::{HashMap, HashSet}; + +use crate::checks::CheckResult; +use tracing::{info, warn}; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/// A single parsed mux entry from the CB config. +pub struct MuxEntry { + pub id: String, + pub relay_identity: String, + pub validator_pubkeys: Vec, +} + +/// Parsed event from a CB INFO/DEBUG log line. +#[derive(Debug, Clone)] +pub struct CbEvent { + /// The log message text (e.g., "using mux config", "received new header") + pub message: String, + /// Key=value pairs extracted from the log line. + pub fields: HashMap, + /// The slot number if present in the fields. + pub slot: Option, + /// The validator pubkey if present. + pub validator: Option, + /// The relay_id if present. + pub relay_id: Option, + /// The mux_id if present. + pub mux_id: Option, +} + +// --------------------------------------------------------------------------- +// Config parsing (unchanged from before) +// --------------------------------------------------------------------------- + +/// Parse a Commit-Boost config file and extract mux routing entries. +/// +/// Supports two file formats: +/// - `.toml` — raw Commit-Boost config (possibly with Go template expressions) +/// - `.yml`/`.yaml` — Kurtosis YAML with `mev_params.commit_boost_config` field +/// +/// Returns `Ok(Some(entries))` if mux sections were found, +/// `Ok(None)` if no mux sections (check will SKIP), +/// `Err` if parsing fails. +pub fn extract_mux_from_config(path: &str) -> eyre::Result>> { + let raw = std::fs::read_to_string(path) + .map_err(|e| eyre::eyre!("Failed to read config '{path}': {e}"))?; + + let template = if path.ends_with(".toml") { + raw + } else if path.ends_with(".yml") || path.ends_with(".yaml") { + extract_commit_boost_config_from_yaml(&raw)? + } else { + return Err(eyre::eyre!( + "Unrecognized config format. Expected .toml (CB config) or .yml/.yaml (Kurtosis config), got: {path}" + )); + }; + + parse_mux_from_toml_template(&template) +} + +fn extract_commit_boost_config_from_yaml(raw: &str) -> eyre::Result { + use serde_yaml::Value as YamlValue; + + let parsed: YamlValue = serde_yaml::from_str(raw) + .map_err(|e| eyre::eyre!("Failed to parse Kurtosis YAML config: {e}"))?; + + let template = parsed + .get("mev_params") + .and_then(|p| p.get("commit_boost_config")) + .and_then(|c| c.as_str()) + .ok_or_else(|| { + eyre::eyre!( + "No mev_params.commit_boost_config found in Kurtosis YAML config" + ) + })?; + + Ok(template.to_string()) +} + +fn has_mux_sections(text: &str) -> bool { + text.lines().any(|l| l.trim() == "[[mux]]") +} + +fn parse_mux_from_toml_template(template: &str) -> eyre::Result>> { + if !has_mux_sections(template) { + return Ok(None); + } + + let mut entries: Vec = Vec::new(); + let mut lines = template.lines().peekable(); + + while let Some(line) = lines.next() { + let trimmed = line.trim(); + if trimmed != "[[mux]]" { + continue; + } + + let entry = match parse_one_mux_section(&mut lines) { + Ok(Some(e)) => e, + Ok(None) => continue, + Err(e) => return Err(eyre::eyre!("Failed to parse mux section: {e}")), + }; + + entries.push(entry); + } + + if entries.is_empty() { + return Ok(None); + } + + Ok(Some(entries)) +} + +fn relay_identity_from_mux_id(id: &str) -> String { + if let Some(pos) = id.rfind("to_") { + let ident = id[pos + 3..].trim().to_string(); + if !ident.is_empty() { + return ident; + } + } + id.to_string() +} + +fn parse_one_mux_section<'a>( + lines: &mut std::iter::Peekable>, +) -> eyre::Result> { + let mut id: Option = None; + let mut pubkeys: Option> = None; + + loop { + let is_section_header = lines + .peek() + .map(|l| l.trim().starts_with("[[")) + .unwrap_or(false); + + if is_section_header { + let header = lines.peek().unwrap().trim().to_string(); + if header.starts_with("[[mux.relays]]") { + let _ = lines.next(); + let _ = parse_mux_relay_body(lines)?; + continue; + } + break; + } + + let Some(line) = lines.next() else { + break; + }; + let trimmed = line.trim(); + + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + if let Some((key, raw_val)) = parse_key_value(trimmed) { + match key { + "id" => { + id = Some(raw_val.trim_matches('"').to_string()); + } + "validator_pubkeys" => { + pubkeys = Some(parse_pubkey_array(raw_val, lines)?); + } + _ => {} + } + } + } + + let id = id.ok_or_else(|| eyre::eyre!("[[mux]] section missing 'id' field"))?; + let relay_identity = relay_identity_from_mux_id(&id); + let pubkeys = pubkeys + .ok_or_else(|| eyre::eyre!("[[mux]] section '{id}' missing 'validator_pubkeys'"))?; + + Ok(Some(MuxEntry { + id, + relay_identity, + validator_pubkeys: pubkeys, + })) +} + +fn parse_mux_relay_body( + lines: &mut std::iter::Peekable, +) -> eyre::Result> { + loop { + if lines + .peek() + .map(|l| l.trim().starts_with("[[")) + .unwrap_or(false) + { + break; + } + + let Some(line) = lines.next() else { + break; + }; + let trimmed = line.trim(); + + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + if let Some((key, raw_val)) = parse_key_value(trimmed) { + if key == "url" { + let val = raw_val.trim_matches('"'); + return Ok(parse_relay_index_from_template(val)); + } + } + } + + Ok(None) +} + +fn parse_relay_index_from_template(val: &str) -> Option { + let val = val.trim(); + let stripped = val + .trim_start_matches("{{") + .trim_end_matches("}}") + .trim(); + let parts: Vec<&str> = stripped.split_whitespace().collect(); + if parts.len() >= 3 && parts[0] == "index" && parts[1] == ".Relays" { + parts[2].parse::().ok() + } else { + None + } +} + +fn parse_key_value(s: &str) -> Option<(&str, &str)> { + let eq_pos = s.find('=')?; + let key = s[..eq_pos].trim(); + let raw_val = s[eq_pos + 1..].trim(); + Some((key, raw_val)) +} + +fn parse_pubkey_array( + rest: &str, + lines: &mut std::iter::Peekable, +) -> eyre::Result> { + let mut accum = rest.to_string(); + + if !accum.trim_end().ends_with(']') { + loop { + let Some(next) = lines.next() else { + break; + }; + accum.push('\n'); + accum.push_str(next); + if next.trim().ends_with(']') { + break; + } + } + } + + let raw = accum.trim(); + let start = raw.find('[').ok_or_else(|| { + eyre::eyre!("Could not find opening '[' in pubkey array: {raw:.50}...") + })?; + let end = raw.rfind(']').ok_or_else(|| { + eyre::eyre!("Could not find closing ']' in pubkey array: {raw:.50}...") + })?; + + let inner = &raw[start + 1..end]; + let mut pubkeys = Vec::new(); + for item in inner.split(',') { + let item = item.trim().trim_matches('"').trim(); + if item.is_empty() || item.contains("{{") || item.contains("}}") { + continue; + } + pubkeys.push(item.to_string()); + } + + Ok(pubkeys) +} + +fn normalize_pubkey(pk: &str) -> String { + pk.trim_start_matches("0x") + .trim_start_matches("0X") + .to_lowercase() +} + +// --------------------------------------------------------------------------- +// CB log parsing +// --------------------------------------------------------------------------- + +/// Strip ANSI escape codes from a string. +/// +/// ANSI codes look like `\x1b[32m` (color) or `\x1b[0m` (reset). +/// This removes them so we can parse the actual text content. +fn strip_ansi_codes(s: &str) -> String { + let mut result = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\x1b' { + // Skip the '[' and everything until we hit a letter (m, H, J, etc.) + if chars.peek() == Some(&'[') { + chars.next(); // skip '[' + while let Some(&ch) = chars.peek() { + chars.next(); + if ch.is_ascii_alphabetic() { + break; + } + } + } + } else { + result.push(c); + } + } + result +} + +/// Parse a CB tracing log line into a `CbEvent`. +/// +/// CB uses `tracing` with a compact format: +/// `timestamp LEVEL : message key=value key=value ...` +/// +/// We extract the message and all key=value pairs. +pub fn parse_cb_log_line(line: &str) -> Option { + let line = line.trim(); + if line.is_empty() { + return None; + } + + // Strip kurtosis prefix: "[service-name] rest" + let line = if line.starts_with('[') { + if let Some(pos) = line.find(']') { + line[pos + 1..].trim_start() + } else { + line + } + } else { + line + }; + + // Strip ANSI escape codes (e.g., "\x1b[32m" for green text) + let line = strip_ansi_codes(line); + // Continue parsing with the cleaned line (now owned String) + // All subsequent code uses `line` as a String, not &str + + // Find the message portion after "LEVEL : " or "LEVEL ". + let after_level: String = if let Some(pos) = line.find(" : ") { + line[pos + 3..].to_string() + } else { + let levels = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR"]; + let mut found = line.clone(); + for lvl in &levels { + if let Some(pos) = line.find(&format!(" {} ", lvl)) { + found = line[pos + lvl.len() + 2..].to_string(); + break; + } + if line.starts_with(lvl) { + found = line[lvl.len()..].trim_start().to_string(); + break; + } + } + found + }; + + // Find the message/key boundary by scanning for " key=" where key is + // a valid identifier (alphanumeric + underscore, starting with alpha). + // This is more reliable than a fixed list of known keys. + let mut message_end = after_level.len(); + let bytes = after_level.as_bytes(); + for i in 0..bytes.len() { + if bytes[i] == b' ' { + // Check if what follows is "key=" where key starts with alpha/underscore + let rest = &after_level[i + 1..]; + if rest.is_empty() { + continue; + } + let first = rest.as_bytes()[0]; + if first.is_ascii_alphabetic() || first == b'_' { + // Find the '=' after the key + if let Some(eq_pos) = rest.find('=') { + let key = &rest[..eq_pos]; + // Key must be all alphanumeric/underscore + if key.chars().all(|c| c.is_alphanumeric() || c == '_') { + // Value after '=' must not be empty (or must be quote) + let after_eq = &rest[eq_pos + 1..]; + if !after_eq.is_empty() { + message_end = i; + break; + } + } + } + } + } + } + + let message = after_level[..message_end].trim().to_string(); + let kv_part = &after_level[message_end..].to_string(); + + let mut fields = HashMap::new(); + let mut slot = None; + let mut validator = None; + let mut relay_id = None; + let mut mux_id = None; + + for kv in kv_part.split_whitespace() { + if let Some((key, val)) = kv.split_once('=') { + let val = val.trim_matches('"').to_string(); + fields.insert(key.to_string(), val.clone()); + + match key { + "slot" => { slot = val.parse().ok(); } + "validator" | "pubkey" => { validator = Some(normalize_pubkey(&val)); } + "relay_id" => { relay_id = Some(val); } + "mux_id" => { mux_id = Some(val); } + _ => {} + } + } + } + + Some(CbEvent { + message, + fields, + slot, + validator, + relay_id, + mux_id, + }) +} + + + +// --------------------------------------------------------------------------- +// Log fetching +// --------------------------------------------------------------------------- + +/// Fetch logs from a Kurtosis service, filtered to relevant mux/pbs lines. +/// +/// Fetches all logs and filters client-side. The `--regex-match` flag is +/// tried first as an optimization, but some kurtosis versions ignore it. +pub fn fetch_service_logs(enclave: &str, service: &str) -> eyre::Result { + info!( + "mux check: fetching logs from service '{service}' (enclave={enclave})..." + ); + + let output = std::process::Command::new("kurtosis") + .args([ + "service", "logs", enclave, service, + "-n", "200000", + ]) + .output() + .map_err(|e| eyre::eyre!("Failed to run 'kurtosis service logs': {e}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(eyre::eyre!( + "kurtosis service logs {enclave} {service} failed (rc={:?}): {}", + output.status.code(), + stderr.trim() + )); + } + + // Combine stdout and stderr — kurtosis writes to either depending on version. + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let all_logs = format!("{}\n{}", stdout, stderr); + + // Filter to relevant lines client-side. + let result: String = all_logs + .lines() + .filter(|line| { + line.contains("using mux config") + || line.contains("received new header") + || line.contains("auction winner") + || line.contains("received unblinded block") + || line.contains("CRITICAL: no payload") + }) + .collect::>() + .join("\n"); + + if result.is_empty() { + let sample: String = all_logs + .lines() + .filter(|l| !l.trim().is_empty()) + .take(3) + .collect::>() + .join("\n"); + warn!( + "mux check: service '{service}' returned no relevant log lines. Total: {} bytes stdout, {} bytes stderr. Sample:\n{}", + stdout.len(), + stderr.len(), + sample + ); + } else { + info!( + "mux check: service '{service}' returned {} relevant log line(s)", + result.lines().count() + ); + } + + Ok(result) +} + +// --------------------------------------------------------------------------- +// Verification +// --------------------------------------------------------------------------- + +/// Run the mux routing check by parsing CB PBS service logs. +/// +/// Fetches logs from each CB PBS service in the enclave, parses structured +/// events, and verifies that the mux routing decisions match the mux config. +/// +/// PASS: All logged routing decisions match the mux config. +/// FAIL: A pubkey was routed to the wrong mux/relay. +/// WARN: No relevant log lines found (no getHeader requests occurred). +pub async fn check_mux_routing( + enclave: &str, + cb_service_names: &[String], + entries: &[MuxEntry], +) -> CheckResult { + if entries.is_empty() { + return CheckResult::skip( + "mux.routing", + 1, + "No [[mux]] sections in CB config — nothing to verify", + ); + } + + // Build: normalized_pubkey → expected_mux_id + let mut expected_mux: HashMap = HashMap::new(); + for entry in entries { + for pk in &entry.validator_pubkeys { + expected_mux.insert(normalize_pubkey(pk), entry.id.clone()); + } + } + + // Fetch and parse logs from each CB PBS service. + let mut all_events: Vec = Vec::new(); + + for service_name in cb_service_names { + let logs = match fetch_service_logs(enclave, service_name) { + Ok(l) => l, + Err(e) => { + warn!( + "mux check: failed to fetch logs from service '{service_name}': {e}. \ + Skipping this service." + ); + continue; + } + }; + + if logs.is_empty() { + continue; + } + + for line in logs.lines() { + if let Some(event) = parse_cb_log_line(line) { + all_events.push(event); + } + } + } + + // Filter to events relevant to mux verification. + let mux_events: Vec<&CbEvent> = all_events + .iter() + .filter(|e| { + e.message.starts_with("using mux") + || e.message.starts_with("received new header") + || e.message.starts_with("auction winner") + || e.message.starts_with("received unblinded block") + || e.message.contains("CRITICAL: no payload") + }) + .collect(); + + let total_events = mux_events.len(); + + let data = serde_json::json!({ + "total_mux_entries": entries.len(), + "total_log_events": total_events, + "pubkeys_verified": 0, + "violations": [], + "violation_count": 0, + "mux_entries_seen": [], + }); + + if total_events == 0 { + return CheckResult::warn( + "mux.routing", + 1, + format!( + "No mux-related log lines found in any CB PBS service. \ + No getHeader requests were recorded — mux config is valid \ + but routing could not be verified at runtime. muxes=[{}]", + entries.iter().map(|e| e.id.as_str()).collect::>().join(", ") + ), + ).with_data(data); + } + + // Verify: for each "using mux config" event, does the pubkey match? + let mut violations: Vec = Vec::new(); + let mut pubkeys_verified: HashSet = HashSet::new(); + let mut mux_entries_seen: HashSet = HashSet::new(); + + for event in &mux_events { + if let Some(ref mux_id) = event.mux_id { + mux_entries_seen.insert(mux_id.clone()); + } + + if let Some(ref pk_norm) = event.validator { + pubkeys_verified.insert(pk_norm.clone()); + + if let Some(expected_mux_id) = expected_mux.get(pk_norm) { + if let Some(ref actual_mux_id) = event.mux_id { + if actual_mux_id != expected_mux_id { + let expected_relay = entries + .iter() + .find(|e| e.id == *expected_mux_id) + .map(|e| e.relay_identity.as_str()) + .unwrap_or("?"); + let actual_relay = entries + .iter() + .find(|e| e.id == *actual_mux_id) + .map(|e| e.relay_identity.as_str()) + .unwrap_or("?"); + + violations.push(serde_json::json!({ + "slot": event.slot, + "proposer_pubkey": format!("0x{pk_norm}"), + "routed_to_mux": actual_mux_id, + "routed_to_relay": actual_relay, + "expected_mux": expected_mux_id, + "expected_relay": expected_relay, + })); + + warn!( + "mux check: MISROUTING — pubkey 0x{pk_norm}.. should route to \ + '{expected_mux_id}' ({expected_relay}) but was routed to \ + '{actual_mux_id}' ({actual_relay})" + ); + } + } + } + } + } + + let data = serde_json::json!({ + "total_mux_entries": entries.len(), + "total_log_events": total_events, + "pubkeys_verified": pubkeys_verified.len(), + "violations": violations, + "violation_count": violations.len(), + "mux_entries_seen": mux_entries_seen.iter().cloned().collect::>(), + }); + + let mux_ids: Vec<&str> = entries.iter().map(|e| e.id.as_str()).collect(); + let mux_detail = format!("muxes=[{}]", mux_ids.join(", ")); + + if !violations.is_empty() { + CheckResult::fail( + "mux.routing", + 1, + format!( + "{} mux routing violation(s): CB PBS routed a pubkey to the wrong mux/relay. {}", + violations.len(), + mux_detail, + ), + ) + .with_data(data) + } else { + CheckResult::pass( + "mux.routing", + 1, + format!( + "All {} mux routing decision(s) verified ✓ CB PBS correctly routed \ + every getHeader request according to mux config. {}", + total_events, + mux_detail, + ), + ) + .with_data(data) + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_relay_identity_from_mux_id() { + assert_eq!(relay_identity_from_mux_id("node_0_to_helix"), "helix"); + assert_eq!(relay_identity_from_mux_id("node_1_to_flashbots"), "flashbots"); + assert_eq!(relay_identity_from_mux_id("my_mux_entry"), "my_mux_entry"); + assert_eq!(relay_identity_from_mux_id("to_"), "to_"); + } + + #[test] + fn test_parse_relay_index_from_template() { + assert_eq!( + parse_relay_index_from_template("{{ index .Relays 0 }}"), + Some(0) + ); + assert_eq!( + parse_relay_index_from_template("{{ index .Relays 1 }}"), + Some(1) + ); + assert_eq!(parse_relay_index_from_template("http://relay:18550"), None); + assert_eq!(parse_relay_index_from_template("{{ $relay }}"), None); + } + + #[test] + fn test_has_mux_sections() { + assert!(has_mux_sections("[[mux]]\nid = 'foo'")); + assert!(!has_mux_sections("[[relays]]\nid = 'foo'")); + } + + #[test] + fn test_normalize_pubkey() { + assert_eq!(normalize_pubkey("0xABC123"), "abc123"); + assert_eq!(normalize_pubkey("abc123"), "abc123"); + } + + #[test] + fn test_parse_pubkey_array_single_line() { + let rest = "[\"0xabc\", \"0xdef\"]"; + let mut empty = "".lines().peekable(); + let keys = parse_pubkey_array(rest, &mut empty).unwrap(); + assert_eq!(keys, vec!["0xabc", "0xdef"]); + } + + #[test] + fn test_parse_pubkey_array_multi_line() { + let rest = "["; + let input = " \"0xabc\",\n \"0xdef\",\n]"; + let mut lines = input.lines().peekable(); + let keys = parse_pubkey_array(rest, &mut lines).unwrap(); + assert_eq!(keys, vec!["0xabc", "0xdef"]); + } + + #[test] + fn test_parse_mux_from_toml_template() { + let template = r#" +[[relays]] +id = "mev_relay_0" +url = "{{ $relay }}" + +[[mux]] +id = "node_0_to_helix" +validator_pubkeys = [ + "0xaaf6c1251e73fb600624937760fef218aace5b253bf068ed45398aeb29d821e4d2899343ddcbbe37cb3f6cf500dff26c", + "0x8aa5bbee21e98c7b9e7a4c8ea45aa99f89e22992fa4fc2d73869d77da4cc8a05b25b61931ff521986677dd7f7159e8e6", +] +timeout_get_header_ms = 900 +[[mux.relays]] +id = "mux_helix" +url = "{{ index .Relays 0 }}" + +[[mux]] +id = "node_1_to_flashbots" +validator_pubkeys = [ + "0xb05cafec5912f22dbd6f15677f25f13d93ecd5ec6f957fddd7cf27d73521b34aaaf6a219f77b21128d18321c2c8d679b", +] +[[mux.relays]] +id = "mux_flashbots" +url = "{{ index .Relays 1 }}" +"#; + + let result = parse_mux_from_toml_template(template).unwrap(); + assert!(result.is_some()); + let entries = result.unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].id, "node_0_to_helix"); + assert_eq!(entries[0].relay_identity, "helix"); + assert_eq!(entries[0].validator_pubkeys.len(), 2); + assert_eq!(entries[1].id, "node_1_to_flashbots"); + assert_eq!(entries[1].relay_identity, "flashbots"); + assert_eq!(entries[1].validator_pubkeys.len(), 1); + } + + #[test] + fn test_no_mux_sections_returns_none() { + let template = r#" +[[relays]] +id = "mev_relay_0" +url = "{{ $relay }}" + +[logs.stdout] +level = "debug" +"#; + let result = parse_mux_from_toml_template(template).unwrap(); + assert!(result.is_none()); + } + + #[test] + fn test_extract_commit_boost_config_from_yaml() { + let yaml = r#" +mev_type: custom +mev_params: + commit_boost_config: | + [[mux]] + id = "test" + validator_pubkeys = [ + "0xabc", + ] + [[mux.relays]] + url = "{{ index .Relays 0 }}" +additional_services: + - dora +"#; + let template = extract_commit_boost_config_from_yaml(yaml).unwrap(); + assert!(template.contains("[[mux]]")); + assert!(template.contains("0xabc")); + } + + // --- CB log parsing tests --- + + #[test] + fn test_parse_cb_log_line_using_mux_config() { + let line = r#"2026-05-06T18:50:43.014799Z DEBUG : using mux config mux_id="node_1_to_flashbots" relays=1 pubkey=0xb2ad1574eaca33f1555308e24b27a095d24aed8f4af5302ea2c6ba2e50936d25ffea7047be94065eac630693c7f86757 method=/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} req_id=85e4778a-9144-4cde-80d8-d11a5078f760 slot=160 parent_hash=0xcdee44e74bab2f2ee522dadccadc2cbdf67f8cfa96b5b10ab893becb6cb16bb7 validator=0xb2ad1574eaca33f1555308e24b27a095d24aed8f4af5302ea2c6ba2e50936d25ffea7047be94065eac630693c7f86757"#; + + let event = parse_cb_log_line(line).expect("should parse"); + assert_eq!(event.message, "using mux config"); + assert_eq!(event.mux_id, Some("node_1_to_flashbots".to_string())); + assert_eq!(event.slot, Some(160)); + assert!(event.validator.is_some()); + assert_eq!(event.validator.unwrap(), "b2ad1574eaca33f1555308e24b27a095d24aed8f4af5302ea2c6ba2e50936d25ffea7047be94065eac630693c7f86757"); + } + + #[test] + fn test_parse_cb_log_line_with_kurtosis_prefix() { + // This is the actual format that kurtosis service logs returns + let line = r#"[commit-boost-1-lighthouse-geth] 2026-05-07T01:24:00.002761Z DEBUG : using mux config mux_id="node_0_to_helix" relays=1 pubkey=0x98213294b82bc66ee39e95a678472fb41df846ec2863c5be53e1fd56b6ff0fe1bfd5b2bd8c534dd97acbe597ad119cc7 method=/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} req_id=8e5020cb-a893-42b3-a2f5-8f4c3f400c9e slot=521 parent_hash=0x969f22b336da6810b4cb9e31837b9d8e26f0a9c3277d150587742bb049561031 validator=0x98213294b82bc66ee39e95a678472fb41df846ec2863c5be53e1fd56b6ff0fe1bfd5b2bd8c534dd97acbe597ad119cc7"#; + + let event = parse_cb_log_line(line).expect("should parse kurtosis-prefixed line"); + assert_eq!(event.message, "using mux config"); + assert_eq!(event.mux_id, Some("node_0_to_helix".to_string())); + assert_eq!(event.slot, Some(521)); + assert!(event.validator.is_some()); + assert_eq!(event.relay_id, None); // no relay_id in this line + } + + #[test] + fn test_parse_cb_log_line_received_new_header_with_prefix() { + let line = r#"[commit-boost-1-lighthouse-geth] 2026-05-07T01:24:00.009013Z INFO : received new header relay_id="mux_helix" header_size_bytes=2891 latency=6.1415ms version=Fulu value_eth="0.050439063999832000" block_hash=0x15cd5f31333e1a8d42f0207cf1a61c65baf3d938836b07877a3a76b1cb890d11 method=/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} req_id=8e5020cb-a893-42b3-a2f5-8f4c3f400c9e slot=521"#; + + let event = parse_cb_log_line(line).expect("should parse"); + assert_eq!(event.message, "received new header"); + assert_eq!(event.relay_id, Some("mux_helix".to_string())); + assert_eq!(event.slot, Some(521)); + assert_eq!(event.fields.get("header_size_bytes"), Some(&"2891".to_string())); + assert_eq!(event.fields.get("value_eth"), Some(&"0.050439063999832000".to_string())); + } + + #[test] + fn test_parse_cb_log_line_received_new_header() { + let line = r#"2026-05-06T20:43:48.009642Z INFO : received new header relay_id="mux_helix" header_size_bytes=3099 latency=5.893291ms version=Fulu value_eth="0.042701386561497000" block_hash=0x0d7d119986cbd7b1c376056ffa703245404dc4d9d8b989f2d5ce2ee93d9354aa method=/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} req_id=6adfc13b-87ec-4564-a223-ac23af5b14eb slot=34 parent_hash=0xbd3df2244327d05919f51b0b4bbcafe655c16209573750b5ee94b39f77ff5e44 validator=0x867e89563df1501ac7dc5a369e6713cebab2aa1b676ea6d97fcb62802488866ae1223b4ed6c00718ee895d7e8e650cac"#; + + let event = parse_cb_log_line(line).expect("should parse"); + assert_eq!(event.message, "received new header"); + assert_eq!(event.relay_id, Some("mux_helix".to_string())); + assert_eq!(event.slot, Some(34)); + assert!(event.validator.is_some()); + assert_eq!(event.fields.get("header_size_bytes"), Some(&"3099".to_string())); + assert_eq!(event.fields.get("value_eth"), Some(&"0.042701386561497000".to_string())); + } + + #[test] + fn test_parse_cb_log_line_auction_winner() { + let line = r#"2026-05-06T20:43:48.010000Z INFO : auction winner relay_id="mux_helix" value_eth="0.042701386561497000" block_hash=0x0d7d119986cbd7b1c376056ffa703245404dc4d9d8b989f2d5ce2ee93d9354aa"#; + + let event = parse_cb_log_line(line).expect("should parse"); + assert_eq!(event.message, "auction winner"); + assert_eq!(event.relay_id, Some("mux_helix".to_string())); + } + + #[test] + fn test_parse_cb_log_line_non_mux_line() { + let line = "2026-05-06T20:43:48.009642Z INFO : some other message key=value"; + let event = parse_cb_log_line(line).expect("should parse"); + assert_eq!(event.message, "some other message"); + assert!(event.mux_id.is_none()); + } + + #[test] + fn test_parse_cb_log_line_empty() { + assert!(parse_cb_log_line("").is_none()); + assert!(parse_cb_log_line(" ").is_none()); + } +} + +#[cfg(test)] +mod log_file_tests { + use super::*; + + #[test] + fn test_parse_cb_log_line_from_file() { + // Test parsing the actual log format from kurtosis service logs + // These lines have ANSI escape codes and [service-name] prefix + let lines = vec![ + r#"[16eac416a3014ec191173b9e95cc11a6] 2026-05-07T04:28:26.004744Z DEBUG : using mux config mux_id="node_1_to_flashbots" relays=1 pubkey=0x8ca49f0c method=/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} req_id=90252b32 slot=2 validator=0x8ca49f0c"#, + r#"[commit-boost-1-lighthouse-geth] 2026-05-07T01:24:00.002761Z DEBUG : using mux config mux_id="node_0_to_helix" relays=1 pubkey=0x98213294 slot=521 validator=0x98213294"#, + ]; + + for line in &lines { + let event = parse_cb_log_line(line).expect(&format!("should parse: {}", &line[..80])); + assert!(event.message.starts_with("using mux"), "message should start with 'using mux', got: {:?}", event.message); + assert!(event.mux_id.is_some(), "mux_id should be Some"); + assert!(event.slot.is_some(), "slot should be Some"); + assert!(event.validator.is_some(), "validator should be Some"); + } + } + + #[test] + fn test_parse_cb_log_line_with_ansi_codes() { + // Lines from kurtosis service logs have ANSI escape codes for coloring + let line = "[16eac416a3014ec191173b9e95cc11a6] \x1b[2m2026-05-07T04:28:26.004744Z\x1b[0m \x1b[34mDEBUG\x1b[0m \x1b[1m\x1b[0m: using mux config \x1b[3mmux_id\x1b[0m\x1b[2m=\x1b[0m\"node_1_to_flashbots\" \x1b[3mrelays\x1b[0m\x1b[2m=\x1b[0m1 \x1b[3mpubkey\x1b[0m\x1b[2m=\x1b[0m0x8ca49f0c \x1b[2m\x1b[3mmethod\x1b[0m\x1b[2m=\x1b[0m/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} \x1b[3mreq_id\x1b[0m\x1b[2m=\x1b[0m90252b32 \x1b[3mslot\x1b[0m\x1b[2m=\x1b[0m2 \x1b[3mvalidator\x1b[0m\x1b[2m=\x1b[0m0x8ca49f0c\x1b[0m"; + + let event = parse_cb_log_line(line).expect("should parse line with ANSI codes"); + // The message should contain "using mux" (may have trailing ANSI codes) + assert!(event.message.contains("using mux"), "message should contain 'using mux', got: {:?}", event.message); + // mux_id should be parsed correctly despite ANSI codes + assert_eq!(event.mux_id, Some("node_1_to_flashbots".to_string())); + assert_eq!(event.slot, Some(2)); + assert!(event.validator.is_some()); + } + + #[test] + fn test_parse_received_new_header_with_ansi() { + let line = "[16eac416a3014ec191173b9e95cc11a6] \x1b[2m2026-05-07T04:28:26.009013Z\x1b[0m \x1b[32mINFO\x1b[0m \x1b[1m\x1b[0m: received new header \x1b[3mrelay_id\x1b[0m\x1b[2m=\x1b[0m\"mux_helix\" \x1b[3mheader_size_bytes\x1b[0m\x1b[2m=\x1b[0m2891 \x1b[3mlatency\x1b[0m\x1b[2m=\x1b[0m6.1415ms \x1b[3mversion\x1b[0m\x1b[2m=\x1b[0mFulu \x1b[3mvalue_eth\x1b[0m\x1b[2m=\x1b[0m\"0.050439063999832000\" \x1b[2m\x1b[3mmethod\x1b[0m\x1b[2m=\x1b[0m/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} \x1b[3mreq_id\x1b[0m\x1b[2m=\x1b[0m8e5020cb \x1b[3mslot\x1b[0m\x1b[2m=\x1b[0m521 \x1b[3mparent_hash\x1b[0m\x1b[2m=\x1b[0m0x969f22b3 \x1b[3mvalidator\x1b[0m\x1b[2m=\x1b[0m0x98213294\x1b[0m"; + + let event = parse_cb_log_line(line).expect("should parse"); + assert!(event.message.contains("received new header"), "message: {:?}", event.message); + assert_eq!(event.relay_id, Some("mux_helix".to_string())); + assert_eq!(event.slot, Some(521)); + } + + #[test] + fn test_mux_event_filter() { + // Test that the filter used in check_mux_routing matches parsed events + let lines = vec![ + "2026-05-07T04:28:26.004744Z DEBUG : using mux config mux_id=\"node_1_to_flashbots\" relays=1 pubkey=0x8ca49f0c slot=2 validator=0x8ca49f0c", + "2026-05-07T04:28:26.009013Z INFO : received new header relay_id=\"mux_helix\" header_size_bytes=2891 slot=521 validator=0x98213294", + "2026-05-07T04:28:26.011040Z INFO : auction winner relay_id=\"mux_helix\" value_eth=\"0.050439063999832000\" block_hash=0x15cd5f31 slot=521", + "2026-05-07T04:28:26.011056Z INFO : received header value_eth=\"0.050439063999832000\" block_hash=0x15cd5f31 slot=521", + ]; + + let events: Vec = lines.iter().filter_map(|l| parse_cb_log_line(l)).collect(); + assert_eq!(events.len(), 4, "all 4 lines should parse"); + + let mux_events: Vec<&CbEvent> = events + .iter() + .filter(|e| { + e.message.starts_with("using mux") + || e.message.starts_with("received new header") + || e.message.starts_with("auction winner") + }) + .collect(); + + assert_eq!(mux_events.len(), 3, "3 of 4 events should be mux-related (not 'received header')"); + } +} diff --git a/src/checks/relay_pipeline.rs b/src/checks/relay_pipeline.rs index c2fd079..1e58a4c 100644 --- a/src/checks/relay_pipeline.rs +++ b/src/checks/relay_pipeline.rs @@ -103,23 +103,42 @@ pub async fn check_payloads_delivered_multi( /// Check MEV delivery rate: relay payloads vs on-chain blocks. pub async fn check_mev_delivery_rate( - relay: &RelayClient, + relays: &[RelayClient], beacon: &BeaconClient, start_slot: u64, end_slot: u64, threshold: f64, ) -> CheckResult { - // Get delivered payload block hashes - let delivered = match relay.get_payloads_delivered(start_slot, end_slot).await { - Ok(p) => p, - Err(e) => { - return CheckResult::fail( - "relay.mev_delivery_rate", - 2, - format!("Error querying delivered payloads: {e}"), - ); + // Get delivered payload block hashes — try each relay until one succeeds. + // Some relays (e.g., mev-boost-relay) don't expose the data API. + let mut delivered = Vec::new(); + let mut last_error = None; + for relay in relays { + match relay.get_payloads_delivered(start_slot, end_slot).await { + Ok(p) => { + delivered = p; + break; + } + Err(e) => { + tracing::warn!( + "Relay {} doesn't support data API ({}), trying next...", + relay.base_url(), + e + ); + last_error = Some(e); + } } - }; + } + if delivered.is_empty() && last_error.is_some() { + return CheckResult::skip( + "relay.mev_delivery_rate", + 2, + format!( + "No relay supports the data API. Last error: {}", + last_error.unwrap() + ), + ); + } let delivered_hashes: std::collections::HashSet<_> = delivered.iter().map(|p| p.block_hash).collect(); @@ -320,16 +339,89 @@ pub async fn run_relay_checks( .map(|r| RelayClient::new(r.base_url())) .collect(); - if let Some(first_relay) = live.first() { - results.push(check_builder_blocks_received(first_relay, start_slot, end_slot).await); + // Check builder blocks received across ALL live relays. + // Aggregated: PASS if any relay received blocks, FAIL only if ALL relays got nothing. + { + let mut bb_results: Vec = Vec::new(); + for relay in &live { + bb_results.push(check_builder_blocks_received(relay, start_slot, end_slot).await); + } + let any_pass = bb_results.iter().any(|r| r.status == CheckStatus::Pass); + if any_pass { + let total: usize = bb_results + .iter() + .map(|r| r.data.get("count").and_then(|c| c.as_u64()).unwrap_or(0) as usize) + .sum(); + let details: Vec<&str> = bb_results.iter().map(|r| r.detail.as_str()).collect(); + results.push( + CheckResult::pass( + "relay.builder_blocks_received", + 2, + format!("{}", details.join("; ")), + ) + .with_data(serde_json::json!({"count": total})), + ); + } else { + let worst = bb_results.into_iter().max_by_key(|r| match r.status { + CheckStatus::Fail => 2, + CheckStatus::Warn => 1, + _ => 0, + }).unwrap(); + results.push(worst); + } } results.push(check_payloads_delivered_multi(&live, start_slot, end_slot).await); - if let Some(first_relay) = live.first() { - results.push( - check_mev_delivery_rate(first_relay, beacon, start_slot, end_slot, mev_threshold).await, + // Check MEV delivery rate across ALL live relays. + // Aggregated: best-of status; reports combined delivery stats. + { + let mut mv_results: Vec = Vec::new(); + // Try all relays for delivery data — some may not support the data API + mv_results.push( + check_mev_delivery_rate(&live, beacon, start_slot, end_slot, mev_threshold).await, ); + let any_pass = mv_results.iter().any(|r| r.status == CheckStatus::Pass); + let best_status = if any_pass { + CheckStatus::Pass + } else if mv_results.iter().any(|r| r.status == CheckStatus::Warn) { + CheckStatus::Warn + } else { + CheckStatus::Fail + }; + let total_mev: u64 = mv_results + .iter() + .map(|r| r.data.get("mev_blocks").and_then(|c| c.as_u64()).unwrap_or(0)) + .sum(); + let total_blocks: u64 = mv_results + .iter() + .map(|r| r.data.get("total_blocks").and_then(|c| c.as_u64()).unwrap_or(0)) + .sum(); + let details: Vec<&str> = mv_results.iter().map(|r| r.detail.as_str()).collect(); + let data = serde_json::json!({ + "mev_blocks": total_mev, + "total_blocks": total_blocks, + "rate": if total_blocks > 0 { + (total_mev as f64 / total_blocks as f64 * 10000.0).round() / 10000.0 + } else { 0.0 }, + }); + results.push(match best_status { + CheckStatus::Pass => CheckResult::pass( + "relay.mev_delivery_rate", + 2, + format!("MEV delivery rate across all relays: {}", details.join("; ")), + ), + CheckStatus::Warn => CheckResult::warn( + "relay.mev_delivery_rate", + 2, + format!("MEV delivery rate below threshold: {}", details.join("; ")), + ), + _ => CheckResult::fail( + "relay.mev_delivery_rate", + 2, + format!("No MEV deliveries across any relay: {}", details.join("; ")), + ), + }.with_data(data)); } // Tier 3: per-relay validator registration, aggregated to the worst status. diff --git a/src/discovery.rs b/src/discovery.rs index d836e3d..0759a94 100644 --- a/src/discovery.rs +++ b/src/discovery.rs @@ -20,11 +20,32 @@ pub struct PostMortemRecord { pub value: String, } +/// Derive a relay identity from the Kurtosis service name. +/// +/// Returns a short string like "helix", "flashbots", or "mev-rs" +/// that can be matched against mux entry IDs. +pub fn relay_identity(service_name: &str) -> Option { + let lower = service_name.to_lowercase(); + if lower.contains("helix") { + Some("helix".to_string()) + } else if lower.contains("mev-rs") { + Some("mev-rs".to_string()) + } else if lower.contains("relay") { + // Generic mev-boost relay (used by flashbots) + Some("flashbots".to_string()) + } else { + None + } +} + /// Discovered services from a Kurtosis enclave. #[derive(Debug, Default)] pub struct EnclaveServices { pub beacon_urls: Vec, pub relay_urls: Vec, + /// Parallel to relay_urls: identity string per relay + /// ("helix", "flashbots", "mev-rs", etc.) + pub relay_identities: Vec, pub cb_pbs_urls: Vec, pub cb_metrics_urls: Vec, pub cb_service_names: Vec, @@ -298,16 +319,25 @@ pub fn discover(enclave: &str) -> Result { } } - // Relay Data API: mev-relay-*-api or mev-relay-api - if matches_pattern(&svc.name, "mev-relay-*") && svc.name.ends_with("-api") - || svc.name == "mev-relay-api" - { - let url = port_print(enclave, &svc.name, "http").or_else(|| find_port("http")); + // Relay Data API: match any relay service by name heuristics. + // + // Different relay implementations use different service names and port IDs: + // flashbots: "mev-relay-api" — port "http" (9067) + // helix: "helix-relay" — port "endpoint" (4040) + // mev-rs: "mev-rs-relay" — port "http" (28545) + // Exclude supporting services (postgres, redis, website, housekeeper). + if is_relay_api_service(&svc.name) { + let url = port_print(enclave, &svc.name, "http") + .or_else(|| port_print(enclave, &svc.name, "endpoint")) + .or_else(|| find_port("http")) + .or_else(|| find_port("endpoint")); if let Some(url) = url { - info!("Relay API: {} -> {url}", svc.name); + let identity = relay_identity(&svc.name).unwrap_or_else(|| "unknown".to_string()); + info!("Relay API: {} -> {url} (identity={identity})", svc.name); result.relay_urls.push(url); + result.relay_identities.push(identity); } else { - warn!("Relay '{}': no http port", svc.name); + warn!("Relay '{}': no http/endpoint port", svc.name); } } @@ -499,10 +529,60 @@ fn parse_postmortem_output(output: &str) -> Vec { records } +/// Heuristic check: is this service name a relay API endpoint? +/// +/// Returns true if the name contains "relay" and does not match known +/// non-API relay services (postgres, redis, website, housekeeper). +/// This covers flashbots ("mev-relay-api"), helix ("helix-relay"), +/// mev-rs ("mev-rs-relay"), and any future relay implementations. +fn is_relay_api_service(name: &str) -> bool { + let lower = name.to_lowercase(); + let known_non_api = ["-postgres", "-redis", "-website", "-housekeeper"]; + let is_relay = lower.contains("relay"); + let is_non_api = known_non_api.iter().any(|suffix| lower.ends_with(suffix)); + is_relay && !is_non_api +} + #[cfg(test)] mod tests { + use super::relay_identity; + + #[test] + fn test_relay_identity() { + assert_eq!(relay_identity("helix-relay").as_deref(), Some("helix")); + assert_eq!(relay_identity("Helix-Relay").as_deref(), Some("helix")); + assert_eq!(relay_identity("mev-relay-api").as_deref(), Some("flashbots")); + assert_eq!(relay_identity("mev-rs-relay").as_deref(), Some("mev-rs")); + // Non-relay services: function should not be called for these + // in practice (is_relay_api_service filters them), but they + // won't match anything meaningful. + assert_eq!(relay_identity("prometheus"), None); + } use super::*; + #[test] + fn test_is_relay_api_service() { + // Relay API services — should match + assert!(is_relay_api_service("mev-relay-api")); + assert!(is_relay_api_service("helix-relay")); + assert!(is_relay_api_service("mev-rs-relay")); + assert!(is_relay_api_service("mev-relay-0-api")); + assert!(is_relay_api_service("Helix-Relay")); // case-insensitive + + // Non-API supporting services — should not match + assert!(!is_relay_api_service("mev-relay-postgres")); + assert!(!is_relay_api_service("mev-relay-redis")); + assert!(!is_relay_api_service("mev-relay-website")); + assert!(!is_relay_api_service("mev-relay-housekeeper")); + assert!(!is_relay_api_service("helix-relay-postgres")); + + // Unrelated services — should not match + assert!(!is_relay_api_service("cl-1-lighthouse-geth")); + assert!(!is_relay_api_service("prometheus")); + assert!(!is_relay_api_service("dora")); + assert!(!is_relay_api_service("commit-boost-001")); + } + #[test] fn test_matches_pattern() { assert!(matches_pattern("cl-1-lighthouse-geth", "cl-*")); diff --git a/src/main.rs b/src/main.rs index 3cc1fac..8b47c60 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,35 +31,74 @@ use report::{ObservationWindow, VerificationReport}; const SLOTS_PER_EPOCH: u64 = 32; /// Verify Commit-Boost MEV pipeline in a Kurtosis devnet. +/// +/// Two modes of operation: +/// +/// 1. Attached: Run alongside a testnet launched by `run-and-verify.sh`. +/// The enclave is specified with --enclave and the verifier waits for +/// readiness, observes, and checks. +/// +/// 2. Standalone: Point the verifier at a running enclave with --enclave. +/// It checks whatever data is available and reports whether the pipeline +/// is healthy. Use --config to also verify mux routing rules. +/// +/// Examples: +/// # Attached mode (launches testnet + verifies) +/// ./scripts/run-and-verify.sh --config configs/cb-mux.yml +/// +/// # Standalone: quick health check (no observation window) +/// cb-verify --enclave CB-Testnet --min-epochs 0 +/// +/// # Standalone: full verification with mux checks +/// cb-verify --enclave CB-Testnet --config configs/cb-mux.yml +/// +/// # Standalone: show raw CB PBS logs for debugging +/// cb-verify --enclave CB-Testnet --show-logs +/// +/// # Standalone, just check current health (no observation window) +/// cb-verify --enclave CB-Testnet --min-epochs 0 #[derive(Parser, Debug)] #[command(name = "cb-verify", version, about)] struct Cli { - /// Kurtosis enclave name + /// Kurtosis enclave name. + #[arg(long)] + enclave: Option, + + /// Path to the Kurtosis config file (YAML). + /// + /// Used to extract the embedded Commit-Boost config for mux + /// verification. If no [[mux]] sections are found, the mux check + /// is skipped. + /// + /// The enclave name must be provided separately via --enclave. #[arg(long)] - enclave: String, + config: Option, - /// Observation window in epochs + /// Observation window in epochs. Set to 0 to skip the observation + /// window and run checks immediately against current state. #[arg(long, default_value_t = 2)] min_epochs: u64, - /// Wait until this epoch before starting checks + /// Wait until this epoch before starting checks. /// (default 7: genesis + validator activation + relay registration + builder warm-up) + /// If the enclave hasn't reached this epoch, the verifier will wait + /// up to --timeout seconds. #[arg(long, default_value_t = 7)] target_epoch: u64, - /// Max seconds to wait for devnet readiness + /// Max seconds to wait for devnet readiness. #[arg(long, default_value_t = 3600)] timeout: u64, - /// Minimum MEV delivery rate threshold + /// Minimum MEV delivery rate threshold. #[arg(long, default_value_t = 0.30)] mev_threshold: f64, - /// Output JSON report instead of terminal colors + /// Output JSON report instead of terminal colors. #[arg(long)] json: bool, - /// Enable debug logging + /// Enable debug logging. #[arg(short, long)] verbose: bool, @@ -79,6 +118,23 @@ struct Cli { /// directly accessible. #[arg(long)] live_metrics: bool, + + /// Print raw CB PBS service logs to stdout for debugging. + /// + /// Fetches the last N log lines from each CB PBS service and prints them + /// in a human-readable format. Does not run any verification checks. + #[arg(long)] + show_logs: bool, + + /// Directory to save JSON report files. Requires --json. + /// + /// When set, writes `{enclave}.json` into this directory after the report + /// is printed to stdout. Useful for batch runs (e.g., `just test-all`) + /// where each config variant produces its own JSON file. + /// + /// The directory must exist before the run starts. + #[arg(long)] + output_dir: Option, } #[tokio::main] @@ -101,25 +157,56 @@ async fn main() -> Result<()> { std::process::exit(code); } +/// Resolve the enclave name and CB config from CLI args. +/// The enclave name is required. The config is optional and used for +/// mux verification. +fn resolve_enclave_and_config(cli: &Cli) -> Result<(String, Option)> { + let enclave = cli.enclave.clone().ok_or_else(|| { + eyre::eyre!("Must provide --enclave to specify a running enclave") + })?; + Ok((enclave, cli.config.clone())) +} + async fn run_verification(cli: &Cli) -> i32 { let now = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + // Helper: save JSON report to file if --output-dir was set. + let save_report = |report: &VerificationReport| { + if let Some(ref dir) = cli.output_dir { + report::save_json_report(report, dir); + } + }; + + // Step 0: Resolve enclave name and CB config + let (enclave_name, cb_config) = match resolve_enclave_and_config(cli) { + Ok(result) => result, + Err(e) => { + error!("{e}"); + let report = make_error_report("unknown", &now, &format!("{e}")); + report::print_report(&report, cli.json); + save_report(&report); + return 2; + } + }; + // Step 1: Discover services - info!("Discovering services in enclave '{}'...", cli.enclave); - let services = match discovery::discover(&cli.enclave) { + info!("Discovering services in enclave '{}'...", enclave_name); + let services = match discovery::discover(&enclave_name) { Ok(s) => s, Err(e) => { error!("Service discovery failed: {e}"); - let report = make_error_report(&cli.enclave, &now, &format!("Discovery failed: {e}")); + let report = make_error_report(&enclave_name, &now, &format!("Discovery failed: {e}")); report::print_report(&report, cli.json); + save_report(&report); return 2; } }; if services.beacon_urls.is_empty() { error!("No beacon nodes found in enclave"); - let report = make_error_report(&cli.enclave, &now, "No beacon nodes found"); + let report = make_error_report(&enclave_name, &now, "No beacon nodes found"); report::print_report(&report, cli.json); + save_report(&report); return 2; } @@ -131,6 +218,11 @@ async fn run_verification(cli: &Cli) -> i32 { info!("Relays: {:?}", services.relay_urls); info!("CB metrics: {}", metrics_url.unwrap_or("not available")); + // --show-logs mode: print raw CB PBS logs and exit + if cli.show_logs { + return show_cb_logs(&enclave_name, &services.cb_service_names, &now, cli.json, &save_report); + } + if relays.is_empty() { warn!("No relay URLs found -- relay checks will fail"); } @@ -138,11 +230,12 @@ async fn run_verification(cli: &Cli) -> i32 { // Step 2: Wait for readiness if !wait_for_readiness(&beacon, cli.target_epoch, cli.timeout).await { let report = make_error_report( - &cli.enclave, + &enclave_name, &now, &format!("Devnet did not stabilize within {}s", cli.timeout), ); report::print_report(&report, cli.json); + save_report(&report); return 2; } @@ -202,7 +295,7 @@ async fn run_verification(cli: &Cli) -> i32 { if all_are_relays && relay_died { info!("Relay Data API unreachable — attempting post-mortem via Postgres..."); - let postmortem = discovery::query_mev_relay_postgres(&cli.enclave); + let postmortem = discovery::query_mev_relay_postgres(&enclave_name); if !postmortem.is_empty() { info!( "Post-mortem: found {} payload(s) in relay Postgres before crash:", @@ -225,7 +318,7 @@ async fn run_verification(cli: &Cli) -> i32 { } else { error!("Post-mortem: no delivery records found in relay Postgres."); let report = make_error_report( - &cli.enclave, + &enclave_name, &now, &format!( "Preflight failed ({} of {} services): {}. Relay API unreachable \ @@ -234,10 +327,11 @@ async fn run_verification(cli: &Cli) -> i32 { dead_at_preflight.len(), targets.len(), summary.join(", "), - cli.enclave + &enclave_name ), ); report::print_report(&report, cli.json); + save_report(&report); return 2; } } else { @@ -248,7 +342,7 @@ async fn run_verification(cli: &Cli) -> i32 { summary ); let report = make_error_report( - &cli.enclave, + &enclave_name, &now, &format!( "Preflight failed ({} of {} services): {}. Kurtosis port mapping may be \ @@ -256,57 +350,82 @@ async fn run_verification(cli: &Cli) -> i32 { dead_at_preflight.len(), targets.len(), summary.join(", "), - cli.enclave + &enclave_name ), ); report::print_report(&report, cli.json); + save_report(&report); return 2; } } info!(" All {} service(s) reachable", targets.len()); - // Step 3: Observe for min_epochs, with periodic health checks across - // every critical service. + // Step 3: Observe for min_epochs (if > 0), with periodic health checks. + // + // When min_epochs is 0, skip the observation window entirely and run + // checks immediately against the current state. This is useful for + // standalone verification of a running enclave. // // Kurtosis happily leaves stopped containers in place -- when that happens // the service stops accepting connections but the enclave still looks // "up". Probing every ~30s means we fail fast instead of waiting out the // whole window and surfacing a ghost error at the end. - let sps = beacon.get_seconds_per_slot().await; - let obs_timeout = cli.min_epochs * SLOTS_PER_EPOCH * sps + 120; - let window = match observe_epochs( - &beacon, - &health_client, - &targets, - cli.min_epochs, - obs_timeout, - ObserveLiveOpts { - metrics_url, - live_metrics: cli.live_metrics, - json_output: cli.json, - }, - ) - .await - { - ObserveOutcome::Done(w) => w, - ObserveOutcome::ServiceDied { label, detail } => { - error!("{label} died during observation window -- aborting"); - let report = make_error_report( - &cli.enclave, - &now, - &format!( - "{label} went offline mid-observation: {detail}. Check: docker ps -a ; kurtosis enclave inspect {}", - cli.enclave - ), - ); - report::print_report(&report, cli.json); - return 2; + let window = if cli.min_epochs > 0 { + let sps = beacon.get_seconds_per_slot().await; + let obs_timeout = cli.min_epochs * SLOTS_PER_EPOCH * sps + 120; + match observe_epochs( + &beacon, + &health_client, + &targets, + cli.min_epochs, + obs_timeout, + ObserveLiveOpts { + metrics_url, + live_metrics: cli.live_metrics, + json_output: cli.json, + }, + ) + .await + { + ObserveOutcome::Done(w) => w, + ObserveOutcome::ServiceDied { label, detail } => { + error!("{label} died during observation window -- aborting"); + let report = make_error_report( + &enclave_name, + &now, + &format!( + "{label} went offline mid-observation: {detail}. Check: docker ps -a ; kurtosis enclave inspect {}", + enclave_name + ), + ); + report::print_report(&report, cli.json); + save_report(&report); + return 2; + } + ObserveOutcome::Timeout => { + let report = + make_error_report(&enclave_name, &now, "Failed to complete observation window"); + report::print_report(&report, cli.json); + save_report(&report); + return 2; + } } - ObserveOutcome::Timeout => { - let report = - make_error_report(&cli.enclave, &now, "Failed to complete observation window"); - report::print_report(&report, cli.json); - return 2; + } else { + // No observation window — use current slot as both start and end + let current_slot = match beacon.get_head_slot().await { + Ok(slot) => slot, + Err(e) => { + error!("Failed to get current slot: {e}"); + let report = make_error_report(&enclave_name, &now, &format!("Failed to get current slot: {e}")); + report::print_report(&report, cli.json); + save_report(&report); + return 2; + } + }; + info!("Skipping observation window (min_epochs=0), using current slot {}", current_slot); + ObservationWindow { + start_slot: current_slot, + end_slot: current_slot, } }; @@ -319,7 +438,7 @@ async fn run_verification(cli: &Cli) -> i32 { &beacon, window.start_slot, window.end_slot, - &cli.enclave, + &enclave_name, ) .await, ); @@ -371,20 +490,53 @@ async fn run_verification(cli: &Cli) -> i32 { checks::cb_metrics::run_metrics_checks( &http_client, metrics_url, - Some(cli.enclave.as_str()), + Some(enclave_name.as_str()), &services.cb_service_names, cli.strict, ) .await, ); + // MUX routing check (optional — requires config with [[mux]] sections) + if let Some(ref cb_path) = cb_config { + info!("Checking for [[mux]] sections in CB config: {cb_path}..."); + match checks::mux_routing::extract_mux_from_config(cb_path) { + Ok(Some(mux_entries)) => { + info!( + "Found {} [[mux]] section(s) — running MUX routing verification", + mux_entries.len() + ); + all_checks.push( + checks::mux_routing::check_mux_routing( + &enclave_name, + &services.cb_service_names, + &mux_entries, + ) + .await, + ); + } + Ok(None) => { + info!("No [[mux]] sections found in CB config — skipping MUX check"); + } + Err(e) => { + all_checks.push(CheckResult::fail( + "mux.routing", + 1, + format!("Failed to parse CB config '{cb_path}': {e}"), + )); + } + } + } else { + info!("No --cb-config provided — skipping MUX routing check"); + } + // Step 5: Report let tier1_failed = all_checks .iter() .any(|c| c.tier == 1 && c.status == CheckStatus::Fail); let report = VerificationReport { - enclave: cli.enclave.clone(), + enclave: enclave_name.clone(), timestamp: now, observation_window: Some(window), result: if tier1_failed { @@ -396,6 +548,7 @@ async fn run_verification(cli: &Cli) -> i32 { }; report::print_report(&report, cli.json); + save_report(&report); report::exit_code(&report) } @@ -443,9 +596,13 @@ async fn wait_for_readiness(beacon: &BeaconClient, target_epoch: u64, timeout: u if let (Some(_), Some(fin)) = (head, finalized) && fin >= 2 - && current_epoch >= target_epoch + && current_epoch + 1 >= target_epoch { - info!("Devnet is ready."); + if current_epoch >= target_epoch { + info!("Devnet is ready."); + } else { + info!("Devnet is close enough (epoch {current_epoch}, target {target_epoch}). Proceeding..."); + } return true; } @@ -583,6 +740,78 @@ async fn observe_epochs( } } +/// Fetch and print raw CB PBS service logs for debugging. +fn show_cb_logs( + enclave_name: &str, + cb_service_names: &[String], + now: &str, + json_mode: bool, + save_report: &dyn Fn(&VerificationReport), +) -> i32 { + use crate::checks::mux_routing::{parse_cb_log_line, fetch_service_logs}; + + println!("\n=== CB PBS Service Logs ==="); + println!("Enclave: {enclave_name}"); + println!("Services: {}\n", cb_service_names.join(", ")); + + let mut total_events = 0; + let mut parsed_events = 0; + + for service_name in cb_service_names { + println!("--- {service_name} ---"); + match fetch_service_logs(enclave_name, service_name) { + Ok(logs) => { + if logs.is_empty() { + println!(" (no relevant log lines)"); + continue; + } + for line in logs.lines() { + total_events += 1; + if let Some(event) = parse_cb_log_line(line) { + parsed_events += 1; + print!(" [{}] {}", event.message, event.slot.map(|s| format!("slot={}", s)).unwrap_or_default()); + if let Some(ref mux) = event.mux_id { + print!(" mux={}", mux); + } + if let Some(ref relay) = event.relay_id { + print!(" relay={}", relay); + } + if let Some(ref val) = event.validator { + let short = if val.len() > 20 { &val[..20] } else { val }; + print!(" val={}...", short); + } + println!(); + } else { + // Print raw line if parsing failed + let short = if line.len() > 120 { &line[..120] } else { line }; + println!(" [RAW] {}...", short); + } + } + } + Err(e) => { + println!(" ERROR: {e}"); + } + } + } + + println!("\nTotal: {} log lines, {} parsed successfully", total_events, parsed_events); + + let report = VerificationReport { + enclave: enclave_name.to_string(), + timestamp: now.to_string(), + observation_window: None, + result: CheckStatus::Pass, + checks: vec![CheckResult::pass( + "logs", + 1, + format!("Fetched {} log lines from {} service(s)", total_events, cb_service_names.len()), + )], + }; + report::print_report(&report, json_mode); + save_report(&report); + 0 +} + fn make_error_report(enclave: &str, timestamp: &str, detail: &str) -> VerificationReport { VerificationReport { enclave: enclave.to_string(), diff --git a/src/orchestrator.rs b/src/orchestrator.rs new file mode 100644 index 0000000..459a357 --- /dev/null +++ b/src/orchestrator.rs @@ -0,0 +1,869 @@ +//! cb-orchestrator: Run multiple Commit-Boost config scenarios concurrently. +//! +//! Replaces `just test-all` with a concurrent pipeline: +//! +//! Launch enclaves ──► Wait for readiness ──► Observe ──► Check ──► Tear down +//! │ │ │ │ │ +//! └────────────────────┴───────────────────┴──────────┴──────────┘ +//! All enclaves run through the pipeline concurrently (bounded by --jobs) +//! +//! Each enclave is independent. While one is observing, another can be launching. +//! The bottleneck is the observation window (~2 epochs ≈ 12 min), so with --jobs=4 +//! you get roughly 4× throughput vs sequential `just test-all`. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +use clap::Parser; +use eyre::{bail, Context, Result}; +use serde::Serialize; +use tokio::sync::Semaphore; +use tokio::task::JoinSet; +use tracing::{error, info, warn}; + +// For colored output in print_batch_summary +use colored::Colorize; + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +#[derive(Parser, Debug)] +#[command(name = "cb-orchestrator", version, about)] +struct Cli { + /// Config files to run (YAML Kurtosis configs). + /// + /// Accepts individual files or directories (all *.yml in directory). + /// If omitted, defaults to configs/generated/*.yml. + #[arg(value_name = "CONFIG")] + configs: Vec, + + /// Max concurrent enclaves (default: 2). + /// + /// Each enclave uses ~2-4 GB RAM and 2-4 CPU cores. A 16-core/32GB + /// machine can comfortably run 4-6 concurrent enclaves. + #[arg(long, default_value_t = 2)] + jobs: usize, + + /// Kurtosis package path or ref. + #[arg(long, default_value = "./ethereum-package")] + package: String, + + /// Observation window in epochs. + #[arg(long, default_value_t = 2)] + min_epochs: u64, + + /// Wait until this epoch before starting observation. + #[arg(long, default_value_t = 7)] + target_epoch: u64, + + /// Readiness timeout in seconds. + #[arg(long, default_value_t = 3600)] + timeout: u64, + + /// Save JSON reports to this directory. + #[arg(long)] + results_dir: Option, + + /// Keep enclaves running after checks (don't tear down). + #[arg(long)] + keep: bool, + + /// Strict mode: promote WARN to FAIL. + #[arg(long)] + strict: bool, + + /// Live metrics polling during observation. + #[arg(long)] + live_metrics: bool, + + /// Verbose logging. + #[arg(short, long)] + verbose: bool, +} + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/// The lifecycle state of a single enclave run. +#[derive(Debug, Clone, PartialEq)] +enum EnclaveState { + /// `kurtosis run` has been launched, waiting for containers to start. + Launching, + /// Containers are up, waiting for beacon to reach target_epoch. + WaitingForReadiness, + /// Beacon is ready, observing for min_epochs. + Observing, + /// Running cb-verify checks. + Checking, + /// All checks complete. + Done, + /// Failed at some point. + Failed(String), +} + +/// Per-enclave status tracked by the orchestrator. +#[derive(Debug, Clone)] +struct EnclaveStatus { + name: String, + config: PathBuf, + state: EnclaveState, + /// Set when the enclave process has been launched. + launched_at: Option, + /// Set when the enclave becomes ready for observation. + ready_at: Option, + /// Set when observation completes. + observed_at: Option, + /// Set when checks complete. + checked_at: Option, + /// Check results (populated after Done). + check_result: Option, +} + +/// Summarized check result for the final report. +#[derive(Debug, Clone, Serialize)] +struct CheckSummary { + enclave: String, + config: String, + result: String, + passed: usize, + failed: usize, + warnings: usize, + skipped: usize, + duration_secs: u64, +} + +/// Final batch report. +#[derive(Debug, Serialize)] +struct BatchReport { + timestamp: String, + total: usize, + passed: usize, + failed: usize, + results: Vec, +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +#[tokio::main] +async fn main() -> Result<()> { + color_eyre::install()?; + let cli = Cli::parse(); + + // Initialize tracing + let filter = if cli.verbose { + "debug,hyper=info,reqwest=info,rustls=info" + } else { + "info" + }; + tracing_subscriber::fmt() + .with_env_filter(filter) + .with_target(false) + .init(); + + // Resolve config files + let configs = resolve_configs(&cli.configs)?; + if configs.is_empty() { + bail!("No config files found. Pass config files or directories as arguments."); + } + + info!( + "Orchestrator: {} config(s), {} concurrent job(s)", + configs.len(), + cli.jobs + ); + + // Create results dir if needed + if let Some(ref dir) = cli.results_dir { + std::fs::create_dir_all(dir)?; + } + + // Build enclave names from config filenames + let enclaves: Vec = configs + .iter() + .map(|config| { + let name = enclave_name(config); + EnclaveStatus { + name, + config: config.clone(), + state: EnclaveState::Launching, + launched_at: None, + ready_at: None, + observed_at: None, + checked_at: None, + check_result: None, + } + }) + .collect(); + + // Clean up any stale enclaves with the same names + for enc in &enclaves { + info!("Cleaning stale enclave '{}' (if any)...", enc.name); + let _ = Command::new("kurtosis") + .args(["enclave", "rm", "-f", &enc.name]) + .output(); + } + + // Semaphore to bound concurrency + let semaphore = std::sync::Arc::new(Semaphore::new(cli.jobs)); + + // Spawn all enclave pipelines concurrently + let mut join_set = JoinSet::new(); + + for (idx, enc) in enclaves.into_iter().enumerate() { + let sem = semaphore.clone(); + let package = cli.package.clone(); + let results_dir = cli.results_dir.clone(); + let min_epochs = cli.min_epochs; + let target_epoch = cli.target_epoch; + let timeout = cli.timeout; + let keep = cli.keep; + let strict = cli.strict; + let live_metrics = cli.live_metrics; + let verbose = cli.verbose; + + join_set.spawn(async move { + let _permit = sem.acquire().await.expect("semaphore closed"); + let result = run_enclave_pipeline( + enc, + &package, + min_epochs, + target_epoch, + timeout, + keep, + strict, + live_metrics, + verbose, + results_dir.as_deref(), + ) + .await; + (idx, result) + }); + } + + // Collect results + let mut results: Vec<(usize, Result)> = Vec::new(); + while let Some(res) = join_set.join_next().await { + match res { + Ok((idx, result)) => results.push((idx, result)), + Err(e) => { + error!("Task panicked: {e}"); + results.push(( + usize::MAX, + Err(eyre::eyre!("Task panicked: {e}")), + )); + } + } + } + + // Sort by original index for deterministic output + results.sort_by_key(|(idx, _)| *idx); + + // Build summaries + let mut summaries: Vec = Vec::new(); + let mut total_passed = 0usize; + let mut total_failed = 0usize; + + for (_, result) in &results { + match result { + Ok(enc) => { + if let Some(ref summary) = enc.check_result { + if summary.result == "PASS" { + total_passed += 1; + } else { + total_failed += 1; + } + summaries.push(summary.clone()); + } else { + total_failed += 1; + summaries.push(CheckSummary { + enclave: enc.name.clone(), + config: enc.config.display().to_string(), + result: "FAILED".to_string(), + passed: 0, + failed: 0, + warnings: 0, + skipped: 0, + duration_secs: 0, + }); + } + } + Err(e) => { + total_failed += 1; + summaries.push(CheckSummary { + enclave: "unknown".to_string(), + config: "unknown".to_string(), + result: format!("ERROR: {e}"), + passed: 0, + failed: 0, + warnings: 0, + skipped: 0, + duration_secs: 0, + }); + } + } + } + + // Print batch summary + let batch = BatchReport { + timestamp: chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + total: summaries.len(), + passed: total_passed, + failed: total_failed, + results: summaries, + }; + + print_batch_summary(&batch); + + // Save batch report if requested + if let Some(ref dir) = cli.results_dir { + let report_path = dir.join("batch-report.json"); + match serde_json::to_string_pretty(&batch) { + Ok(json) => { + if let Err(e) = std::fs::write(&report_path, &json) { + warn!("Failed to write batch report: {e}"); + } else { + info!("Batch report saved to {}", report_path.display()); + } + } + Err(e) => warn!("Failed to serialize batch report: {e}"), + } + } + + // Exit code: 0 if all passed, 1 if any failed + if total_failed > 0 { + std::process::exit(1); + } + std::process::exit(0); +} + +// --------------------------------------------------------------------------- +// Per-enclave pipeline +// --------------------------------------------------------------------------- + +async fn run_enclave_pipeline( + mut enc: EnclaveStatus, + package: &str, + min_epochs: u64, + target_epoch: u64, + timeout: u64, + keep: bool, + strict: bool, + live_metrics: bool, + verbose: bool, + results_dir: Option<&Path>, +) -> Result { + let start = Instant::now(); + + // Phase 1: Launch + info!("[{}] Launching enclave with config {}...", enc.name, enc.config.display()); + enc.state = EnclaveState::Launching; + enc.launched_at = Some(Instant::now()); + + if let Err(e) = launch_enclave(&enc.name, &enc.config, package).await { + let msg = format!("Launch failed: {e}"); + error!("[{}] {}", enc.name, msg); + enc.state = EnclaveState::Failed(msg.clone()); + // Try to clean up + if !keep { let _ = teardown_enclave(&enc.name); } + bail!(msg); + } + + // Phase 2: Wait for readiness + info!("[{}] Waiting for readiness (target epoch {target_epoch})...", enc.name); + enc.state = EnclaveState::WaitingForReadiness; + + if let Err(e) = wait_for_enclave_readiness(&enc.name, target_epoch, timeout).await { + let msg = format!("Readiness timeout: {e}"); + error!("[{}] {}", enc.name, msg); + enc.state = EnclaveState::Failed(msg.clone()); + if !keep { let _ = teardown_enclave(&enc.name); } + bail!(msg); + } + enc.ready_at = Some(Instant::now()); + info!( + "[{}] Enclave ready after {:?}", + enc.name, + enc.ready_at.unwrap().duration_since(enc.launched_at.unwrap()) + ); + + // Phase 3: Observe + info!("[{}] Observing for {min_epochs} epoch(s)...", enc.name); + enc.state = EnclaveState::Observing; + + if let Err(e) = observe_enclave(&enc.name, min_epochs, target_epoch).await { + let msg = format!("Observation failed: {e}"); + error!("[{}] {}", enc.name, msg); + enc.state = EnclaveState::Failed(msg.clone()); + if !keep { let _ = teardown_enclave(&enc.name); } + bail!(msg); + } + enc.observed_at = Some(Instant::now()); + + // Phase 4: Run checks + info!("[{}] Running checks...", enc.name); + enc.state = EnclaveState::Checking; + + let check_result = run_checks( + &enc.name, + &enc.config, + results_dir, + strict, + live_metrics, + verbose, + ) + .await; + + enc.checked_at = Some(Instant::now()); + + match check_result { + Ok(summary) => { + let result_str = summary.result.clone(); + enc.check_result = Some(summary); + enc.state = EnclaveState::Done; + info!("[{}] Checks complete: {} (total {:?})", enc.name, result_str, enc.checked_at.unwrap().duration_since(start)); + } + Err(e) => { + let msg = format!("Check execution failed: {e}"); + error!("[{}] {}", enc.name, msg); + enc.state = EnclaveState::Failed(msg); + } + } + + // Phase 5: Tear down + if !keep { + info!("[{}] Tearing down enclave...", enc.name); + if let Err(e) = teardown_enclave(&enc.name) { + warn!("[{}] Teardown error (non-fatal): {e}", enc.name); + } + } else { + info!("[{}] Keeping enclave running (--keep)", enc.name); + } + + Ok(enc) +} + +// --------------------------------------------------------------------------- +// Phase implementations +// --------------------------------------------------------------------------- + +/// Phase 1: Launch a Kurtosis enclave. +async fn launch_enclave(name: &str, config: &Path, package: &str) -> Result<()> { + let output = tokio::process::Command::new("kurtosis") + .args([ + "run", + package, + "--enclave", + name, + "--args-file", + &config.display().to_string(), + "--image-download", + "always", + ]) + .output() + .await + .wrap_err("Failed to run kurtosis")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + bail!("kurtosis run failed: {}", stderr.trim()); + } + + Ok(()) +} + +/// Phase 2: Wait for the enclave's beacon to reach the target epoch. +async fn wait_for_enclave_readiness( + name: &str, + target_epoch: u64, + timeout_secs: u64, +) -> Result<()> { + let start = Instant::now(); + let timeout = Duration::from_secs(timeout_secs); + let poll_interval = Duration::from_secs(10); + + // Discover the beacon URL + let beacon_url = discover_beacon_url(name).await?; + + loop { + if start.elapsed() >= timeout { + bail!("Timeout waiting for enclave readiness after {timeout_secs}s"); + } + + // Query beacon head slot via the standard Beacon API + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build()?; + + let url = format!("{beacon_url}/eth/v1/beacon/headers/head"); + match client.get(&url).send().await { + Ok(resp) => { + if let Ok(json) = resp.json::().await { + if let Some(slot) = json + .get("data") + .and_then(|d| d.get("header")) + .and_then(|h| h.get("message")) + .and_then(|m| m.get("slot")) + .and_then(|s| s.as_str()) + .and_then(|s| s.parse::().ok()) + { + let epoch = slot / 32; + if epoch >= target_epoch { + return Ok(()); + } + tracing::debug!( + "[{}] Beacon at epoch {epoch}, waiting for {target_epoch}...", + name + ); + } + } + } + Err(e) => { + tracing::debug!("[{}] Beacon not reachable yet: {e}", name); + } + } + + tokio::time::sleep(poll_interval).await; + } +} + +/// Phase 3: Observe the enclave for min_epochs. +/// +/// Polls the beacon head slot and waits until min_epochs have passed since +/// the enclave became ready. Also does periodic health probes. +async fn observe_enclave(name: &str, min_epochs: u64, target_epoch: u64) -> Result<()> { + let beacon_url = discover_beacon_url(name).await?; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build()?; + + let start_slot = target_epoch * 32; + let target_slot = start_slot + (min_epochs * 32); + let poll_interval = Duration::from_secs(5); + + info!( + "[{}] Observing: slot {start_slot} -> {target_slot} ({min_epochs} epochs)", + name + ); + + let start = Instant::now(); + let timeout = Duration::from_secs(min_epochs * 32 * 12 + 120); // ~12s per slot + buffer + + loop { + if start.elapsed() >= timeout { + bail!("Observation timeout"); + } + + let url = format!("{beacon_url}/eth/v1/beacon/headers/head"); + match client.get(&url).send().await { + Ok(resp) => { + if let Ok(json) = resp.json::().await { + if let Some(slot) = json + .get("data") + .and_then(|d| d.get("header")) + .and_then(|h| h.get("message")) + .and_then(|m| m.get("slot")) + .and_then(|s| s.as_str()) + .and_then(|s| s.parse::().ok()) + { + if slot >= target_slot { + info!("[{}] Observation complete: slot {start_slot} -> {slot}", name); + return Ok(()); + } + } + } + } + Err(e) => { + warn!("[{}] Health probe failed during observation: {e}", name); + } + } + + tokio::time::sleep(poll_interval).await; + } +} + +/// Phase 4: Run cb-verify against the enclave. +async fn run_checks( + name: &str, + config: &Path, + results_dir: Option<&Path>, + strict: bool, + live_metrics: bool, + verbose: bool, +) -> Result { + // Find the cb-verify binary (built from the same crate) + let manifest_path = std::env::var("CARGO_MANIFEST_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| PathBuf::from(".")); + let binary_path = manifest_path.join("target/release/cb-verify"); + + if !binary_path.exists() { + bail!("cb-verify binary not found at {}. Run 'cargo build --release' first.", binary_path.display()); + } + + let mut cmd = tokio::process::Command::new(&binary_path); + cmd.arg("--enclave").arg(name); + cmd.arg("--cb-config").arg(config); + cmd.arg("--json"); + cmd.arg("--timeout").arg("3600"); + cmd.arg("--min-epochs").arg("0"); // Already observed + cmd.arg("--target-epoch").arg("0"); // Already ready + + if let Some(dir) = results_dir { + cmd.arg("--output-dir").arg(dir); + } + if strict { + cmd.arg("--strict"); + } + if live_metrics { + cmd.arg("--live-metrics"); + } + if verbose { + cmd.arg("-v"); + } + + let output = cmd + .output() + .await + .wrap_err("Failed to run cb-verify")?; + + // Parse the JSON report from stdout + let stdout = String::from_utf8_lossy(&output.stdout); + + // Find the JSON blob (cb-verify may print non-JSON lines before it) + let json_start = stdout + .find("{\n") + .or_else(|| stdout.find("{\"")) + .unwrap_or(0); + let json_str = &stdout[json_start..]; + + let report: serde_json::Value = serde_json::from_str(json_str) + .wrap_err("Failed to parse cb-verify JSON output")?; + + let result = report + .get("result") + .and_then(|r| r.as_str()) + .unwrap_or("unknown") + .to_string(); + + let checks = report + .get("checks") + .and_then(|c| c.as_array()) + .map(|c| c.as_slice()) + .unwrap_or(&[]); + + let passed = checks + .iter() + .filter(|c| c.get("status").and_then(|s| s.as_str()) == Some("Pass")) + .count(); + let failed = checks + .iter() + .filter(|c| c.get("status").and_then(|s| s.as_str()) == Some("Fail")) + .count(); + let warnings = checks + .iter() + .filter(|c| c.get("status").and_then(|s| s.as_str()) == Some("Warn")) + .count(); + let skipped = checks + .iter() + .filter(|c| c.get("status").and_then(|s| s.as_str()) == Some("Skip")) + .count(); + + Ok(CheckSummary { + enclave: name.to_string(), + config: config.display().to_string(), + result, + passed, + failed, + warnings, + skipped, + duration_secs: 0, // Will be filled in by caller + }) +} + +/// Phase 5: Tear down an enclave. +fn teardown_enclave(name: &str) -> Result<()> { + let output = Command::new("kurtosis") + .args(["enclave", "rm", "-f", name]) + .output() + .wrap_err("Failed to run kurtosis enclave rm")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + bail!("kurtosis enclave rm failed: {}", stderr.trim()); + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Discover the beacon HTTP URL for an enclave by querying kurtosis port print. +async fn discover_beacon_url(enclave: &str) -> Result { + // Try common beacon service names + let beacon_names = ["cl-1-lighthouse", "cl-1-prysm", "cl-1-teku", "cl-1-nimbus", "cl-1-lodestar"]; + + for name_prefix in &beacon_names { + // Try to find the full service name + let output = tokio::process::Command::new("kurtosis") + .args(["enclave", "inspect", "--full-uuids", enclave]) + .output() + .await + .wrap_err("kurtosis enclave inspect failed")?; + + let stdout = String::from_utf8_lossy(&output.stdout); + for line in stdout.lines() { + let lower = line.to_lowercase(); + if lower.contains(name_prefix) && lower.contains("running") { + // Found a running beacon service, get its HTTP port + let service_name = line + .split_whitespace() + .next() + .unwrap_or("") + .trim() + .to_string(); + + if service_name.is_empty() { + continue; + } + + // Try to get the HTTP port URL + let port_output = tokio::process::Command::new("kurtosis") + .args(["port", "print", enclave, &service_name, "http"]) + .output() + .await; + + if let Ok(port_out) = port_output { + let url = String::from_utf8_lossy(&port_out.stdout).trim().to_string(); + if !port_out.status.success() || url.is_empty() { + // Try "cl-http" as port name + let port_output2 = tokio::process::Command::new("kurtosis") + .args(["port", "print", enclave, &service_name, "cl-http"]) + .output() + .await; + if let Ok(port_out2) = port_output2 { + let url2 = + String::from_utf8_lossy(&port_out2.stdout).trim().to_string(); + if !url2.is_empty() { + return Ok(url2); + } + } + continue; + } + return Ok(url); + } + } + } + } + + // Fallback: try to use the enclave's default beacon port + bail!("Could not discover beacon URL for enclave '{enclave}'") +} + +/// Derive an enclave name from a config filename. +fn enclave_name(config: &Path) -> String { + let stem = config + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown"); + + // Strip common prefixes like "cb-" for cleaner names + let name = stem.strip_prefix("cb-").unwrap_or(stem); + format!("CB-{name}") +} + +/// Resolve config file paths: expand directories to *.yml files. +fn resolve_configs(inputs: &[PathBuf]) -> Result> { + let mut configs = Vec::new(); + + if inputs.is_empty() { + // Default: configs/generated/*.yml + let default_dir = PathBuf::from("configs/generated"); + if default_dir.is_dir() { + for entry in std::fs::read_dir(&default_dir)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) == Some("yml") + || path.extension().and_then(|e| e.to_str()) == Some("yaml") + { + configs.push(path); + } + } + } + configs.sort(); + return Ok(configs); + } + + for input in inputs { + if input.is_dir() { + for entry in std::fs::read_dir(input)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) == Some("yml") + || path.extension().and_then(|e| e.to_str()) == Some("yaml") + { + configs.push(path); + } + } + } else if input.is_file() { + configs.push(input.clone()); + } else { + warn!("Skipping non-existent path: {}", input.display()); + } + } + + configs.sort(); + Ok(configs) +} + +// --------------------------------------------------------------------------- +// Output +// --------------------------------------------------------------------------- + +fn print_batch_summary(batch: &BatchReport) { + println!(); + println!("╔══════════════════════════════════════════════════════════════╗"); + println!("║ Batch Verification Report ║"); + println!("╠══════════════════════════════════════════════════════════════╣"); + println!( + "║ Time: {:48} ║", + batch.timestamp + ); + println!( + "║ Total: {:48} ║", + batch.total + ); + println!( + "║ Passed: {:48} ║", + batch.passed.to_string().green() + ); + println!( + "║ Failed: {:48} ║", + batch.failed.to_string().red() + ); + println!("╠══════════════════════════════════════════════════════════════╣"); + + for result in &batch.results { + let status_icon = match result.result.as_str() { + "PASS" => "✓".green(), + "FAIL" => "✗".red(), + _ => "?".yellow(), + }; + let name = Path::new(&result.config) + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or(&result.config); + println!( + "║ {} {:20} {:6} ({}p / {}f / {}w / {}s) ║", + status_icon, name, result.result, result.passed, result.failed, result.warnings, result.skipped + ); + } + + println!("╚══════════════════════════════════════════════════════════════╝"); +} diff --git a/src/relay.rs b/src/relay.rs index fff9427..f5ec668 100644 --- a/src/relay.rs +++ b/src/relay.rs @@ -56,37 +56,63 @@ impl RelayClient { /// /// Returns payloads delivered in the given slot range. /// - /// The relay API supports a `cursor` param (not in the standard alloy query type) - /// which acts as the upper bound slot. We pass it as a raw query param alongside - /// the typed query fields. + /// The relay enforces a maximum limit of 200. We paginate using `cursor` + /// (which is an opaque DB ID from the last item's `block_number` field) until + /// we've fetched all payloads in the slot range or the relay returns no more results. pub async fn get_payloads_delivered( &self, start_slot: u64, end_slot: u64, ) -> Result> { - let limit = end_slot.saturating_sub(start_slot) + 1; + let mut all = Vec::new(); + let mut cursor: Option = None; + let max_pages = 50; // safety: 50 × 200 = 10,000 payloads max - let resp: Vec = self - .client - .get(format!( + for _ in 0..max_pages { + let url = format!( "{}/relay/v1/data/bidtraces/proposer_payload_delivered", self.base_url - )) - .query(&[ - ("cursor", end_slot.to_string()), - ("limit", limit.to_string()), - ]) - .send() - .await? - .error_for_status()? - .json() - .await?; + ); + let mut req = self.client.get(&url).query(&[("limit", "200")]); + if let Some(ref c) = cursor { + req = req.query(&[("cursor", c)]); + } + let resp: Vec = req + .send() + .await? + .error_for_status()? + .json() + .await?; + + if resp.is_empty() { + break; + } + + // Check if we've gone past our slot range + let min_slot = resp.iter().map(|p| p.slot).min().unwrap_or(0); + let _max_slot = resp.iter().map(|p| p.slot).max().unwrap_or(0); + + // Filter to our slot range + for p in &resp { + if p.slot >= start_slot && p.slot <= end_slot { + all.push(p.clone()); + } + } + + // If the oldest result is before our range, we can stop + if min_slot < start_slot { + break; + } + + // Use the last item's block_number as cursor for pagination + if let Some(last) = resp.last() { + cursor = Some(last.block_number.to_string()); + } else { + break; + } + } - // Filter to our slot range - Ok(resp - .into_iter() - .filter(|p| p.slot >= start_slot && p.slot <= end_slot) - .collect()) + Ok(all) } /// GET /relay/v1/data/bidtraces/builder_blocks_received?slot={slot} diff --git a/src/report.rs b/src/report.rs index e6e1639..7290037 100644 --- a/src/report.rs +++ b/src/report.rs @@ -100,6 +100,30 @@ pub fn print_report(report: &VerificationReport, json_mode: bool) { } } +/// Save a JSON report to `{output_dir}/{enclave}.json`. +/// +/// Only writes if the directory exists (caller must create it). +/// Logs a warning on failure but does not return an error — the +/// verification itself has already completed. +pub fn save_json_report(report: &VerificationReport, output_dir: &str) { + let report_path = format!("{}/{}.json", output_dir.trim_end_matches('/'), report.enclave); + match serde_json::to_string_pretty(report) { + Ok(json) => match std::fs::write(&report_path, &json) { + Ok(_) => { + // Intentionally not using tracing here — this module has no + // tracing dep and adding one would be overkill for a single message. + eprintln!("Report saved to {report_path}"); + } + Err(e) => { + eprintln!("Failed to write report to {report_path}: {e}"); + } + }, + Err(e) => { + eprintln!("Failed to serialize report for {report_path}: {e}"); + } + } +} + /// Determine process exit code from the report. /// /// - 0: all tier-1 checks passed