From d43cc8c7b7bbd26586426a5ec5a3d04772c90154 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Tue, 5 May 2026 16:53:10 -0700 Subject: [PATCH 01/20] autogenerate working configs --- .gitignore | 3 + justfile | 15 + keys/README.md | 7 + keys/node-0-pubkeys.json | 1 + keys/node-1-pubkeys.json | 1 + scripts/generate_kurtosis_configs.py | 431 +++++++++++++++++++++++++++ 6 files changed, 458 insertions(+) create mode 100644 keys/README.md create mode 100644 keys/node-0-pubkeys.json create mode 100644 keys/node-1-pubkeys.json create mode 100644 scripts/generate_kurtosis_configs.py diff --git a/.gitignore b/.gitignore index a29b902..d25c522 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ *.swp *.swo .DS_Store +configs/generated +kurtosis-configs/ +*/__pycache__ diff --git a/justfile b/justfile index 8a7ead2..698af24 100644 --- a/justfile +++ b/justfile @@ -49,3 +49,18 @@ verify-strict enclave="CB-Testnet" target_epoch="7" min_epochs="2": --timeout 3600 \ --live-metrics \ --strict + +# Generate Kurtosis YAML configs from templates into kurtosis-configs/ +generate-configs: + python3 scripts/generate_kurtosis_configs.py --output-dir kurtosis-configs/ + +# Verify that the generator reproduces ground truth exactly +verify-configs: + #!/usr/bin/env bash + set -euo pipefail + TMPDIR=$(mktemp -d) + python3 scripts/generate_kurtosis_configs.py --output-dir "$TMPDIR" + for f in cb-basic cb-multiple-relays cb-skip-sigverify cb-timing-games cb-extra-validation cb-mux; do + diff -q "$TMPDIR/${f}.yml" configs/generated/${f}.yml + done + echo "All configs match ground truth." diff --git a/keys/README.md b/keys/README.md new file mode 100644 index 0000000..5e6921f --- /dev/null +++ b/keys/README.md @@ -0,0 +1,7 @@ +# Keys +These are the deterministically created public keys from mnemonic: +``` +"giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete" +``` + +There are 128 per file and they are used to configure the mux settings within the Commit-Boost configs. diff --git a/keys/node-0-pubkeys.json b/keys/node-0-pubkeys.json new file mode 100644 index 0000000..acc8669 --- /dev/null +++ b/keys/node-0-pubkeys.json @@ -0,0 +1 @@ +["0xaaf6c1251e73fb600624937760fef218aace5b253bf068ed45398aeb29d821e4d2899343ddcbbe37cb3f6cf500dff26c","0x8aa5bbee21e98c7b9e7a4c8ea45aa99f89e22992fa4fc2d73869d77da4cc8a05b25b61931ff521986677dd7f7159e8e6","0x996323af7e545fb6363ace53f1538c7ddc3eb0d985b2479da3ee4ace10cbc393b518bf02d1a2ddb2f5bdf09b473933ea","0xa1584dfe1573df8ec88c7b74d76726b4821bfe84bf886dd3c0e3f74c2ea18aa62ca44c871fb1c63971fccf6937e6501f","0xac69ae9e6c385a368df71d11ac68f45f05e005306df3c2bf98ed3577708256bd97f8c09d3f72115444077a9bb711d8d1","0xa54fe5c26059ed60b4f0b66ef7b0bf167580504525f83c169507dc812816df41b1da6128341c23977300dffd32a32f41","0xad9222dec71ff8ee6bc0426ffe7b5e66f96738225db281dd20027a1556d089fdebd040abfbc2041d6c1a0d8fdcfce183","0x87231421a08ed28e7d357e2b37a26a458155c8d822d829344bd1029e5d175b5edfaa78f16f784f724a2caef124944c4f","0xb72cb106b7bc1ecae219e0ae1830a509ed18a042b56a2779f4033419de69ba8ae8017090caed1f5377bfa68506157360","0xb27ad13afc8ff30e087797b344c8382bb0a84447549f1b0274059ddd652276e7b148ba8808a10cc45746762957d4efbe","0xaaddb0cb69ca18f14aed7054e98a24df0ff606aeff919d489f7884fd1bd183bcb46ea54bc363146e1a88db36dc20a7a4","0x996d10c3026b9344532b06c70a596f972a1e779a1f6106d3da9f6ba376bbf7ec82d2f52629e5dbf3f7d03b00f6b862af","0x91709ee06497b9ac049325853d64947290189a8c2322e3a500d91e23ea02dc158b6db63ae558b3b7670357a151cd6071","0xa03c2a82374e04b2e0594c4ce14fb3f225b46f13188f0d8002a523c7dcfb939ae4856053c2c9c695374d7c3685df1ca5","0xab72cbc6575c3179680a58c0ecd5de46d2678ccbafc016746348ee5688edcb21b4e15bd37c70c508e3ea73103c2d566b","0xafa10af166a0dbf3a25ff86cd6f8e44cccc818c5e70cd70e4e98e226b158f3563450b3fb184d2649adbb11e53080d1ca","0xabd12678c73463ecea5867a80caf256d5c5e6ba53ff188b143a4d5be83365ad257edf39eaa1ba8753c4cdf4c632ff99e","0xa35c6004f387430c3797ab0157af7b824c8fe106241c7cdeb897d900c0f9e4bb945ff2a6b88cbd10e35ec48aaa554ecb","0x8a8bb292bcc481070d3afdbbc8789e2ab4b29c9603936e6d85f5ff71e23fc5b6d61009f0fa636b5d5b2dc309d39e3d75","0xae940a07850cf904b44f31cbf0e44824bae5ec36dcfdb7fad858f2a39dba38de82ca12b0ae939a34fce7a02e4b9789f8","0xa75ca9447dca3a3745ada36731187ddd1f6a152cf15d7446b785eab381e5c8562c1202a6e7a24080bc6b619a161113db","0x84a687ffdf21a0ad754d0164d1e2c03035613ab76359e7f5cf51ea4a425a6ee026725ec0a0dbd336f7dab759596f0bf8","0x96947de9e6068c22a7716656a2755a9551b0b66c2d1a741bf84a088fe1e840e992dc39861bf8ba3e8d5b6d21e8f57e64","0xb570dde8ee80512e3d031caf22e775c60f7f5a6cbdeb3e52e24cf8c867d38569a53dd19cdc36a03a1bbb3a8d94b03670","0xae5302796cfeca685eaf37ffd5baeb32121f2f07415bee26cc0051ee513ff3932d2c365e3d9f87b0949a5980445cb64c","0x8de5a6200cebb09b2198e69fed84bcd512ec5cf317c5f1ee99aad03d2a9a8564bf3807c08da2664222268d59c34a06e4","0xa1d9840eda3036fbf63eeea40146e4548553e6e1b2a653ab349b376f31b367c40d71fb59ff8e94b91daa99c262ec8b52","0x8419cf00f2783c430dc861a710984d0429d3b3a7f6db849b4f5c05e0d87339704c5c7f5eede6adfc8776d666587b5932","0x8d46e9aa0c1986056e407efc7013b7f271027d3c98ce96667faa98074ab0588a61681faf78644c11819a459a95689dab","0x930743bfc7e18d3bd7351eaa74f477505268c1e4e1fd1ca3ccccdefb2595517343bbb8f5589c435c3c39323a4c0080f8","0x81ea9f74ef7d935b807474e38954ae3934856219a23e074954b2e860c5a3c400f9aedb42cd27cb4ceb697ca36d1e58cb","0xa804e4fa8d1391a9d078aa93985a12503b84ce4f6f1f9e70ab7fca421e1cf972538666299d4c1bfc39327b469b2db7a8","0xab40dc1cfe273ad0da700c64f8fc94f91db253ca3acf20e336d9bd09de67eec5c7d3506285d83c7bb6a08d64b77e5f2d","0x8dfa86c051edd28c3554a30e40531c898e5936ad3002711616ddd1b27054bc39caedd505a200c3d23a1c3f6b26c50ae9","0x81fa222737fe818b43f55f209f42adaee135b2801d02709617fc88c2871852358260ace97cf323e761b5cc18bc7325b3","0xa4ee6d37dc259cbb5237e4265429a9fd8ab5643af81628cc101e0d8b4a333ef2618a37df89ea3f92b5ea4333d8cda393","0xa759f6bcca8f35fcaadc406cc4b828c016c0ed23882987a79f52f2933b5cedefe24e31df6fd0d38e8a802dbafd750d01","0x8d028a021c5c31a1aa1e18eda74cfaf0fba1c454c17c2e0fc730dd07a19d0c77f7a905d54017292f3e800ca06b6977cd","0xa2e2d8384fc87a512ee34eb43405fd82572c9d7cd96e155a382cda284e8df9eb7189c25b7473d89c63ea4e6080e10ff8","0x81b676591b823270a3284ace7d81cbce2d6cdce55bb0e053874d7e3a08f729453009d3e662ec3130379f43c0f3210b6d","0xb5e898a1fc06d51c695712928f44646d15451340d1b3e480a40f03250160bc07d3b6691ec94361dd524d59d9df7f76d3","0x84dc37ca3cd621d3da0fbdd11ca84021e0cd81a73d772dd6fcf19775b72eb64af4e573213378ccee0915dde92ac83ba6","0x84d08d58c31bcd3cddf93e13d6f50203897384afa34644bff1135efe8e01c81c6a91ca6c234bb1e51ca32e41b828aaf9","0xb2225575d5e70da1257db7a0d1222c5041b52aac61cf161e8fc8126a3fdf5eb4f0867d98dfe272199c36cf8f02661b3d","0xa8fa3584a92b079c8c73ed1553e5e161a0b21325fc2fc4e24a892354a899c7fc0bfb436a97a7ed1fc71bccda438ea715","0x9918433b8f0bc5e126da3fdef8d7b71456492dae6d2d07f2e10c7a7f852046f84ed0ce6d3bfec42200670db27dcf3037","0xb24391aa97bfff29adc935d06a2b6d583433caf82f92de1980e0192d3b270323bdbf24b86dc61520a40c419dde3df4b3","0xa62c0205fb22df8535c0b70076486e69dfa908feddae79e4a94a9d47b97ed190d228e1c6217e84a59882bb992dacae30","0xb63f327df68581cdc02a66c1c65e906a06a1a3a8d7a6e38f7b6da944e8e6cc2db85fced5327d8c12945ceb33018272ca","0x8aec5129a518010912215e1887191da94be419b4e75904c2ea745e2d253d707c088fa5b2c46dade1d162affe9f7ab17b","0x8725b32751419f22a54485790f8187d1ba52d84a31ad45738a93777fcd1ccbec1652229923f82f37793ce0fc2763fb4c","0xab64f900c770e2b99de6b86b4390bbd1579bd48dccec55800adbcf52e006f22128e9971bbf3a92cc0105b0974849935a","0xa0485d71f1f5e177f7d5bc9d98c5248a6a2d0de4554c2eaf02abae48f5a3e273b2ee7765784cf2a4cb7df84f617177c9","0xb09cb155daf2022afd18114a352e506a84065c80573cb0c7c310cbe92e2706cdcf91f74bbd9e464f74e3d831386d5033","0x99d83a0ba33161d8c6bbe80929fd9046d4dfdac43477ff85fea5bae925e6c179ad28eb338375ee2417acbd6576ee670a","0x958c2692b86b4d20eaea3bb45e9447ebbc5b93ccaf8d21ef659d0cefedf5c4371b31b460ae40e8243682bde505abac1e","0x8d8985e5dd341c9035b37bf7391c5944c28131b47c7d5359d18fca598010ba9a63e27c55e6b421a807038c320564db17","0xaf89ab00a0eab1131645292a9cfba583a69a1e3ac58b210e262494853e67385aeb50d4af428bdd577b9399daa96d8b20","0x896a51e0b0de0f29029af38b796db1f1e6d0f9f9085ade40a313a60cb723fa3d58f6587175570086c4fbf0fe5331f1c8","0x9763dde1b8028136a3ffd6dafd1f450e2cafb2819c7fa901f7c6e9cde8f2897ee7e9a45da6947fde1ad0d3836188eab5","0x8fda66b8607af873f4c2c8218dd3ffc7940d411047eb199b5cd010156af4845d21dd2e65b0e44cfffb5e78271e9bb29d","0x86e014747c7922ccfc2b9d4bf6c1ecf0dc800197037858d0b85ab1944b4c3c14b95e0ed325bc42a6f467bc47ec27bc7b","0x8c0d15baa72bfcd317e9b9402ca9bb6e7ae1db35ffce7faccae0bd19b3c8e5de7d5524aef0377770b3a90626627a9304","0xaf61f263addfb41c46d66e60ecfb598a5942f648f58718b6b4e4c92019fdb12328efbff98703134bcf28e9c1fab4bb60","0x8de7ec501d574152f52a962bf588573df2fc3563fd0c6077651208ed20f24f3d8572425706b343117b48bdca56808416","0xb97ecbcfe8c52b9bcdca9e75da13c5650b751b037c570934ea6b6441ff32de6566c50dafc0557e63105b2ee7e8cbb39e","0xac30aacd9e91cb0727c34ca6b40fbfd4d255b998471e25c443cf6cf777d6bb823a58e162958f32c3c5ca80453387a5d2","0xb43ccb05317c2b666470ab251e987d6bf31f5ead6b5edac5fe007dd334ae6ce1a92e24c19e5ab387cd8fa253b63bb78c","0x966c488d807b3208bb1b10a1af422bac8d363c8015cda4e24d214549ced019cd3dd575545dd887461cae3f70d95cb061","0xb3faeebfbebd085b9123ae0e09af9cd15d3b1db6a25f3e82d8b48b68e53522b41b342a3a3c8b008897df356048862d98","0xb8c6663371dc37bf083134dea26a20115ccc52b7c15a662bcfa33435e4ade14c6bc9714a5cdee492530accf8a327b2aa","0xa39731d5cb52838d02d4ff897ab908c0f76a9ef837f9288c634ed3091a1f69d5347dc65cd2c8009a5207a369a4c6bdae","0xb97dbe4add8aefd96c575ae9de19d1ac590bb7d92f23a9e4e113f7271c2243cf689e7645879efbb546d58ec44f5263d6","0x95833097520df43a5cb013e97f80041a7a0b7d84a4ec79e2f16baeeb6edfbcf62ede97becfde73883831bb65e1415dc0","0xaee3fbc60f939c125877a4f4529517edcf114fdb83715f9f4041eccebb91323ce8c4784ff87815c38752517b3d2e2725","0x92b68717b3b88b77716884d492966713d902eb35196cf71faf1fb5625327ddbdebae94786c77dde207b5666ffa6dff98","0x91da49dbfb1a4a339ee8e1b902bf18302c3ca948da1dacfecfe9934d231013544dbd97689c2996f22b456f7e408a138c","0x93d491d7211af181ccab8353567ad10c065a6e991c9b70def7215c62fa3bf843a177e18b00580deead4b5678f46d4d39","0x8890da2859ccb2afd0742c1c791075104d7acc207b6fa478bb1f94fd6665ac75e5dff4d2cac0f81e4168448bc3a4c90d","0xb320e188ef282109ba8dc3d2573f9edc33831c1025a29844c86e7fc60a25627e507964725ec8dbeccfcbaa12e7fb5e1a","0xa90dab71ad924fec01a579727736fd9a19a147bf57ef471255527b55a4702cfd54bb0300623c506823e723e7c30cf4dd","0x95d7b5b39931578c673a6ffdcd5b3618ddf08b40909fd0ee04a74c70aae4e88c55a43d5b71cdf40c514195a7805019b3","0x978eb7a3aee4238207d1e78684a6769e3d71f1a3b9e42ac53c347c97e15eec3ac0082abf65ad9cfd126c687c51436cc1","0x98213294b82bc66ee39e95a678472fb41df846ec2863c5be53e1fd56b6ff0fe1bfd5b2bd8c534dd97acbe597ad119cc7","0xabd3280a86fbb8d736717d8bd950920873ff7bb8a68ac4d7c339bb3c783c6f4e6e29912f6a3528e04b13aef57d4290a1","0xb0b263c298eb4d09de14dd71005fc683c4405efdb230c6661b9697d3a0978e5ed7736e76729b2233341e75ea46ecff1f","0x85fc722fa6d2c54b9610307ef86f0752d1428bdec6dc7e46ae14318bd9203f32df1ca8d0a420c36973067a50930b7720","0x958b57a4b16322f680eb4eebf37c538f1b5bf96400e51f99df6a4a439b75b9acd8866caf4a091d02f25510d3bba1aa26","0x889dbdf3bd68af1f6fd84cb6173b1fa1f7c5e6ba63297dc1e2f45cd1a82bb6231ba832adc5228143c5cff3ef0b1caae2","0x99377a407f49949e88651b57160044764930ddd9fc404f4b610a581c5edf9e906f3edaad611364d240336a3eaf4dab82","0xb6cd22959866607d91e13122bd34b050a7da426dad5e9779bcbb99e4a2e7bd5d18ce39f266bc61ef7524a2e2adfaf765","0x8548d74ab33e8dd285e72b1ea3c15eb08b66d555493bcf39aa4355af06ccf0f4469e09f110544dc04b1f3d0e7880ac1d","0x8457b3220923283912ea67a58687eab0fd1747497c94a8f73c4f75147fcc548acbe662e7d2542f5fc3c94749f2f6bd6f","0x961a7d85c4e61428f07c4c4d41cc03115ec423bf40172b21f9db161edec282c950782ade73a6dfa915da1c39c716eb1f","0xb26fae53e4f7d2088de3fd67ce06fde88a84a7963af20ff8901613bd8be30e9660b244e5e0bbcc39c940c814f6f3f318","0x986a201d308cce68d381d103680725fbce0fd9f618de12d371fd5c909049ba3081de693de9e0560364c201505b2fabaa","0xb165f7ea9728a1e55dcebc59c89321586290bb3c6aad2114217e81990104863b779ffac9728ed736b6069a36a26aad47","0x8e8d4d32b9c20bfa05ae904524dd1fcffe5ef1cfa451d6f0289eb54844a6ba033da0c656c1b69a508a45bb572797e195","0xa9fdcd176de75a9a1ec07d553a267413ec5406ea5f46a65d924519da3fa4b3009e2bb350fed52a51bd8683fca5d6184a","0x889fd954b35f31ea3f3eca4496ba13b9678a527e4d3137c8aea484b4d741886bdfff59f99a89721165f5ba7c9f5a366b","0xb4147399957d387dd0ad99bc9fbd7ddd3ad85809c9658d121879099c602db62c4a4ce4d122915b6d8135bbd16c13ffbc","0xb9882d1217732ebc2e6e2eaf42a9fc606ccd61e6a28a01f77c0d6829e255b51e50cebb38cc86254b37bcab1fe817e7b5","0xb8f3af1b7f9ea13cb73348da2fa847c6e0b89bb415dae2b62ea29e294060ea5c73c51bf00c50d8b76af2df9b35ca8c45","0xac97a570a795d24af13ef32709d71a37c0fb90a49581e735d635721581c5553c10eb15f43531121b6026f55d603d73f7","0xb0d4372ed0f55fa767a5fefc734c155a33821ffb6e6be4f628955a6477cf1d4f12c1d0e82426a8ca14b9e92ef094472f","0xa11c83b69a43111201fe36f54212afe6f12cd3a8ad551b586061901775a8205b816cc8638956c57a339228d61b520aa7","0x867e89563df1501ac7dc5a369e6713cebab2aa1b676ea6d97fcb62802488866ae1223b4ed6c00718ee895d7e8e650cac","0xb4cb3c19c6974ff1ff486b97cfc43eccba5e61e288ff008349cc99d1adb317ad2e9e28adafc48057322998eef132ab26","0xb1f2588848bded71a0ad83d124b5ff2d1ab964068e4eab97edc678eddc9aaa6175b51f561c4ec96c818ad5f6c59aa936","0xb4c9e24cb284d23c40bccfe377f9963dcdc1d22d5daf0766a6291354eb4ff3b4d1c21913dadae80fad218bc02778f5f2","0xac2d58a25bea23bb5160dc19a9e3a936dcf276e81079326e2925704ac9ba43907d653b841ebbdbbfbabd0f1ea1fac717","0xaa080afda88d384d10c98431dfec91c4072d6c9a4b43f302e5918d6292fdca1969f37a26cb05af494b6e46b3d61eb053","0xa7ed4d8afeea6c020adddf320ed86a863ec5d048236c28e56f61f83b4603cb9e4f2f2bf3f6dc10864936c98c0784c038","0x899f89c23e08b2e89a5416a7c6c9e76c7e1a064c49cb2f0825699d4412784bd4e34799c06b3faf0539b6fb4ab10c104e","0xa878b6b608d51a556d4d599810a70dac104d7971a8d99ac72341ff2685fa3a75561ca1fc4e5ffb4b73b5dcfce372350c","0xa8d747a2b2602fe32095a7dda37a86d94f8a59cc771902be8c45de5bfd9d58fc1bf46fb0cf867054cd5eacf4f331dfd1","0x851cb7093041931bbda885ef6d5a0411353e41b2baf7713ff7688b70d505dc97561a6b6482571aca964486d8a9b76a47","0x8d9dc5d04f5b105ddd3e84529e16f33748969fea81729ea7dc087fee8f1e3e2faeaaee3ca7ee5dc264a22cbe0ff809b3","0xb8f404f4d6965ff42eb6b325a85570ad85c4e8bfa9953d10eda05dc5db0070b8b43242eef8c2fbbe3a549149419a6428","0xa8494e9a4ca0b6fb595347fda04978eb76c0520b07b760ebce609c5051d6a7e2b01dc68f2b2eab6aca5a8fdbbdcd9346","0xa3baebcb3b1f6364d27e5bb8cab4cdee8b8388f29bf5ea235251742563e171a6b7e02a854ed24691725c58b5a7a88987","0xb3a1fbfd06f4d68415d699ced0f6d3f10ab370214fe66e2e4843bde2821bb4b878de8b7c9b2e38b45de941d06a665117","0xaa8ec705d7394909c67619fe6ba077508290a2756f0bd39b46f53950da8965c8a2fe8d1f2ce19bca22a8aeb9efbdbff6","0xb967afc934f2efd001a6901490e770602f80df9ec0eb3293490ed8c55e14d08f5bebb7bd183f485de29a8423d859a501","0xae2dd5d89fc56a0368ebba3891020b24ecf7acbdced259277d10eddf8d812568542ee3df43fab157034cc3620f17b75b","0xaffdd643395f3ca4138646f729aa0f5d1bfdd085df8d2559e075f72cbcaa24d9310074491910572390fbca5478d7d369","0x81093820fe0770a18a816945494db8fd957f10f7693da18f782e0968ef28ea8f23afefb0dc203925262352925f2739df","0xb9e03b94bb696b0e4c7939bce96d9e4fb1938074233d87b290b20cf66d3e48a7f3d852d89969f45c075e3dca91945832"] \ No newline at end of file diff --git a/keys/node-1-pubkeys.json b/keys/node-1-pubkeys.json new file mode 100644 index 0000000..1bdcc96 --- /dev/null +++ b/keys/node-1-pubkeys.json @@ -0,0 +1 @@ +["0xb05cafec5912f22dbd6f15677f25f13d93ecd5ec6f957fddd7cf27d73521b34aaaf6a219f77b21128d18321c2c8d679b","0x870342ee85d1d3eda564de4126f20880d59164e2f88652d9dcf3dc93d0bf19e22ca3a11305f1cba1cadaf2d117028936","0xb0db46ced0115b365df2d2c1e29ef3333b0bd4ed297288f7a09ca9c1de5e702ef8f2cbeb89d0d70a584cfe991cf7bb65","0xa567c07d1f258a7dc4f685b9c45c3217e9e640d8cbb3fde3a875e31b0212df6d48985f8524922205aaf6917a5b577d89","0xb7f216edfaf073f84d71ed41b7376d8fe85c88b40297636a2870278d00b452fd37b41af5e357a2dbe1297b53fb027e9a","0x9188ee0eb50d3a88a27a2ce4334ea8cac1662593526f4c280cb3049bc91afd8381ddbda124b9fab871aacf378ab5380a","0x8bf6583d6de04a89b9ca61c69977de4ca440c4c9b13c7a1c65e205979849376e239d03cb46d8bcdf58afca96a8e0fcfa","0xa9f291de2f415ab3a4002206769493d82a094e1934b78d98b79c93f70ffff7389d8ea0962b34b6df04ba999442a0fda5","0xb7ec2f481129da715b78d3c6bc1ab2e04bedadb937812497196409c08d7837d133c8bf52aec70689b1b180d8eef2676c","0xb0862b7f0788739e315de558384d6f95531088016f18330ab572fe89f3600098284f7a08a1ea8e7e33666904b918b17c","0x947e056beba42f6d7fec6712de16458c85ae513391dc4d42120e5877ed21a0681820a6150cb45af4f0cbf98c1d25c8cc","0xa03f7aa96fd1dcb385cf9f4f29f9f3ef4c25a47efdb090c18763a51608e92267daa98695f765705a7630269561f9086c","0xa62b23a8a25355c20cf4ebf93bc43a7b4076ab247ac7bff133ddaa7cfb9588e598d1022285b0e2f2fdd0ee3fd51a6f39","0x84c1b3ec3752b9d7e3ddfe87c68c59443b5fca9d18d78e5441c80e4fb6c0ecec27f8074811c2e5f823364a71e0567394","0xb7c37e964efdbe3916e611daa4a1241bad3aeedf6a57a21c87c23bec872452c05816ee9179d021c3cf36843063c687d0","0xae2bb170260de9afb23f49d5770474a65b4d380d904aa41cab7c2852074ed8ad9fb94f6d7e5a5a1f6f71c0dce0bddd12","0x8faadcfded5c85beb36f1cdc234f0cfd8ebb0820a18899445accf3e6e35efc0ee34419eeb0cfb097dff5fd6462b9ed90","0x837d8e7320247799d20afbddb410fda1bb7fcd31e36ccc841012e4dac0d643e8e5e12d2467d8d3219aab33b64280eaea","0x859155dd5a22f116ae8f61b1516770f8ff41ec0ea24b8b745171b4cf34981bb7d235e7e1a739a0589e7c7ff69ede9b15","0xb869ba1794050a014193ad467452efa3a54bfe6c6d1689bf7de9576eac2d2c2dfd4383a219e2d450b00cd9a70fc5e2a7","0xb06cb2010c1167c72840b3149ba92b326799375b1b05a1c0ca38ace5e8f61cba48c1b350a7220938e9c2c0fe6b6c2881","0xadcb081ad4dd8f1acfdf1a71360c6d5655bdd58d9bb1f09e4de43b7ff8a6da60b61df9ce65f9ffc951740dfc69812667","0x8d1df9e9132058de96486f102ff4ef6e34c988b6dd42a9462954218b8728310bdc25a4251c092eec8128bfdde893049e","0x8257c261afa77e79086b503de88dad720443a1d135cbc14f8a6de408a03ac5b9c4263731d6693bf843f0a9657aa3c4e8","0x8afc5f7128e998c2a59b9e2dd1805ccda56220d1bb25ea94c13d4f9abb4ab55a522518aabface4f10d54f45d08dffa8d","0xadc1c39301fa1fe99678a7b7887e895c8df24e15546b13d2237ab2795cc7004e6b68e69724f2c0922f119d5af8819bc9","0x94ef22c9183e15da2e4ad8e05a75c5b9201d52e9ad7f66cb0061b0c68779ba5a1fd0f11b9c365c1721d00199d287923c","0x8c3f73416c86d93ae2dbc2a468b5d6cc39d42c2ea7e20cc215212d3dd3b8efbed324bcd4a28de941fe855c54a92c6973","0x914dc0bb6af111bc9021887ca4dfb27cc2063c9ebd2be133754d30ca18fa8698787005538f556bd94867cc1c5f7b817a","0x95e8e9e5c389be338759c40d2e408b1f0b78ebc0ffbafae360e33b683ada3638414338b83490e9ab1ff067425d25d785","0x88ce4d8fee80abef17438d6e1dee0da0087b3fde540c7cd157e169866c603af0af26fe3c3d2ba517dead10376d60df89","0xb956f93f164469f3bb6b2e95c7d3db7d979b03457be68c6b227a9bda9b4be639198e89c9a62452162640cf95a00ad339","0x902b0898a017b3e98d334cdf49d5411e507f6043fca0624e937945c7dfb1829a3a2e1d0bdba654aef7d2e3c14c76b48d","0xb8a434ac1fa9c6a2c2c37da065576261d981d870923222a50b225422097a3e59598f548be8c6bc0b4764c02546f11b7c","0x98eeb5417011d88a2924cbe7ff7ed616a6b7dfe273187360757c667f27349f525ac60665f9f7ba2d07d91d2f94566f1e","0xa83c93593c32b08e89f0089e1a0892dcf121a856199b751ec40959433b4e64ee7fea260a16f4929261c5e2ebc148042c","0x995c28e8767e8677ef93cf6aa49453a8ad6c279622820e321ff9d352ceafffaa7ec082713e4c85e50cf9f7e11439cc16","0xb83d8622fe3180d6e8fd95b59d5e5bc6b7c0451040ec3b14588b1405410bbb249c15d449dd9afa4d9b42650b26a58fcc","0xa757657d95e795d3460d5454b0f3987885f0a8138c4de92e08d2843709f808adb191c9d2b22399bb9445dbb94e190382","0x81f9ac9f60825d682d5ab33098117dbdcf3c5245116c8c03a8c0493a5d441ba578a0b3d069d745cfdff70122c65e421a","0x8d19ef6c96e7ea917640ea3ad0c6c6a9ac8320456c3ec046cebaa625d415476f71d854d186a1758c79843925b2d268ac","0xa1c646e753dd9d9811cd75f58c371e74ce83f05606bc076d64bdc77adb313b1bfd8ceceae38d8a8b7d4795f10ac68d18","0x8cb3b628e5ec89b1cae8fffb946a1277613deec1f3f0e7d75708fe3f6b15c5efb75a0ec8dce885812cf14f3d589b6f91","0x933d88e34601df31b7a66bae1d49079548a2ae2d72037c7c3b2fb8925631989c1ae353311c36fe325b2f1fdc1f648194","0x8ff3e5eb9c905e42b88cd8fd8593bde4d44e22cbf296d47917a4b0f144fbbe2ab69a4421b2c0ff3500141dcf88a0b007","0x9592c95f5c1574b8c510e546759f383779a0364fefa009f84dfcebb2efc4e86b909a5fffb90ed79b5f66980b420db7e4","0x9218096756d3ccec228caddd27979b9050b58d7304f2be4f7d4aafb6ae19fdab0bb9149ce29517a383f59facaacf6d39","0x8de14b70a834e78b74d6bd46bd0f5f92d878a1b19f9081808dae86391aeab2a048368f0ae5b93ab412561db70482dba0","0xa0833e9ca3181e33a9b51d7c31722cd07c8f0a18a34ba083eaeb091529c2a454c3950ec449ab7f6f237995bf1ec0a802","0x98249520dcdc8be36b8afed019336e2eb478d44a0f5ebb6d7c994710856abf32ed25313867e060f83b205ea5ebe9f0fc","0x8f14d19885ec7d1bf7b6c4668d505283d68d1af78588532a15aa80ecf96a2d8bbeb813f63c25c15cd8f04c10fca4c49f","0xa6094298efdad03170daefba1bf92ee8691aa86fb753afcc252090c74c5f97c7d6cdfbbe0017f57e25d406d4ccebdb0e","0xa94d36839c1557727aa55fcca24faaab0d1e5f0ecff1dd709e22f0c55408b8b74d22173bc312d27baaa8688ae1b4be2c","0xaa228e16f801dfbde52d5f57b44e7e9a613fef33f893fad979ce003eeed012c8f45e9050db814e241be87ae94c0c4011","0xa8f3c2f7371294ca58c85e7b1513c155a39e5bd76eedd90f50a4fb50d6ae55b40623d5ba0c5d85c40def89d6c67d5fc3","0x8fe444e4b5610d3583a667dcf23af26a9f686db05e7de9ebc03f8ea0a756cc96b4f55611f5788d61f26e5bda1df70f68","0xadfdf512e8ac8e3f01ab9a03733c6d0bd0a5403778eebf691d2906380bc01591e93b43cae86c94f657c92c29d5698a06","0xaaafd2bc633a130d0798c8f398ebd00f5c1b131c2b4d48cbaddde1c8ed59eae6af8290d27700852df2b3cf23d630d807","0x863fb35bdce0573031210c5a6d3521b5a3e11cc99356de9e8adccf3accb4c4d387f954b202d0db647e65c331b5019226","0xb1748deeb17775232e5b53f55bc2d1b08c494a80cb727a2f2361af61464afa85a1afcae978e5d802e6c75e3be60965da","0x999664783bb5eb59491a99142392f020fb4e3b607203c3f9bec7ccb3537e8f7856d7ddfbede9dc3332cf40bdd4334a64","0x84e8581cf13f7df6a96835ad1330593bf5d1f4e5bd6341f7a54063b0233e921250d0c09a48849155953e293cf635d7b6","0xa608c06384bc606f723cde2a6f7d64a29de9cd987c0626dabea2414bd5b653647b31953c1156803191484d5ecf4630f1","0x81ef1d058664e94bcdd4876d049fc40e9a2d55a104a2fbcd33b63774a55bf994d72e6c7dc641af29bfa57362413db705","0x944a35116eaa393571e02f6751f1a2820ae5c4075e9bcb9746143a320c5a6ef5c3f4f939629dd0a2753b27bf5c5317a0","0xa1c888d5d2c76f2b388d6a42b42005adeca9af4d454c6342b9b634bcc6944432398823d240b9bd1bc7f37f58b390405d","0xa2869de5721730e34b8b074d27e6e4c2c6d09f6bffdf892edc6b3590830362f496e9c41c601a3e38eb80dcaf6f0c65c3","0xa069b0ecd8be3a45dc395f87b4ea9ac575f2f39dd922a1201ff0a3b9a5ec551f64a325647e6fa58770d94af853d6d6e5","0x8eece2624f5aeefc66c93625462a310c55f8df3c0632543d590db8799df2642ae39a0e77a9842e5d1982701311d46e42","0x8b1748a729412116660036812e9196fce6b9c553b47ffd942dc40c60a018bd49ae026e2bf4a1999773521b31bc12e64a","0x88cee7a3c398f11dc6eaa039542e9467bc7cfcab47718d92533d7adf7e36021e7cbee54342320837aca410b4e1dd6a33","0xade9004ebdbf580c9c29ab6d34b548f489609c98961b95b9f0b1c6bbad66ea4941143398510bf4f1ae0a7c72555cb9e4","0x93641e8f9e440bd769fa86db3f72ecf076da7d2cdb01355415c7c54fc90075899e40fe005898d63d706f3f94bfe9373d","0xa85336ac11331940f64d8bcded1f4a3dde75808d847311b2ff79bad985af2e2665fa5bf58d67b9eedf001d5663c08030","0xa7372f1be3e7bf1aa20000103a787929d2b884c2081392571cfc7f5034a91b79127706c6a2acde2c0277009fbd9943d1","0x953ba2a21db4a6b4063eccf6b599003803c9b03d24009ca78cd139a4d2cf26c602c34fb186e106dd0649985a3e5718a3","0x8d1c890c3036ed8913e69c6fa9f37816425ff5c06d1821db0d32e338c5d9b8db47e9146dd1327fe2c8358911b4b2fa26","0xb2ad1574eaca33f1555308e24b27a095d24aed8f4af5302ea2c6ba2e50936d25ffea7047be94065eac630693c7f86757","0x9806dff63a4ea679c0283a72d42a38d10d7949e6e7b5964ec1eced6e7ff1add4f32d7ff92d4097e7b0773633cded5051","0xa826dbfe4cccb4198a2689e1b32e59be8b774af0abb56c720ee2fc7472552f3ddc7a1ad15ddb2c2bf4189ecd5ada3761","0x8eeeef43e0617ee768138e98103fa9aaadd9491966a3904a24b51e5c955f3ed007a5ae64de4b541533a19246b3357e3a","0xa4bacbd1f1195b0ab5253b7437e05d235d663f9e8109dded69ef45a71a5366f337c2787679717ea06f8321cd0982c22d","0x8188a5547ef304357dc50d0e7ff3505202b18ed0dacb2f856e0450bdab0be01c509d4659ec44cf518cc5fcade8b3111d","0x9320225d3ed15c0fd4c0d9d629a6c8bff13af7eead740f4941aa850918649f00e42e4fcba371ffcdb2864ada1c654c6e","0xb44a24c31c8330ab530f2d4066256d26a12fbd3163ebbfa8d3cfe091a3926d16b17a5d2f3b4f87ee5cbe91c2fdfc5f05","0xb3abb58d18a587ff17cdbd85be687622eb2add07e80ca5190745badb03a216f8969b59405c85bd5a188008bcdf5a58c5","0xa5c521377c52dfe16b14e171bf3cddfceb7a6dbd768a7ce9716447b342466ab23e5c0e77854649735c4e41dd5f981e74","0x8ca49f0c11f05aaf0d7f7277fce738c55cf907d9b3a03a7eb61ed31b4c6808397506624310000a6dba2441e96c7b9ce0","0x84b2c527a35b380c1d407f89a70f28e5241ab6ebc902f13e3f8dabde85c1c2a4fc5bcb9b16474ff0cbda39dbabcef906","0x92a6792fe1ffb91a4ae57c6f8b82fa12d9f5f8485d21232304b7a8a971c2f5c0759839b90833cf561b4be224ccc23018","0xa45e18dc474050ec2d4a16975e6d59a9714f7a06458ce995e7b4882e21dbd2f4f1094733f69da85a6a636254b2d228bb","0x879336a84d4871909368d36a4896036e4482ea11204aeb57ee703f4bd323307f4521f038f766a1776a0b648d795901cd","0xb8716a085bf7915e9fedb32b67c1daf91fa32638a06e2edf1b38966566e8ca10ca06deaa0faec734e3e03899c8c919fc","0xa11525182b424d0c561b7e08882e05717cb4c93eb3002927efd7e077f37f20bcfc6f2fbd3f0ca9533d36f499564b279e","0xa067c96285c9e48306065cc1b0217e2de81be782fc101b7cb0a1cc1fd25f10e7bd428df87593a04babc7cdb0ceff31aa","0xa01ba34e831653404dc9cddc23155e6085f631c713681fa56412ff7c6ed8f5702d384649f42791f6b36843522ca0b8de","0xa49bb35ada704f33f70d2d77a72498b3e2fa01fc9b7f8f02b2a6397a825b043b3eaf00e440c48ae522e42119095c88bd","0xabe99cd6c855608a1d0d34e724a6f4dea47cb8148544809169d7d323ad47efd55402adffe2d20f346fe17e3f60402e37","0xab22c3c322bc0e0594926d5ac48e2fad0eeaa24d7dc616fc9bb0af6652188242d1dc59072f35a1552b3a0bae0ad6b170","0x881b0b35d01247b4d7e5a1a3d2b071c40904d0c1892eed4f8b7f1c531b18feba1e5b81acf8c69cd6b6ad0cb31f658333","0xb586ce5eee6b78c9d65284be1496d8b6d32080ff573361ad2d69e26859772e8c2e0263b0f173489adf9c7b7cd98bf95f","0xb602e252aea22de9c5d5733b8ddcd2b4db8aaf24fe57f73b5584bbc9acb5f89af75717290ec7f7ea2a9fbc1ee7dd82d7","0x8a48989a82f473050e42cd9cb58d8538d285e069d775b034df1d3a703fd5821d8ae6e2bc5cc5ee3bc770cb28ba59f6bd","0x87c10f9555a9040368909a1c4670de4c2f725b4a485a77716e5e060e65c47c0edb926f28a422d2ec24868bdaf771dbd3","0xa38f647bb270cfd59a8567c5afe5e02463eafd2fc9d0379494c0b83b235594cf3283d5cf67ae6ddcca00f78bce7d78d2","0x97844029484238a8951e707cd4f46207722e0f9fd15b050fc2798362d87293af7021398188e1fc033423d75a604fd830","0xaf49927c935c201799ab1f4961786ce1bc01266bcd6d2adb81b193c3362b73f2dac32db526b597ec5baec292dea920d5","0xb79d0e7e5004f1679bd96e89fd94c5cc2ee5c368d9207b4c778b6983f288e5f4cee3bf84c7ff4a16935140da31615b9f","0xb7e03f23fac574fc7be4a455aad6807d0c71bf06fd74cd4a7d72cbf1c1db1be5a01f50dbd51457264a16d8874a6a1648","0xac1ada1e9db792ed12d314638c5b16e7820eeb0d178aeeeed1e986b6919eb886c7fda24e058768ec0c63e96c1b2c632f","0x8ff47ed158f7547c94a40492dd7996ebf2a5223f2d1313219b0e56ef41ad3a7676a59ed91f1870d7c77e0b398d1848fd","0x94ef5705ed74f1618a6918d903de2b950df9fa5273f27ce89481155c1f5b084370d59c9ac2df9ab2bacd6a9ff9fd1138","0xa25eef6fa1f3b3bd7abb301c28230897aaf57fad66ce0871c54241948328bf40092f343b5f092d89ffa79b462a0ad0f1","0x89108f74cef974fd3a3d6263831f54007aab5d272ae9c0e7193575406eb9556445c1bab5f1338a7fefe97d030ce2c0ad","0xb92e5259c50d22699af4fd397cdd32177154a01f26c3d2e687489af5a94e86aa7cf342213d0827e4c90801a8cfafaad3","0x815e815cf1a66a872ea0f171341d096afd5b026a6f6ef60d176f1598a6dca647379ddb82d00df0b28c0479d291a90b9a","0x9491aca6c6afd9319152796ab99fe3eedffcfcb959e7d7da0ba68e04754534d8576c394d80f407d84cfd3f25d90df4a0","0x84189eff6ff8fe060064c0bb0c9c50e8680ca4198c14878807ac781f62376662c13d6c8cf4c588428fba90541d12b35e","0x9995cd3fe60b1757fb734715aea9941479756e8e1a8912cfd4ab20e1f9e3c2496740a4bfe66c2633876606d5136f73ae","0xb9b76b28c192907820d1ea5e0fab9c545e8859cadf0a25e17bbeebeaaa8a01f36b0971124e844ef976fa076689ed03a8","0x876df2629991ad5014a9552fe4bef866ad93db67c291fed081254316441b0dc974d11b02d318c2342e2379293d88036e","0x84cc6208ab0086c1a439a4ae3787a46a133847a5daeb9f7476814490be581f25c5bb515d43ad6092d4a8a01083917a6f","0xad8f20f15d45a0293e58e0188f19cc59e649daabc63e03e66da3ef58306ecfe8f2501d6e0ea3e3017734585ec398507d","0xb0d6f1ceefb752039bd31dfd2c26dc4f96a5a0d8f0772e08737438aa6e324f557d09c643891dfc5358b23b25ba3fd310","0xae4dc089d92c027f4cc3141c309edb5e23f0bc0935146bf3db2592317479e7757fbc65624ef9c1ecfb54d21602079d06","0x91edd2ba701744a581d1dad1b67b862d08e0dad9558f8c31826692cb9eb11a97f25ea32859113300e69136b9b01b4ac2","0xb9b781554f467aea192418c18dae0bdc986d447f6a020224d5062143cb96c1bde68d8365d1108c11162492a79238fd96","0xa09202a971426cc12fc4609b8c63cf8b1e11e2ba59db9238da5d7eaff9b8df01d7ed6cd927f82685839edd3a621112ed"] \ No newline at end of file diff --git a/scripts/generate_kurtosis_configs.py b/scripts/generate_kurtosis_configs.py new file mode 100644 index 0000000..3f15ba6 --- /dev/null +++ b/scripts/generate_kurtosis_configs.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +"""Generate Kurtosis YAML configs for Commit-Boost testing scenarios.""" + +import argparse +import json +import os +import sys + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +KEYS_CONFIGS_DIR = os.path.join(SCRIPT_DIR, "../keys") + +# --------------------------------------------------------------------------- +# Shared YAML fragments +# --------------------------------------------------------------------------- + +COMMON_PARTICIPANTS = """\ +participants: + - el_type: geth + cl_type: lighthouse""" + +COMMON_ADDITIONAL_SERVICES = """\ +additional_services: + - dora + - spamoor + - prometheus + - assertoor""" + +COMMON_NETWORK_PARAMS = ( + "network_params:\n" + ' network: kurtosis\n' + ' network_id: "3151908"\n' + ' deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa"\n' + " seconds_per_slot: 12\n" + " slot_duration_ms: 12000\n" + " num_validator_keys_per_node: 128\n" + " preregistered_validator_keys_mnemonic:\n" + ' "giant issue aisle success illegal bike spike\n' + " question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy\n" + ' very lucky have athlete"' +) + +ASSERTOOR_PARAMS = ( + "assertoor_params:\n" + ' run_stability_check: false\n' + ' run_block_proposal_check: false\n' + " tests:\n" + ' file: "http://host.docker.internal:8888/assertoor/cb-mev-pipeline.yaml"' +) + +MUX_NETWORK_PARAMS = ( + "network_params:\n" + ' network: kurtosis\n' + ' network_id: "3151908"\n' + ' deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa"\n' + " seconds_per_slot: 12\n" + " slot_duration_ms: 12000\n" + " num_validator_keys_per_node: 256\n" + " preregistered_validator_keys_mnemonic:\n" + ' "giant issue aisle success illegal bike spike\n' + " question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy\n" + ' very lucky have athlete"' +) +# --------------------------------------------------------------------------- + +def load_pubkeys(filename): + path = os.path.join(KEYS_CONFIGS_DIR, filename) + if not os.path.isfile(path): + print(f"Error: missing pubkey file {path}", file=sys.stderr) + sys.exit(1) + with open(path, "r") as f: + return json.load(f) + + +def format_pubkey_list(pubkeys): + """Return a multiline list literal with 4-space entry indentation. + + When placed inside a YAML literal block that is itself indented 4 spaces, + the entries end up at 8 spaces total — matching the ground truth. + """ + lines = ["["] + for i, pk in enumerate(pubkeys): + comma = "" if i == len(pubkeys) - 1 else "," + lines.append(f' "{pk}"{comma}') + lines.append("]") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# TOML builders (raw; get indented 4 spaces by build_mev_params) +# --------------------------------------------------------------------------- + +def build_cb_toml_basic(timeout_get_header_ms, timeout_get_payload_ms, + extra_pbs_lines=None, per_relay_lines=None): + lines = [ + 'chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" }', + "", + "[pbs]", + 'host = "0.0.0.0"', + "port = {{ .Port }}", + f"timeout_get_header_ms = {timeout_get_header_ms}", + f"timeout_get_payload_ms = {timeout_get_payload_ms}", + "late_in_slot_time_ms = 2000", + ] + + if extra_pbs_lines: + # Insert after port (idx 4), before timeouts (idx 5) + insert_idx = 5 + for line in extra_pbs_lines: + lines.insert(insert_idx, line) + insert_idx += 1 + + lines.append("") + lines.append("{{ range $index, $relay := .Relays }}") + lines.append("[[relays]]") + lines.append('id = "mev_relay_{{$index}}"') + lines.append('url = "{{ $relay }}"') + + if per_relay_lines: + for line in per_relay_lines: + lines.append(line) + + lines.append("{{- end }}") + lines.append("") + lines.append("[logs.stdout]") + lines.append('level = "debug"') + lines.append("") + lines.append("[logs.file]") + lines.append("enabled = false") + + return "\n".join(lines) + + +def build_cb_toml_mux(pubkeys_node0, pubkeys_node1): + node0_list = format_pubkey_list(pubkeys_node0) + node1_list = format_pubkey_list(pubkeys_node1) + + lines = [ + 'chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" }', + "", + "[pbs]", + 'host = "0.0.0.0"', + "port = {{ .Port }}", + "timeout_get_header_ms = 950", + "timeout_get_payload_ms = 4000", + "late_in_slot_time_ms = 2000", + "", + "{{ range $index, $relay := .Relays }}", + "[[relays]]", + 'id = "mev_relay_{{$index}}"', + 'url = "{{ $relay }}"', + "{{- end }}", + "", + "[[mux]]", + 'id = "node_0_to_helix"', + f"validator_pubkeys = {node0_list}", + "timeout_get_header_ms = 900", + "[[mux.relays]]", + 'id = "mux_helix"', + 'url = "{{ index .Relays 0 }}"', + "", + "[[mux]]", + 'id = "node_1_to_flashbots"', + f"validator_pubkeys = {node1_list}", + "timeout_get_header_ms = 900", + "[[mux.relays]]", + 'id = "mux_flashbots"', + 'url = "{{ index .Relays 1 }}"', + "", + "[logs.stdout]", + 'level = "debug"', + "", + "[logs.file]", + "enabled = false", + ] + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# MEV params builder +# --------------------------------------------------------------------------- + +def build_mev_params(relays, images, toml_block): + lines = ["mev_params:"] + + if isinstance(relays, list): + lines.append(" mev_relay:") + for r in relays: + lines.append(f" - {r}") + else: + lines.append(f" mev_relay: {relays}") + + lines.append(" mev_sidecar: commit-boost") + lines.append(" mev_builder: flashbots") + lines.append("") + + for key, val in images.items(): + lines.append(f" {key}: {val}") + + lines.append("") + lines.append(" commit_boost_config: |") + # Indent every non-empty TOML line by 4 spaces; keep blanks truly empty + for line in toml_block.splitlines(): + if line.strip(): + lines.append(f" {line}") + else: + lines.append("") + # NB: no trailing empty line here — that is handled by the caller's join + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Scenario generators +# --------------------------------------------------------------------------- + +def generate_basic(): + comment = ( + "# cb-basic: Single relay (helix) with default Commit-Boost config.\n" + "#\n" + "# Tests the core MEV pipeline through Commit-Boost with a single Helix\n" + "# relay as the only relay endpoint." + ) + images = { + "helix_relay_image": "helix-relay:kurtosis", + "mev_boost_image": "commit-boost/pbs:kurtosis", + "mev_builder_cl_image": "sigp/lighthouse:latest", + "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + } + toml = build_cb_toml_basic(950, 4000) + mev_params = build_mev_params("helix", images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + COMMON_NETWORK_PARAMS, + ASSERTOOR_PARAMS, + ]) + "\n" + + +def generate_multiple_relays(): + comment = ( + "# cb-multiple-relays: Two relays (helix + flashbots) behind a single\n" + "# Commit-Boost sidecar.\n" + "#\n" + "# Tests that CB correctly routes get_header requests to both relays,\n" + "# aggregating responses and selecting the best bid." + ) + images = { + "helix_relay_image": "helix-relay:kurtosis", + "mev_relay_image": "ethpandaops/mev-boost-relay:main", + "mev_boost_image": "commit-boost/pbs:kurtosis", + "mev_builder_cl_image": "sigp/lighthouse:latest", + "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + } + toml = build_cb_toml_basic(950, 4000) + mev_params = build_mev_params(["helix", "flashbots"], images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + COMMON_NETWORK_PARAMS, + ]) + "\n" + + +def generate_skip_sigverify(): + comment = ( + "# cb-skip-sigverify: Signature verification disabled for header responses.\n" + "#\n" + "# Tests the CB fast path where BLS verification is skipped. This trades\n" + "# correctness for speed — useful to verify that the path exists and is\n" + "# reachable under load." + ) + images = { + "helix_relay_image": "helix-relay:kurtosis", + "mev_boost_image": "commit-boost/pbs:kurtosis", + "mev_builder_cl_image": "sigp/lighthouse:latest", + "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + } + toml = build_cb_toml_basic(950, 4000, extra_pbs_lines=["skip_sigverify = true"]) + mev_params = build_mev_params("helix", images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + COMMON_NETWORK_PARAMS, + ]) + "\n" + + +def generate_timing_games(): + comment = ( + "# cb-timing-games: Aggressive timing game configuration.\n" + "#\n" + "# Tests CB's ability to orchestrate repeated get_header polls with\n" + "# short timeouts in order to arrive at the best bid as late as possible\n" + "# in the slot. Per-relay timing overrides are enabled for all relays." + ) + images = { + "helix_relay_image": "helix-relay:kurtosis", + "mev_relay_image": "ethpandaops/mev-boost-relay:main", + "mev_boost_image": "commit-boost/pbs:kurtosis", + "mev_builder_cl_image": "sigp/lighthouse:latest", + "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + } + toml = build_cb_toml_basic( + 400, + 2000, + per_relay_lines=[ + "enable_timing_games = true", + "target_first_request_ms = 100", + "frequency_get_header_ms = 200", + ], + ) + mev_params = build_mev_params(["helix", "flashbots"], images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + COMMON_NETWORK_PARAMS, + ]) + "\n" + + +def generate_extra_validation(): + comment = ( + "# cb-extra-validation: Enable extra validation of get_header responses\n" + "# via a local execution layer client.\n" + "#\n" + "# Tests that CB will RPC-call the execution client to verify block\n" + "# parameters before returning a header to the beacon node." + ) + images = { + "helix_relay_image": "helix-relay:kurtosis", + "mev_boost_image": "commit-boost/pbs:kurtosis", + "mev_builder_cl_image": "sigp/lighthouse:latest", + "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + } + toml = build_cb_toml_basic( + 950, + 4000, + extra_pbs_lines=[ + "extra_validation_enabled = true", + 'rpc_url = "http://el-1-geth-lighthouse:8545"', + ], + ) + mev_params = build_mev_params("helix", images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + COMMON_NETWORK_PARAMS, + ]) + "\n" + + +def generate_mux(pubkeys_node0, pubkeys_node1): + comment = ( + "# cb-mux: Multiplexed relay routing per validator node.\n" + "#\n" + "# Routes all 128 validators from node-0 exclusively to the Helix relay and\n" + "# all 128 validators from node-1 exclusively to the Flashbots relay.\n" + "# This tests CB's ability to partition the validator set and apply\n" + "# per-mux timeout and relay configurations." + ) + images = { + "helix_relay_image": "helix-relay:kurtosis", + "mev_relay_image": "ethpandaops/mev-boost-relay:main", + "mev_boost_image": "commit-boost/pbs:kurtosis", + "mev_builder_cl_image": "sigp/lighthouse:latest", + "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + } + toml = build_cb_toml_mux(pubkeys_node0, pubkeys_node1) + mev_params = build_mev_params(["helix", "flashbots"], images, toml) + return "\n\n".join([ + comment, + COMMON_PARTICIPANTS, + COMMON_ADDITIONAL_SERVICES, + "mev_type: custom", + mev_params, + MUX_NETWORK_PARAMS, + ]) + "\n" + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description="Generate Kurtosis YAML configs for Commit-Boost testing." + ) + parser.add_argument( + "--output-dir", + default="configs/generated", + help="Directory to write generated YAML configs (default: kurtosis-configs).", + ) + args = parser.parse_args() + + output_dir = os.path.abspath(args.output_dir) + os.makedirs(output_dir, exist_ok=True) + + pubkeys_node0 = load_pubkeys("node-0-pubkeys.json") + pubkeys_node1 = load_pubkeys("node-1-pubkeys.json") + + scenarios = { + "cb-basic.yml": generate_basic(), + "cb-multiple-relays.yml": generate_multiple_relays(), + "cb-skip-sigverify.yml": generate_skip_sigverify(), + "cb-timing-games.yml": generate_timing_games(), + "cb-extra-validation.yml": generate_extra_validation(), + "cb-mux.yml": generate_mux(pubkeys_node0, pubkeys_node1), + } + + for filename, content in scenarios.items(): + path = os.path.join(output_dir, filename) + with open(path, "w") as f: + f.write(content) + print(f"Generated {path}") + + +if __name__ == "__main__": + main() From cf96addec1918a4b832068a7016d76878506ad25 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Tue, 5 May 2026 16:58:17 -0700 Subject: [PATCH 02/20] Add ethereum-package fork with helix support --- .gitmodules | 3 +++ ethereum-package | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 ethereum-package diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..bea06ae --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ethereum-package"] + path = ethereum-package + url = https://github.com/JasonVranek/ethereum-package.git diff --git a/ethereum-package b/ethereum-package new file mode 160000 index 0000000..8a11379 --- /dev/null +++ b/ethereum-package @@ -0,0 +1 @@ +Subproject commit 8a11379782f2d1f6a5ca7ed912f0225d4f9ff8e5 From f033474d5034376b2542b207481a55ef7bfe62f3 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Tue, 5 May 2026 18:00:54 -0700 Subject: [PATCH 03/20] use mev_type: custom fork --- ethereum-package | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum-package b/ethereum-package index 8a11379..bff8b58 160000 --- a/ethereum-package +++ b/ethereum-package @@ -1 +1 @@ -Subproject commit 8a11379782f2d1f6a5ca7ed912f0225d4f9ff8e5 +Subproject commit bff8b58edd239693f5ffc48e7d4967bd465cd825 From f752601dac871a872b330bc12af1ffb267ea6df2 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Wed, 6 May 2026 00:02:11 -0700 Subject: [PATCH 04/20] clean up cmds, add ability to detect mux cnfigs (verifying mux adherence correctly broken atm) --- .env.example | 19 + .gitignore | 2 + Cargo.lock | 20 + Cargo.toml | 1 + README.md | 28 +- justfile | 59 ++- scripts/generate_kurtosis_configs.py | 87 ++-- scripts/run-and-verify.sh | 18 +- src/checks/mod.rs | 1 + src/checks/mux_routing.rs | 610 +++++++++++++++++++++++++++ src/checks/relay_pipeline.rs | 86 +++- src/discovery.rs | 56 ++- src/main.rs | 72 ++++ src/report.rs | 24 ++ 14 files changed, 1026 insertions(+), 57 deletions(-) create mode 100644 .env.example create mode 100644 src/checks/mux_routing.rs diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d1000be --- /dev/null +++ b/.env.example @@ -0,0 +1,19 @@ +# --------------------------------------------------------------------------- +# cb-testing Docker image overrides +# --------------------------------------------------------------------------- +# Copy this file to .env and customize for your local images. +# All values have defaults in generate_kurtosis_configs.py, so this file +# is entirely optional — only needed when using custom Docker images. +# --------------------------------------------------------------------------- + +# Helix relay image (your custom relay build) +# Default: helix-relay:kurtosis +# HELIX_RELAY_IMAGE=helix-relay:kurtosis + +# mev-boost relay image (used in multi-relay scenarios) +# Default: ethpandaops/mev-boost-relay:main +# MEV_RELAY_IMAGE=ethpandaops/mev-boost-relay:main + +# Commit-Boost PBS image (your local PBS build) +# Default: commit-boost/pbs:kurtosis +# MEV_BOOST_IMAGE=commit-boost/pbs:kurtosis diff --git a/.gitignore b/.gitignore index d25c522..4bc7c38 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ .DS_Store configs/generated kurtosis-configs/ +scripts/kurtosis-configs/ +.env */__pycache__ diff --git a/Cargo.lock b/Cargo.lock index 3ae42ec..24d22d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -684,6 +684,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "serde_yaml", "tokio", "tracing", "tracing-subscriber", @@ -2701,6 +2702,19 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "serdect" version = "0.2.0" @@ -3238,6 +3252,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 2c9d600..6e83775 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ serde_json = "1" tokio = { version = "1", features = ["full"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +serde_yaml = "0.9.34" [profile.release] strip = true diff --git a/README.md b/README.md index 4a9d532..5eccbd4 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,28 @@ Validator -> CB (submit_block) -> Relay -> Payload on chain If you're testing a local CB build, you also need: - The [commit-boost-client](https://github.com/Commit-Boost/commit-boost-client) repo cloned +## Docker image configuration + +The generated configs embed Docker images for the relay, PBS sidecar, builder CL, +and builder EL. These are hardcoded by default but can be overridden via `.env`: + +```bash +# From the cb-testing/ directory: +cp .env.example .env +# Edit .env to point at your local images +``` + +| Variable | Default | Purpose | +|---|---|---| +| `HELIX_RELAY_IMAGE` | `helix-relay:kurtosis` | Custom Helix relay | +| `MEV_RELAY_IMAGE` | `ethpandaops/mev-boost-relay:main` | mev-boost relay (multi-relay scenarios) | +| `MEV_BOOST_IMAGE` | `commit-boost/pbs:kurtosis` | Commit-Boost PBS sidecar | +| `BUILDER_CL_IMAGE` | `sigp/lighthouse:latest` | Builder consensus client | +| `BUILDER_EL_IMAGE` | `ethpandaops/reth-rbuilder:develop` | Builder execution client | + +The `.env` file is read automatically by `generate_kurtosis_configs.py`. +It is gitignored — do not commit it. Use `.env.example` as the reference. + ## Quick start ```bash @@ -41,10 +63,10 @@ If you're testing a local CB build, you also need: ### Testing a local CB build ```bash -# In the commit-boost-client repo, build the PBS Docker image: +# 1. Configure images in .env (if using custom builds) +# 2. In the commit-boost-client repo, build the PBS Docker image: just build-pbs kurtosis - -# Then run the verifier (it uses the commit-boost/pbs:kurtosis image): +# 3. Run the verifier: ./scripts/run-and-verify.sh --config configs/basic-pbs.yml ``` diff --git a/justfile b/justfile index 698af24..eaea4c8 100644 --- a/justfile +++ b/justfile @@ -50,17 +50,58 @@ verify-strict enclave="CB-Testnet" target_epoch="7" min_epochs="2": --live-metrics \ --strict -# Generate Kurtosis YAML configs from templates into kurtosis-configs/ +# Generate Kurtosis YAML configs from templates into configs/generated/ +# Loads optional .env for Docker image overrides (see .env.example). generate-configs: - python3 scripts/generate_kurtosis_configs.py --output-dir kurtosis-configs/ + python3 scripts/generate_kurtosis_configs.py -# Verify that the generator reproduces ground truth exactly -verify-configs: +# Run kurtosis testnet with verification on target `config` +testnet config: + ./scripts/run-and-verify.sh \ + --config {{config}} \ + --json \ + --live-metrics \ + --min-epochs 1 \ + --target-epoch 2 + +# Run all generated configs sequentially and print a summary. +# +# Iterates configs/generated/*.yml, runs each via `testnet` with +# --json-dir so results are saved. After all complete, aggregates +# the JSON reports into a summary table. +test-all: #!/usr/bin/env bash set -euo pipefail - TMPDIR=$(mktemp -d) - python3 scripts/generate_kurtosis_configs.py --output-dir "$TMPDIR" - for f in cb-basic cb-multiple-relays cb-skip-sigverify cb-timing-games cb-extra-validation cb-mux; do - diff -q "$TMPDIR/${f}.yml" configs/generated/${f}.yml + RESULTS_DIR="$(mktemp -d)" + SCRIPTS_DIR="$(dirname "$0")/scripts" + CONFIGS_DIR="$(dirname "$0")/configs/generated" + echo "Results dir: $RESULTS_DIR" + echo "" + for config in "$CONFIGS_DIR"/*.yml; do + name="$(basename "$config" .yml)" + # Derive a unique enclave name from the config filename + enclave="CB-${name#cb-}" + echo "[$(date +%H:%M:%S)] Running $name..." + "$SCRIPTS_DIR/run-and-verify.sh" \ + --config "$config" \ + --enclave "$enclave" \ + --json-dir "$RESULTS_DIR" \ + --live-metrics \ + --min-epochs 2 \ + --target-epoch 2 \ + 2>&1 | tail -1 || echo " $name: FAILED (exit code $?)" + done + echo "" + echo "========================================" + echo " Batch Summary" + echo "========================================" + for f in "$RESULTS_DIR"/*.json; do + [ -f "$f" ] || continue + name="$(basename "$f" .json)" + result="$(jq -r '.result // "unknown"' "$f" 2>/dev/null || echo "parse-error")" + passed="$(jq '[.checks[] | select(.status == "Pass")] | length' "$f" 2>/dev/null || echo "?")" + failed="$(jq '[.checks[] | select(.status == "Fail")] | length' "$f" 2>/dev/null || echo "?")" + warn="$(jq '[.checks[] | select(.status == "Warn")] | length' "$f" 2>/dev/null || echo "?")" + echo " $name: $result (${passed}p / ${failed}f / ${warn}w)" done - echo "All configs match ground truth." + echo "========================================" diff --git a/scripts/generate_kurtosis_configs.py b/scripts/generate_kurtosis_configs.py index 3f15ba6..adf056d 100644 --- a/scripts/generate_kurtosis_configs.py +++ b/scripts/generate_kurtosis_configs.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 -"""Generate Kurtosis YAML configs for Commit-Boost testing scenarios.""" +"""Generate Kurtosis YAML configs for Commit-Boost testing scenarios. + +Reads optional .env file from the project root for Docker image overrides. +See .env.example for all available variables and defaults. +""" import argparse import json @@ -8,7 +12,44 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -KEYS_CONFIGS_DIR = os.path.join(SCRIPT_DIR, "../keys") +PROJECT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..")) +KEYS_CONFIGS_DIR = os.path.join(PROJECT_DIR, "keys") + +# --------------------------------------------------------------------------- +# Load .env overrides from cb-testing/.env +# --------------------------------------------------------------------------- + +def load_env(): + """Load key=value pairs from .env file in the project root. + + Simple parser: no variable expansion, no quoting tricks. Just + strips comments and blank lines. Missing file = not an error. + Returns a dict of (key, value) pairs. + """ + env_path = os.path.join(PROJECT_DIR, ".env") + result = {} + if not os.path.isfile(env_path): + return result + with open(env_path) as f: + for line in f: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if "=" not in stripped: + continue + key, _, value = stripped.partition("=") + result[key.strip()] = value.strip() + return result + + +ENV = load_env() + +# Image defaults (overridable via .env) +HELIX_RELAY_IMAGE = ENV.get("HELIX_RELAY_IMAGE", "helix-relay:kurtosis") +MEV_RELAY_IMAGE = ENV.get("MEV_RELAY_IMAGE", "ethpandaops/mev-boost-relay:main") +MEV_BOOST_IMAGE = ENV.get("MEV_BOOST_IMAGE", "commit-boost/pbs:kurtosis") +BUILDER_CL_IMAGE = ENV.get("BUILDER_CL_IMAGE", "sigp/lighthouse:latest") +BUILDER_EL_IMAGE = ENV.get("BUILDER_EL_IMAGE", "ethpandaops/reth-rbuilder:develop") # --------------------------------------------------------------------------- # Shared YAML fragments @@ -223,10 +264,8 @@ def generate_basic(): "# relay as the only relay endpoint." ) images = { - "helix_relay_image": "helix-relay:kurtosis", - "mev_boost_image": "commit-boost/pbs:kurtosis", - "mev_builder_cl_image": "sigp/lighthouse:latest", - "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, } toml = build_cb_toml_basic(950, 4000) mev_params = build_mev_params("helix", images, toml) @@ -250,11 +289,9 @@ def generate_multiple_relays(): "# aggregating responses and selecting the best bid." ) images = { - "helix_relay_image": "helix-relay:kurtosis", - "mev_relay_image": "ethpandaops/mev-boost-relay:main", - "mev_boost_image": "commit-boost/pbs:kurtosis", - "mev_builder_cl_image": "sigp/lighthouse:latest", - "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_relay_image": MEV_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, } toml = build_cb_toml_basic(950, 4000) mev_params = build_mev_params(["helix", "flashbots"], images, toml) @@ -277,10 +314,8 @@ def generate_skip_sigverify(): "# reachable under load." ) images = { - "helix_relay_image": "helix-relay:kurtosis", - "mev_boost_image": "commit-boost/pbs:kurtosis", - "mev_builder_cl_image": "sigp/lighthouse:latest", - "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, } toml = build_cb_toml_basic(950, 4000, extra_pbs_lines=["skip_sigverify = true"]) mev_params = build_mev_params("helix", images, toml) @@ -303,11 +338,9 @@ def generate_timing_games(): "# in the slot. Per-relay timing overrides are enabled for all relays." ) images = { - "helix_relay_image": "helix-relay:kurtosis", - "mev_relay_image": "ethpandaops/mev-boost-relay:main", - "mev_boost_image": "commit-boost/pbs:kurtosis", - "mev_builder_cl_image": "sigp/lighthouse:latest", - "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_relay_image": MEV_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, } toml = build_cb_toml_basic( 400, @@ -338,10 +371,8 @@ def generate_extra_validation(): "# parameters before returning a header to the beacon node." ) images = { - "helix_relay_image": "helix-relay:kurtosis", - "mev_boost_image": "commit-boost/pbs:kurtosis", - "mev_builder_cl_image": "sigp/lighthouse:latest", - "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, } toml = build_cb_toml_basic( 950, @@ -372,11 +403,9 @@ def generate_mux(pubkeys_node0, pubkeys_node1): "# per-mux timeout and relay configurations." ) images = { - "helix_relay_image": "helix-relay:kurtosis", - "mev_relay_image": "ethpandaops/mev-boost-relay:main", - "mev_boost_image": "commit-boost/pbs:kurtosis", - "mev_builder_cl_image": "sigp/lighthouse:latest", - "mev_builder_image": "ethpandaops/reth-rbuilder:develop", + "helix_relay_image": HELIX_RELAY_IMAGE, + "mev_relay_image": MEV_RELAY_IMAGE, + "mev_boost_image": MEV_BOOST_IMAGE, } toml = build_cb_toml_mux(pubkeys_node0, pubkeys_node1) mev_params = build_mev_params(["helix", "flashbots"], images, toml) diff --git a/scripts/run-and-verify.sh b/scripts/run-and-verify.sh index be471c0..58f32c1 100755 --- a/scripts/run-and-verify.sh +++ b/scripts/run-and-verify.sh @@ -12,19 +12,20 @@ set -euo pipefail # Defaults +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$(dirname "$SCRIPT_DIR")" ENCLAVE="CB-Testnet" CONFIG="" -PACKAGE="github.com/ethpandaops/ethereum-package" +PACKAGE="$REPO_DIR/ethereum-package" KEEP=false JSON_FLAG="" +JSON_DIR_FLAG="" STRICT_FLAG="" LIVE_METRICS_FLAG="" TIMEOUT=3600 MIN_EPOCHS=2 TARGET_EPOCH=7 VERBOSE="" -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_DIR="$(dirname "$SCRIPT_DIR")" usage() { echo "Usage: $0 [OPTIONS]" @@ -32,9 +33,10 @@ usage() { echo "Options:" echo " --config FILE Kurtosis config file (default: configs/basic-pbs.yml)" echo " --enclave NAME Enclave name (default: CB-Testnet)" - echo " --package PATH ethereum-package path or ref (default: ethpandaops/ethereum-package)" + echo " --package PATH ethereum-package path or ref (default: ./ethereum-package)" echo " --keep Don't tear down the enclave on exit" echo " --json Output JSON report" + echo " --json-dir DIR Save JSON report to DIR/{enclave}.json (implies --json)" echo " --strict Promote WARN to FAIL (zero bids, zero deliveries)" echo " --live-metrics Show counter deltas every 30s during observation" echo " --timeout SECS Readiness timeout (default: 1500)" @@ -52,6 +54,7 @@ while [[ $# -gt 0 ]]; do --package) PACKAGE="$2"; shift 2;; --keep) KEEP=true; shift;; --json) JSON_FLAG="--json"; shift;; + --json-dir) JSON_DIR_FLAG="--output-dir $2"; JSON_FLAG="--json"; shift 2;; --strict) STRICT_FLAG="--strict"; shift;; --live-metrics) LIVE_METRICS_FLAG="--live-metrics"; shift;; --timeout) TIMEOUT="$2"; shift 2;; @@ -68,6 +71,11 @@ if [[ -z "$CONFIG" ]]; then CONFIG="$REPO_DIR/configs/basic-pbs.yml" fi +# Default --json implies auto-save to repo root +if [[ -n "$JSON_FLAG" && -z "$JSON_DIR_FLAG" ]]; then + JSON_DIR_FLAG="--output-dir $REPO_DIR" +fi + # Resolve config path relative to CWD if [[ ! -f "$CONFIG" ]]; then echo "Config file not found: $CONFIG" @@ -112,10 +120,12 @@ echo "" # Step 3: Run verification cargo run --manifest-path "$REPO_DIR/Cargo.toml" --release -- \ --enclave "$ENCLAVE" \ + --cb-config "$CONFIG" \ --timeout "$TIMEOUT" \ --min-epochs "$MIN_EPOCHS" \ --target-epoch "$TARGET_EPOCH" \ $JSON_FLAG \ + $JSON_DIR_FLAG \ $STRICT_FLAG \ $LIVE_METRICS_FLAG \ $VERBOSE diff --git a/src/checks/mod.rs b/src/checks/mod.rs index e8e04be..b08bd54 100644 --- a/src/checks/mod.rs +++ b/src/checks/mod.rs @@ -87,5 +87,6 @@ impl CheckResult { pub mod cb_metrics; pub mod chain_health; +pub mod mux_routing; pub mod payload_matching; pub mod relay_pipeline; diff --git a/src/checks/mux_routing.rs b/src/checks/mux_routing.rs new file mode 100644 index 0000000..f02a0ce --- /dev/null +++ b/src/checks/mux_routing.rs @@ -0,0 +1,610 @@ +//! MUX routing verification check. +//! +//! Extracts `[[mux]]` sections from a Commit-Boost config TOML file. +//! If the config contains mux rules, automatically verifies that every +//! delivered payload conforms — zero cross-contamination between relays. +//! If no mux sections are found, the check is skipped. + +use std::collections::{HashMap, HashSet}; + +use crate::checks::{CheckResult, CheckStatus}; +use crate::relay::RelayClient; + +/// A single parsed mux entry with the relay index resolved from the template. +pub struct MuxEntry { + pub id: String, + pub relay_index: usize, + pub validator_pubkeys: Vec, +} + +/// Parse a Commit-Boost config file and extract mux routing entries. +/// +/// Supports two file formats: +/// - `.toml` — raw Commit-Boost config (possibly with Go template expressions) +/// - `.yml`/`.yaml` — Kurtosis YAML with `mev_params.commit_boost_config` field +/// +/// Returns `Ok(Some(entries))` if mux sections were found, +/// `Ok(None)` if no mux sections (check will SKIP), +/// `Err` if parsing fails. +pub fn extract_mux_from_config(path: &str) -> eyre::Result>> { + let raw = std::fs::read_to_string(path) + .map_err(|e| eyre::eyre!("Failed to read config '{path}': {e}"))?; + + let template = if path.ends_with(".toml") { + raw + } else if path.ends_with(".yml") || path.ends_with(".yaml") { + extract_commit_boost_config_from_yaml(&raw)? + } else { + return Err(eyre::eyre!( + "Unrecognized config format. Expected .toml (CB config) or .yml/.yaml (Kurtosis config), got: {path}" + )); + }; + + parse_mux_from_toml_template(&template) +} + +/// Extract the `commit_boost_config` field from a Kurtosis YAML config. +fn extract_commit_boost_config_from_yaml(raw: &str) -> eyre::Result { + use serde_yaml::Value as YamlValue; + + let parsed: YamlValue = serde_yaml::from_str(raw) + .map_err(|e| eyre::eyre!("Failed to parse Kurtosis YAML config: {e}"))?; + + let template = parsed + .get("mev_params") + .and_then(|p| p.get("commit_boost_config")) + .and_then(|c| c.as_str()) + .ok_or_else(|| { + eyre::eyre!( + "No mev_params.commit_boost_config found in Kurtosis YAML config" + ) + })?; + + Ok(template.to_string()) +} + +/// Quick inspection: does the raw text look like a TOML with `[[mux]]`? +fn has_mux_sections(text: &str) -> bool { + text.lines().any(|l| l.trim() == "[[mux]]" || l.trim() == "[[mux]]") +} + +/// Parse `[[mux]]` sections from a Commit-Boost config TOML template. +/// +/// The template may contain Go template expressions like `{{ index .Relays 0 }}` +/// in string values. This works around that by doing a line-by-line parse +/// rather than using a TOML parser (which would choke on template syntax). +fn parse_mux_from_toml_template(template: &str) -> eyre::Result>> { + if !has_mux_sections(template) { + return Ok(None); + } + + let mut entries: Vec = Vec::new(); + let mut lines = template.lines().peekable(); + + while let Some(line) = lines.next() { + let trimmed = line.trim(); + + // Look for [[mux]] section start (NOT [[mux.relays]] sub-sections) + if trimmed != "[[mux]]" { + continue; + } + + // Parse this mux section + let entry = match parse_one_mux_section(&mut lines) { + Ok(Some(e)) => e, + Ok(None) => continue, // parse_one_mux_section advanced but returned nothing + Err(e) => return Err(eyre::eyre!("Failed to parse mux section: {e}")), + }; + + entries.push(entry); + } + + if entries.is_empty() { + return Ok(None); + } + + Ok(Some(entries)) +} + +/// Parse a single `[[mux]]` section (until next `[[` header or EOF). +/// +/// Uses `peek()` to avoid consuming the next `[[mux]]` header — the caller's +/// main loop needs to see that line to start the next iteration. +fn parse_one_mux_section<'a>( + lines: &mut std::iter::Peekable>, +) -> eyre::Result> { + let mut id: Option = None; + let mut pubkeys: Option> = None; + let mut relay_index: Option = None; + + loop { + // Peek at what's next without consuming it + let is_section_header = lines + .peek() + .map(|l| l.trim().starts_with("[[")) + .unwrap_or(false); + + if is_section_header { + let header = lines.peek().unwrap().trim().to_string(); + if header.starts_with("[[mux.relays]]") { + // Consume the [[mux.relays]] header and parse its body + let _ = lines.next(); + let idx = parse_mux_relay_body(lines)?; + if relay_index.is_none() { + relay_index = idx; + } + continue; + } + // Any other [[ section — stop without consuming. + // The caller's main loop will pick up the header. + break; + } + + let Some(line) = lines.next() else { + break; + }; + let trimmed = line.trim(); + + // Skip comments and blank lines + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + // key = "value" + if let Some((key, raw_val)) = parse_key_value(trimmed) { + match key { + "id" => { + id = Some(raw_val.trim_matches('"').to_string()); + } + "validator_pubkeys" => { + pubkeys = Some(parse_pubkey_array(raw_val, lines)?); + } + _ => {} // Ignore other keys + } + } + } + + let id = id.ok_or_else(|| eyre::eyre!("[[mux]] section missing 'id' field"))?; + let pubkeys = pubkeys + .ok_or_else(|| eyre::eyre!("[[mux]] section '{id}' missing 'validator_pubkeys'"))?; + let relay_index = relay_index.ok_or_else(|| { + eyre::eyre!("[[mux]] section '{id}' missing [[mux.relays]] with url template") + })?; + + Ok(Some(MuxEntry { + id, + relay_index, + validator_pubkeys: pubkeys, + })) +} + +/// Parse the body of a [[mux.relays]] sub-section (after its header was +/// consumed) to extract the relay index from `url = "{{ index .Relays N }}"`. +/// +/// Stops at the next `[[` section header without consuming it. +fn parse_mux_relay_body( + lines: &mut std::iter::Peekable, +) -> eyre::Result> { + loop { + // Stop at next section header without consuming + if lines + .peek() + .map(|l| l.trim().starts_with("[[")) + .unwrap_or(false) + { + break; + } + + let Some(line) = lines.next() else { + break; + }; + let trimmed = line.trim(); + + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + if let Some((key, raw_val)) = parse_key_value(trimmed) { + if key == "url" { + let val = raw_val.trim_matches('"'); + if let Some(idx) = parse_relay_index_from_template(val) { + return Ok(Some(idx)); + } + return Ok(None); + } + } + } + + Ok(None) +} + +/// Extract the relay index from a Go template expression like `{{ index .Relays 0 }}`. +fn parse_relay_index_from_template(val: &str) -> Option { + let val = val.trim(); + // Match: {{ index .Relays N }} or {{ index .Relays N}} + // The N is a decimal integer + let stripped = val + .trim_start_matches("{{") + .trim_end_matches("}}") + .trim(); + // Now stripped should be something like: index .Relays 0 + let parts: Vec<&str> = stripped.split_whitespace().collect(); + // Expected: ["index", ".Relays", "N"] + if parts.len() >= 3 && parts[0] == "index" && parts[1] == ".Relays" { + parts[2].parse::().ok() + } else { + None + } +} + +/// Parse a simple key = "value" pair. +fn parse_key_value(s: &str) -> Option<(&str, &str)> { + let eq_pos = s.find('=')?; + let key = s[..eq_pos].trim(); + let raw_val = s[eq_pos + 1..].trim(); + Some((key, raw_val)) +} + +/// Parse a TOML-style array literal: `["0x...", "0x...", ...]` +/// +/// Handles both single-line and multi-line formats. +/// `rest` is what follows the `=` sign. If it starts with `[`, the array +/// is inline. If not, the array starts on the next line(s). +fn parse_pubkey_array( + rest: &str, + lines: &mut std::iter::Peekable, +) -> eyre::Result> { + let mut accum = rest.to_string(); + + // Collect lines until we find the closing `]`. + // Works for both single-line (`rest` already contains `]`) and + // multi-line (rest is just `[`, or rest is the start up to `= [`). + if !accum.trim_end().ends_with(']') { + loop { + let Some(next) = lines.next() else { + break; + }; + accum.push('\n'); + accum.push_str(next); + if next.trim().ends_with(']') { + break; + } + } + } + + // Now parse the array from accum + let raw = accum.trim(); + + // Find the opening and closing brackets + let start = raw.find('[').ok_or_else(|| { + eyre::eyre!("Could not find opening '[' in pubkey array: {raw:.50}...") + })?; + let end = raw.rfind(']').ok_or_else(|| { + eyre::eyre!("Could not find closing ']' in pubkey array: {raw:.50}...") + })?; + + let inner = &raw[start + 1..end]; + + let mut pubkeys = Vec::new(); + for item in inner.split(',') { + let item = item.trim().trim_matches('"').trim(); + if item.is_empty() { + continue; + } + // Strip Go template expressions if any leaked in + if item.contains("{{") || item.contains("}}") { + continue; + } + pubkeys.push(item.to_string()); + } + + Ok(pubkeys) +} + +/// Normalize a public key string: strip `0x` prefix, lowercase. +fn normalize_pubkey(pk: &str) -> String { + pk.trim_start_matches("0x") + .trim_start_matches("0X") + .to_lowercase() +} + +/// Run the mux routing check against the relay data. +/// +/// Returns SKIP if entries is empty (no mux config). +/// Returns PASS/WARN/FAIL based on cross-contamination findings. +pub async fn check_mux_routing( + relays: &[RelayClient], + entries: &[MuxEntry], + start_slot: u64, + end_slot: u64, +) -> CheckResult { + if entries.is_empty() { + return CheckResult::skip( + "mux.routing", + 1, + "No [[mux]] sections in CB config — nothing to verify", + ); + } + + if entries.len() > relays.len() { + return CheckResult::fail( + "mux.routing", + 1, + format!( + "Mux config has {} entries but only {} relay(s) discovered", + entries.len(), + relays.len() + ), + ); + } + + // Validate relay indices are in range + for entry in entries { + if entry.relay_index >= relays.len() { + return CheckResult::fail( + "mux.routing", + 1, + format!( + "Mux '{}' references relay_index {} but only {} relay(s) discovered", + entry.id, + entry.relay_index, + relays.len() + ), + ); + } + } + + // Build: normalized_pubkey -> expected_relay_index + let mut expected_relay: HashMap = HashMap::new(); + for entry in entries { + for pk in &entry.validator_pubkeys { + let nk = normalize_pubkey(pk); + expected_relay.insert(nk, entry.relay_index); + } + } + let total_mapped_pubkeys = expected_relay.len(); + + // Query each relay's delivered payloads in the observation window + let mut violations: Vec = Vec::new(); + let mut seen_pubkeys: HashMap> = HashMap::new(); + + for (relay_idx, relay) in relays.iter().enumerate() { + let Ok(entries) = relay.get_payloads_delivered(start_slot, end_slot).await else { + tracing::warn!( + "mux check: relay[{relay_idx}] {} payload query failed", + relay.base_url() + ); + continue; + }; + + tracing::info!( + "mux check: relay[{relay_idx}] {} returned {} delivered payload(s)", + relay.base_url(), + entries.len() + ); + + for payload in &entries { + let pk_hex = payload.proposer_pubkey.to_string(); + let pk_norm = normalize_pubkey(&pk_hex); + + if let Some(&expected_idx) = expected_relay.get(&pk_norm) { + if expected_idx != relay_idx { + violations.push(serde_json::json!({ + "slot": payload.slot, + "proposer_pubkey": format!("0x{pk_norm}"), + "relay_actual": relay_idx, + "relay_expected": expected_idx, + })); + } + + seen_pubkeys + .entry(pk_norm) + .or_default() + .insert(payload.slot); + } + } + } + + // Mapped pubkeys that never appeared in any relay's deliveries + let mut unseen: Vec = Vec::new(); + for pk_norm in expected_relay.keys() { + if !seen_pubkeys.contains_key(pk_norm) { + unseen.push(format!("0x{pk_norm}")); + } + } + + let data = serde_json::json!({ + "total_mapped_pubkeys": total_mapped_pubkeys, + "violations": violations, + "violation_count": violations.len(), + "unseen_pubkeys": unseen, + "unseen_count": unseen.len(), + "total_deliveries_checked": seen_pubkeys.values().map(|s| s.len()).sum::(), + }); + + let mux_ids: Vec<&str> = entries.iter().map(|e| e.id.as_str()).collect(); + let mux_detail = format!("muxes=[{}]", mux_ids.join(", ")); + + if !violations.is_empty() { + CheckResult::fail( + "mux.routing", + 1, + format!( + "{} mux routing violation(s): pubkey(s) delivered to the wrong relay. {}", + violations.len(), + mux_detail, + ), + ) + .with_data(data) + } else if !unseen.is_empty() { + CheckResult::warn( + "mux.routing", + 1, + format!( + "No routing violations ✓. {} mux-mapped pubkey(s) had zero deliveries \ + in the observation window (they may not have proposed). {}", + unseen.len(), + mux_detail, + ), + ) + .with_data(data) + } else { + CheckResult::pass( + "mux.routing", + 1, + format!( + "All {} mux-mapped pubkey(s) delivered exclusively to their assigned relays. \ + Zero routing violations. {}", + total_mapped_pubkeys, + mux_detail, + ), + ) + .with_data(data) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_relay_index_from_template() { + assert_eq!( + parse_relay_index_from_template("{{ index .Relays 0 }}"), + Some(0) + ); + assert_eq!( + parse_relay_index_from_template("{{ index .Relays 1 }}"), + Some(1) + ); + assert_eq!( + parse_relay_index_from_template("{{ index .Relays 42 }}"), + Some(42) + ); + assert_eq!( + parse_relay_index_from_template("{{ index .Relays 0}}"), + Some(0) + ); + // Not a template + assert_eq!( + parse_relay_index_from_template("http://relay:18550"), + None + ); + // Range variable reference + assert_eq!(parse_relay_index_from_template("{{ $relay }}"), None); + } + + #[test] + fn test_has_mux_sections() { + assert!(has_mux_sections("[[mux]]\nid = 'foo'")); + assert!(!has_mux_sections("[[relays]]\nid = 'foo'")); + } + + #[test] + fn test_normalize_pubkey() { + assert_eq!( + normalize_pubkey("0xABC123"), + "abc123" + ); + assert_eq!( + normalize_pubkey("abc123"), + "abc123" + ); + assert_eq!( + normalize_pubkey("0XABC123"), + "abc123" + ); + } + + #[test] + fn test_parse_pubkey_array_single_line() { + let rest = "[\"0xabc\", \"0xdef\"]"; + let mut empty = "".lines().peekable(); + let keys = parse_pubkey_array(rest, &mut empty).unwrap(); + assert_eq!(keys, vec!["0xabc", "0xdef"]); + } + + #[test] + fn test_parse_pubkey_array_multi_line() { + let rest = "["; + let input = " \"0xabc\",\n \"0xdef\",\n]"; + let mut lines = input.lines().peekable(); + let keys = parse_pubkey_array(rest, &mut lines).unwrap(); + assert_eq!(keys, vec!["0xabc", "0xdef"]); + } + + #[test] + fn test_parse_mux_from_toml_template() { + let template = r#" +[[relays]] +id = "mev_relay_0" +url = "{{ $relay }}" + +[[mux]] +id = "node_0_to_helix" +validator_pubkeys = [ + "0xaaf6c1251e73fb600624937760fef218aace5b253bf068ed45398aeb29d821e4d2899343ddcbbe37cb3f6cf500dff26c", + "0x8aa5bbee21e98c7b9e7a4c8ea45aa99f89e22992fa4fc2d73869d77da4cc8a05b25b61931ff521986677dd7f7159e8e6", +] +timeout_get_header_ms = 900 +[[mux.relays]] +id = "mux_helix" +url = "{{ index .Relays 0 }}" + +[[mux]] +id = "node_1_to_flashbots" +validator_pubkeys = [ + "0xb05cafec5912f22dbd6f15677f25f13d93ecd5ec6f957fddd7cf27d73521b34aaaf6a219f77b21128d18321c2c8d679b", +] +[[mux.relays]] +id = "mux_flashbots" +url = "{{ index .Relays 1 }}" +"#; + + let result = parse_mux_from_toml_template(template).unwrap(); + assert!(result.is_some()); + let entries = result.unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].id, "node_0_to_helix"); + assert_eq!(entries[0].relay_index, 0); + assert_eq!(entries[0].validator_pubkeys.len(), 2); + assert_eq!(entries[1].id, "node_1_to_flashbots"); + assert_eq!(entries[1].relay_index, 1); + assert_eq!(entries[1].validator_pubkeys.len(), 1); + } + + #[test] + fn test_no_mux_sections_returns_none() { + let template = r#" +[[relays]] +id = "mev_relay_0" +url = "{{ $relay }}" + +[logs.stdout] +level = "debug" +"#; + let result = parse_mux_from_toml_template(template).unwrap(); + assert!(result.is_none()); + } + + #[test] + fn test_extract_commit_boost_config_from_yaml() { + let yaml = r#" +mev_type: custom +mev_params: + commit_boost_config: | + [[mux]] + id = "test" + validator_pubkeys = [ + "0xabc", + ] + [[mux.relays]] + url = "{{ index .Relays 0 }}" +additional_services: + - dora +"#; + let template = extract_commit_boost_config_from_yaml(yaml).unwrap(); + assert!(template.contains("[[mux]]")); + assert!(template.contains("0xabc")); + } +} diff --git a/src/checks/relay_pipeline.rs b/src/checks/relay_pipeline.rs index c2fd079..e10e08c 100644 --- a/src/checks/relay_pipeline.rs +++ b/src/checks/relay_pipeline.rs @@ -320,16 +320,90 @@ pub async fn run_relay_checks( .map(|r| RelayClient::new(r.base_url())) .collect(); - if let Some(first_relay) = live.first() { - results.push(check_builder_blocks_received(first_relay, start_slot, end_slot).await); + // Check builder blocks received across ALL live relays. + // Aggregated: PASS if any relay received blocks, FAIL only if ALL relays got nothing. + { + let mut bb_results: Vec = Vec::new(); + for relay in &live { + bb_results.push(check_builder_blocks_received(relay, start_slot, end_slot).await); + } + let any_pass = bb_results.iter().any(|r| r.status == CheckStatus::Pass); + if any_pass { + let total: usize = bb_results + .iter() + .map(|r| r.data.get("count").and_then(|c| c.as_u64()).unwrap_or(0) as usize) + .sum(); + let details: Vec<&str> = bb_results.iter().map(|r| r.detail.as_str()).collect(); + results.push( + CheckResult::pass( + "relay.builder_blocks_received", + 2, + format!("{}", details.join("; ")), + ) + .with_data(serde_json::json!({"count": total})), + ); + } else { + let worst = bb_results.into_iter().max_by_key(|r| match r.status { + CheckStatus::Fail => 2, + CheckStatus::Warn => 1, + _ => 0, + }).unwrap(); + results.push(worst); + } } results.push(check_payloads_delivered_multi(&live, start_slot, end_slot).await); - if let Some(first_relay) = live.first() { - results.push( - check_mev_delivery_rate(first_relay, beacon, start_slot, end_slot, mev_threshold).await, - ); + // Check MEV delivery rate across ALL live relays. + // Aggregated: best-of status; reports combined delivery stats. + { + let mut mv_results: Vec = Vec::new(); + for relay in &live { + mv_results.push( + check_mev_delivery_rate(relay, beacon, start_slot, end_slot, mev_threshold).await, + ); + } + let any_pass = mv_results.iter().any(|r| r.status == CheckStatus::Pass); + let best_status = if any_pass { + CheckStatus::Pass + } else if mv_results.iter().any(|r| r.status == CheckStatus::Warn) { + CheckStatus::Warn + } else { + CheckStatus::Fail + }; + let total_mev: u64 = mv_results + .iter() + .map(|r| r.data.get("mev_blocks").and_then(|c| c.as_u64()).unwrap_or(0)) + .sum(); + let total_blocks: u64 = mv_results + .iter() + .map(|r| r.data.get("total_blocks").and_then(|c| c.as_u64()).unwrap_or(0)) + .sum(); + let details: Vec<&str> = mv_results.iter().map(|r| r.detail.as_str()).collect(); + let data = serde_json::json!({ + "mev_blocks": total_mev, + "total_blocks": total_blocks, + "rate": if total_blocks > 0 { + (total_mev as f64 / total_blocks as f64 * 10000.0).round() / 10000.0 + } else { 0.0 }, + }); + results.push(match best_status { + CheckStatus::Pass => CheckResult::pass( + "relay.mev_delivery_rate", + 2, + format!("MEV delivery rate across all relays: {}", details.join("; ")), + ), + CheckStatus::Warn => CheckResult::warn( + "relay.mev_delivery_rate", + 2, + format!("MEV delivery rate below threshold: {}", details.join("; ")), + ), + _ => CheckResult::fail( + "relay.mev_delivery_rate", + 2, + format!("No MEV deliveries across any relay: {}", details.join("; ")), + ), + }.with_data(data)); } // Tier 3: per-relay validator registration, aggregated to the worst status. diff --git a/src/discovery.rs b/src/discovery.rs index d836e3d..eb92bab 100644 --- a/src/discovery.rs +++ b/src/discovery.rs @@ -298,16 +298,23 @@ pub fn discover(enclave: &str) -> Result { } } - // Relay Data API: mev-relay-*-api or mev-relay-api - if matches_pattern(&svc.name, "mev-relay-*") && svc.name.ends_with("-api") - || svc.name == "mev-relay-api" - { - let url = port_print(enclave, &svc.name, "http").or_else(|| find_port("http")); + // Relay Data API: match any relay service by name heuristics. + // + // Different relay implementations use different service names and port IDs: + // flashbots: "mev-relay-api" — port "http" (9067) + // helix: "helix-relay" — port "endpoint" (4040) + // mev-rs: "mev-rs-relay" — port "http" (28545) + // Exclude supporting services (postgres, redis, website, housekeeper). + if is_relay_api_service(&svc.name) { + let url = port_print(enclave, &svc.name, "http") + .or_else(|| port_print(enclave, &svc.name, "endpoint")) + .or_else(|| find_port("http")) + .or_else(|| find_port("endpoint")); if let Some(url) = url { info!("Relay API: {} -> {url}", svc.name); result.relay_urls.push(url); } else { - warn!("Relay '{}': no http port", svc.name); + warn!("Relay '{}': no http/endpoint port", svc.name); } } @@ -499,10 +506,47 @@ fn parse_postmortem_output(output: &str) -> Vec { records } +/// Heuristic check: is this service name a relay API endpoint? +/// +/// Returns true if the name contains "relay" and does not match known +/// non-API relay services (postgres, redis, website, housekeeper). +/// This covers flashbots ("mev-relay-api"), helix ("helix-relay"), +/// mev-rs ("mev-rs-relay"), and any future relay implementations. +fn is_relay_api_service(name: &str) -> bool { + let lower = name.to_lowercase(); + let known_non_api = ["-postgres", "-redis", "-website", "-housekeeper"]; + let is_relay = lower.contains("relay"); + let is_non_api = known_non_api.iter().any(|suffix| lower.ends_with(suffix)); + is_relay && !is_non_api +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn test_is_relay_api_service() { + // Relay API services — should match + assert!(is_relay_api_service("mev-relay-api")); + assert!(is_relay_api_service("helix-relay")); + assert!(is_relay_api_service("mev-rs-relay")); + assert!(is_relay_api_service("mev-relay-0-api")); + assert!(is_relay_api_service("Helix-Relay")); // case-insensitive + + // Non-API supporting services — should not match + assert!(!is_relay_api_service("mev-relay-postgres")); + assert!(!is_relay_api_service("mev-relay-redis")); + assert!(!is_relay_api_service("mev-relay-website")); + assert!(!is_relay_api_service("mev-relay-housekeeper")); + assert!(!is_relay_api_service("helix-relay-postgres")); + + // Unrelated services — should not match + assert!(!is_relay_api_service("cl-1-lighthouse-geth")); + assert!(!is_relay_api_service("prometheus")); + assert!(!is_relay_api_service("dora")); + assert!(!is_relay_api_service("commit-boost-001")); + } + #[test] fn test_matches_pattern() { assert!(matches_pattern("cl-1-lighthouse-geth", "cl-*")); diff --git a/src/main.rs b/src/main.rs index 3cc1fac..d296e8f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -79,6 +79,29 @@ struct Cli { /// directly accessible. #[arg(long)] live_metrics: bool, + + /// Path to the Commit-Boost config file (TOML or Kurtosis YAML). + /// + /// When set, parses the config for [[mux]] sections and automatically + /// verifies that all delivered payloads conform to the mux routing + /// rules: validator pubkeys are delivered exclusively to their assigned + /// relay. Fails if any pubkey appears on the wrong relay. + /// + /// If no [[mux]] sections are found in the config, the check is skipped. + /// Supports .toml (raw CB config) and .yml/.yaml (Kurtosis config with + /// embedded mev_params.commit_boost_config). + #[arg(long)] + cb_config: Option, + + /// Directory to save JSON report files. Requires --json. + /// + /// When set, writes `{enclave}.json` into this directory after the report + /// is printed to stdout. Useful for batch runs (e.g., `just test-all`) + /// where each config variant produces its own JSON file. + /// + /// The directory must exist before the run starts. + #[arg(long)] + output_dir: Option, } #[tokio::main] @@ -104,6 +127,13 @@ async fn main() -> Result<()> { async fn run_verification(cli: &Cli) -> i32 { let now = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + // Helper: save JSON report to file if --output-dir was set. + let save_report = |report: &VerificationReport| { + if let Some(ref dir) = cli.output_dir { + report::save_json_report(report, dir); + } + }; + // Step 1: Discover services info!("Discovering services in enclave '{}'...", cli.enclave); let services = match discovery::discover(&cli.enclave) { @@ -112,6 +142,7 @@ async fn run_verification(cli: &Cli) -> i32 { error!("Service discovery failed: {e}"); let report = make_error_report(&cli.enclave, &now, &format!("Discovery failed: {e}")); report::print_report(&report, cli.json); + save_report(&report); return 2; } }; @@ -120,6 +151,7 @@ async fn run_verification(cli: &Cli) -> i32 { error!("No beacon nodes found in enclave"); let report = make_error_report(&cli.enclave, &now, "No beacon nodes found"); report::print_report(&report, cli.json); + save_report(&report); return 2; } @@ -143,6 +175,7 @@ async fn run_verification(cli: &Cli) -> i32 { &format!("Devnet did not stabilize within {}s", cli.timeout), ); report::print_report(&report, cli.json); + save_report(&report); return 2; } @@ -238,6 +271,7 @@ async fn run_verification(cli: &Cli) -> i32 { ), ); report::print_report(&report, cli.json); + save_report(&report); return 2; } } else { @@ -260,6 +294,7 @@ async fn run_verification(cli: &Cli) -> i32 { ), ); report::print_report(&report, cli.json); + save_report(&report); return 2; } } @@ -300,12 +335,14 @@ async fn run_verification(cli: &Cli) -> i32 { ), ); report::print_report(&report, cli.json); + save_report(&report); return 2; } ObserveOutcome::Timeout => { let report = make_error_report(&cli.enclave, &now, "Failed to complete observation window"); report::print_report(&report, cli.json); + save_report(&report); return 2; } }; @@ -378,6 +415,40 @@ async fn run_verification(cli: &Cli) -> i32 { .await, ); + // MUX routing check (optional — requires --cb-config) + if let Some(ref cb_path) = cli.cb_config { + info!("Checking for [[mux]] sections in CB config: {cb_path}..."); + match checks::mux_routing::extract_mux_from_config(cb_path) { + Ok(Some(mux_entries)) => { + info!( + "Found {} [[mux]] section(s) — running MUX routing verification", + mux_entries.len() + ); + all_checks.push( + checks::mux_routing::check_mux_routing( + &relays, + &mux_entries, + window.start_slot, + window.end_slot, + ) + .await, + ); + } + Ok(None) => { + info!("No [[mux]] sections found in CB config — skipping MUX check"); + } + Err(e) => { + all_checks.push(CheckResult::fail( + "mux.routing", + 1, + format!("Failed to parse CB config '{cb_path}': {e}"), + )); + } + } + } else { + info!("No --cb-config provided — skipping MUX routing check"); + } + // Step 5: Report let tier1_failed = all_checks .iter() @@ -396,6 +467,7 @@ async fn run_verification(cli: &Cli) -> i32 { }; report::print_report(&report, cli.json); + save_report(&report); report::exit_code(&report) } diff --git a/src/report.rs b/src/report.rs index e6e1639..7290037 100644 --- a/src/report.rs +++ b/src/report.rs @@ -100,6 +100,30 @@ pub fn print_report(report: &VerificationReport, json_mode: bool) { } } +/// Save a JSON report to `{output_dir}/{enclave}.json`. +/// +/// Only writes if the directory exists (caller must create it). +/// Logs a warning on failure but does not return an error — the +/// verification itself has already completed. +pub fn save_json_report(report: &VerificationReport, output_dir: &str) { + let report_path = format!("{}/{}.json", output_dir.trim_end_matches('/'), report.enclave); + match serde_json::to_string_pretty(report) { + Ok(json) => match std::fs::write(&report_path, &json) { + Ok(_) => { + // Intentionally not using tracing here — this module has no + // tracing dep and adding one would be overkill for a single message. + eprintln!("Report saved to {report_path}"); + } + Err(e) => { + eprintln!("Failed to write report to {report_path}: {e}"); + } + }, + Err(e) => { + eprintln!("Failed to serialize report for {report_path}: {e}"); + } + } +} + /// Determine process exit code from the report. /// /// - 0: all tier-1 checks passed From 2ba20bedf1744b31f0a559e3a52a8474901a8872 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 07:56:23 -0700 Subject: [PATCH 05/20] refactor, add mux checks, add pipelined testing --- Cargo.toml | 10 + justfile | 103 ++-- scripts/generate_kurtosis_configs.py | 17 + scripts/run-and-verify.sh | 4 +- src/bin/test_mux.rs | 475 +++++++++++++++ src/checks/chain_health.rs | 12 +- src/checks/mux_routing.rs | 761 +++++++++++++++++------ src/discovery.rs | 38 +- src/main.rs | 309 +++++++--- src/orchestrator.rs | 869 +++++++++++++++++++++++++++ src/relay.rs | 15 +- 11 files changed, 2284 insertions(+), 329 deletions(-) create mode 100644 src/bin/test_mux.rs create mode 100644 src/orchestrator.rs diff --git a/Cargo.toml b/Cargo.toml index 6e83775..645634c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "cb-testnet-verifier" +default-run = "cb-verify" version = "0.1.0" edition = "2024" rust-version = "1.91" @@ -10,6 +11,15 @@ license = "MIT" name = "cb-verify" path = "src/main.rs" +[[bin]] +name = "cb-orchestrator" +path = "src/orchestrator.rs" + +[[bin]] +name = "test-mux" +path = "src/bin/test_mux.rs" + + [dependencies] alloy-primitives = "^1.3.1" alloy-rpc-types-beacon = "^1.0.35" diff --git a/justfile b/justfile index eaea4c8..7efcaa5 100644 --- a/justfile +++ b/justfile @@ -32,9 +32,13 @@ ci: check test lint build-release: cargo build --release +# Build the orchestrator binary (concurrent multi-enclave test runner) +build-orchestrator: + cargo build --release --bin cb-orchestrator + # Run verifier against a running enclave verify enclave="CB-Testnet" target_epoch="7" min_epochs="2": - cargo run --release -- \ + cargo run --release --bin cb-verify -- \ --enclave {{enclave}} \ --target-epoch {{target_epoch}} \ --min-epochs {{min_epochs}} \ @@ -42,7 +46,7 @@ verify enclave="CB-Testnet" target_epoch="7" min_epochs="2": # Run verifier with live metrics and strict mode verify-strict enclave="CB-Testnet" target_epoch="7" min_epochs="2": - cargo run --release -- \ + cargo run --release --bin cb-verify -- \ --enclave {{enclave}} \ --target-epoch {{target_epoch}} \ --min-epochs {{min_epochs}} \ @@ -50,6 +54,31 @@ verify-strict enclave="CB-Testnet" target_epoch="7" min_epochs="2": --live-metrics \ --strict +# Standalone: quick health check (no observation window) +verify-now enclave="CB-Testnet": + cargo run --release --bin cb-verify -- \ + --enclave {{enclave}} \ + --min-epochs 0 \ + --timeout 60 + +# Standalone: verify with config (mux checks + health) +verify-with-config config enclave="CB-Testnet": + cargo run --release --bin cb-verify -- \ + --enclave {{enclave}} \ + --config {{config}} \ + --min-epochs 1 \ + --timeout 300 + +# Show raw CB PBS logs with parsing (for debugging) +show-logs enclave="CB-Testnet": + cargo run --release --bin cb-verify -- \ + --enclave {{enclave}} \ + --show-logs + +# Quick mux routing check (no observation window, just fetch logs and check) +test-mux enclave="CB-Testnet" config="configs/generated/cb-mux.yml": + cargo run --release --bin test-mux -- {{enclave}} {{config}} + # Generate Kurtosis YAML configs from templates into configs/generated/ # Loads optional .env for Docker image overrides (see .env.example). generate-configs: @@ -62,46 +91,38 @@ testnet config: --json \ --live-metrics \ --min-epochs 1 \ - --target-epoch 2 + --target-epoch 2 \ + --keep \ + -v -# Run all generated configs sequentially and print a summary. +# Run a single config with verbose logging (for debugging) +testnet-verbose config: + ./scripts/run-and-verify.sh \ + --config {{config}} \ + --json \ + --live-metrics \ + --min-epochs 1 \ + --target-epoch 2 \ + --keep \ + -v + +# Run all generated configs concurrently and print a summary. # -# Iterates configs/generated/*.yml, runs each via `testnet` with -# --json-dir so results are saved. After all complete, aggregates -# the JSON reports into a summary table. -test-all: +# Uses cb-orchestrator to run multiple enclaves in parallel (bounded by --jobs). +# Each config gets its own enclave. While one is observing, others can launch. +# For N configs with --jobs=4, expect roughly 4× throughput vs sequential. +# +# Usage: +# just test-all # default: 2 jobs, no results dir +# just test-all 4 /tmp/results # 4 jobs, save results to /tmp/results +# just test-all 2 /tmp/results --strict --keep +test-all jobs="2": #!/usr/bin/env bash set -euo pipefail - RESULTS_DIR="$(mktemp -d)" - SCRIPTS_DIR="$(dirname "$0")/scripts" - CONFIGS_DIR="$(dirname "$0")/configs/generated" - echo "Results dir: $RESULTS_DIR" - echo "" - for config in "$CONFIGS_DIR"/*.yml; do - name="$(basename "$config" .yml)" - # Derive a unique enclave name from the config filename - enclave="CB-${name#cb-}" - echo "[$(date +%H:%M:%S)] Running $name..." - "$SCRIPTS_DIR/run-and-verify.sh" \ - --config "$config" \ - --enclave "$enclave" \ - --json-dir "$RESULTS_DIR" \ - --live-metrics \ - --min-epochs 2 \ - --target-epoch 2 \ - 2>&1 | tail -1 || echo " $name: FAILED (exit code $?)" - done - echo "" - echo "========================================" - echo " Batch Summary" - echo "========================================" - for f in "$RESULTS_DIR"/*.json; do - [ -f "$f" ] || continue - name="$(basename "$f" .json)" - result="$(jq -r '.result // "unknown"' "$f" 2>/dev/null || echo "parse-error")" - passed="$(jq '[.checks[] | select(.status == "Pass")] | length' "$f" 2>/dev/null || echo "?")" - failed="$(jq '[.checks[] | select(.status == "Fail")] | length' "$f" 2>/dev/null || echo "?")" - warn="$(jq '[.checks[] | select(.status == "Warn")] | length' "$f" 2>/dev/null || echo "?")" - echo " $name: $result (${passed}p / ${failed}f / ${warn}w)" - done - echo "========================================" + cargo run --release --bin cb-orchestrator -- --jobs {{jobs}} + +# Run a single config through the orchestrator (for debugging) +test-one config jobs="1": + cargo run --release --bin cb-orchestrator -- \ + --jobs {{jobs}} \ + {{config}} diff --git a/scripts/generate_kurtosis_configs.py b/scripts/generate_kurtosis_configs.py index adf056d..4225a31 100644 --- a/scripts/generate_kurtosis_configs.py +++ b/scripts/generate_kurtosis_configs.py @@ -266,6 +266,8 @@ def generate_basic(): images = { "helix_relay_image": HELIX_RELAY_IMAGE, "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, } toml = build_cb_toml_basic(950, 4000) mev_params = build_mev_params("helix", images, toml) @@ -292,6 +294,8 @@ def generate_multiple_relays(): "helix_relay_image": HELIX_RELAY_IMAGE, "mev_relay_image": MEV_RELAY_IMAGE, "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, } toml = build_cb_toml_basic(950, 4000) mev_params = build_mev_params(["helix", "flashbots"], images, toml) @@ -302,6 +306,7 @@ def generate_multiple_relays(): "mev_type: custom", mev_params, COMMON_NETWORK_PARAMS, + ASSERTOOR_PARAMS, ]) + "\n" @@ -316,6 +321,8 @@ def generate_skip_sigverify(): images = { "helix_relay_image": HELIX_RELAY_IMAGE, "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, } toml = build_cb_toml_basic(950, 4000, extra_pbs_lines=["skip_sigverify = true"]) mev_params = build_mev_params("helix", images, toml) @@ -326,6 +333,7 @@ def generate_skip_sigverify(): "mev_type: custom", mev_params, COMMON_NETWORK_PARAMS, + ASSERTOOR_PARAMS, ]) + "\n" @@ -341,6 +349,8 @@ def generate_timing_games(): "helix_relay_image": HELIX_RELAY_IMAGE, "mev_relay_image": MEV_RELAY_IMAGE, "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, } toml = build_cb_toml_basic( 400, @@ -359,6 +369,7 @@ def generate_timing_games(): "mev_type: custom", mev_params, COMMON_NETWORK_PARAMS, + ASSERTOOR_PARAMS, ]) + "\n" @@ -373,6 +384,8 @@ def generate_extra_validation(): images = { "helix_relay_image": HELIX_RELAY_IMAGE, "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, } toml = build_cb_toml_basic( 950, @@ -390,6 +403,7 @@ def generate_extra_validation(): "mev_type: custom", mev_params, COMMON_NETWORK_PARAMS, + ASSERTOOR_PARAMS, ]) + "\n" @@ -406,6 +420,8 @@ def generate_mux(pubkeys_node0, pubkeys_node1): "helix_relay_image": HELIX_RELAY_IMAGE, "mev_relay_image": MEV_RELAY_IMAGE, "mev_boost_image": MEV_BOOST_IMAGE, + "mev_builder_image": BUILDER_EL_IMAGE, + "mev_builder_cl_image": BUILDER_CL_IMAGE, } toml = build_cb_toml_mux(pubkeys_node0, pubkeys_node1) mev_params = build_mev_params(["helix", "flashbots"], images, toml) @@ -416,6 +432,7 @@ def generate_mux(pubkeys_node0, pubkeys_node1): "mev_type: custom", mev_params, MUX_NETWORK_PARAMS, + ASSERTOOR_PARAMS, ]) + "\n" diff --git a/scripts/run-and-verify.sh b/scripts/run-and-verify.sh index 58f32c1..1b459b4 100755 --- a/scripts/run-and-verify.sh +++ b/scripts/run-and-verify.sh @@ -118,9 +118,9 @@ echo "Enclave '$ENCLAVE' is up. Starting verification..." echo "" # Step 3: Run verification -cargo run --manifest-path "$REPO_DIR/Cargo.toml" --release -- \ +cargo run --bin cb-verify --manifest-path "$REPO_DIR/Cargo.toml" --release -- \ --enclave "$ENCLAVE" \ - --cb-config "$CONFIG" \ + --config "$CONFIG" \ --timeout "$TIMEOUT" \ --min-epochs "$MIN_EPOCHS" \ --target-epoch "$TARGET_EPOCH" \ diff --git a/src/bin/test_mux.rs b/src/bin/test_mux.rs new file mode 100644 index 0000000..e22120d --- /dev/null +++ b/src/bin/test_mux.rs @@ -0,0 +1,475 @@ +//! Quick mux routing diagnostic. +//! +//! Fetches CB PBS logs from a running enclave, parses them, and checks +//! mux routing against the provided config. No observation window, no +//! epoch waiting. Just: fetch → parse → check. +//! +//! Usage: +//! cargo run --release --bin test_mux -- +//! +//! Example: +//! cargo run --release --bin test_mux -- CB-Testnet configs/generated/cb-mux.yml + +use std::process::Command; + +fn main() { + let args: Vec = std::env::args().collect(); + if args.len() != 3 { + eprintln!("Usage: {} ", args[0]); + eprintln!("Example: {} CB-Testnet configs/generated/cb-mux.yml", args[0]); + std::process::exit(1); + } + + let enclave = &args[1]; + let config_path = &args[2]; + + // Step 1: Parse mux config + println!("=== Parsing mux config: {config_path} ==="); + let entries = match parse_mux_config(config_path) { + Ok(Some(e)) => { + println!("Found {} mux entries:", e.len()); + for entry in &e { + println!(" {} → relay={} pubkeys={}", entry.id, entry.relay_identity, entry.validator_pubkeys.len()); + } + e + } + Ok(None) => { + println!("No [[mux]] sections found in config. Nothing to check."); + std::process::exit(0); + } + Err(e) => { + eprintln!("ERROR parsing config: {e}"); + std::process::exit(1); + } + }; + + // Step 2: Discover CB PBS services + println!("\n=== Discovering CB PBS services in enclave: {enclave} ==="); + let services = match discover_services(enclave) { + Ok(s) => s, + Err(e) => { + eprintln!("ERROR discovering services: {e}"); + std::process::exit(1); + } + }; + println!("Found {} CB PBS service(s): {:?}", services.len(), services); + + if services.is_empty() { + eprintln!("ERROR: No CB PBS services found"); + std::process::exit(1); + } + + // Step 3: Fetch and parse logs from each service + println!("\n=== Fetching logs ==="); + let mut all_events: Vec = Vec::new(); + let log_file = format!("/tmp/test_mux_{}.log", enclave); + + for service in &services { + println!("\n--- {service} ---"); + match fetch_logs(enclave, service) { + Ok(logs) => { + if logs.is_empty() { + println!(" (no relevant log lines found)"); + continue; + } + // Write raw logs to file for debugging + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(&log_file) { + use std::io::Write; + let _ = writeln!(f, "=== {service} ==="); + let _ = writeln!(f, "{}", logs); + } + + let mut parsed = 0; + let mut failed = 0; + for line in logs.lines() { + match parse_line(line) { + Some(event) => { + parsed += 1; + all_events.push(event); + } + None => { + failed += 1; + if failed <= 3 { + println!(" PARSE FAIL: {}", &line[..line.len().min(120)]); + } + } + } + } + println!(" Parsed: {parsed} lines, Failed: {failed} lines"); + } + Err(e) => { + println!(" ERROR: {e}"); + } + } + } + println!("\nRaw logs written to: {log_file}"); + + // Step 4: Print sample of ALL parsed events and collect unique messages + let mut all_messages: Vec = all_events.iter().map(|e| e.message.clone()).collect(); + all_messages.sort(); + all_messages.dedup(); + println!("\n=== Unique messages found ({} total) ===", all_messages.len()); + for msg in &all_messages { + let count = all_events.iter().filter(|e| &e.message == msg).count(); + println!(" ({}) {}", count, msg); + } + + println!("\n=== Sample of all parsed events (first 10) ==="); + for (i, event) in all_events.iter().take(10).enumerate() { + let pk_short = event.validator.as_ref().map(|v| if v.len() > 16 { &v[..16] } else { v }); + println!(" #{} msg={:?} slot={:?} mux={:?} relay={:?} val={:?}", + i, event.message, event.slot, event.mux_id, event.relay_id, pk_short); + } + if all_events.len() > 10 { + println!(" ... and {} more", all_events.len() - 10); + } + + // Step 5: Filter to mux events + println!("\n=== Mux Events ==="); + let mux_events: Vec<&CbEvent> = all_events + .iter() + .filter(|e| { + e.message.starts_with("using mux") + || e.message.starts_with("received new header") + || e.message.starts_with("auction winner") + }) + .collect(); + + println!("Total mux events: {}", mux_events.len()); + for event in &mux_events { + let pk_short = event.validator.as_ref().map(|v| if v.len() > 20 { &v[..20] } else { v }); + println!( + " [{}] slot={:?} mux={:?} relay={:?} val={:?}", + event.message, + event.slot, + event.mux_id, + event.relay_id, + pk_short + ); + } + + // Step 5: Check mux routing + println!("\n=== Mux Routing Check ==="); + let mut violations = 0; + let mut checked = 0; + + for event in &mux_events { + if let Some(ref pk) = event.validator { + let pk_norm = pk.to_lowercase(); + for entry in &entries { + if entry.validator_pubkeys.iter().any(|e| e.to_lowercase() == pk_norm) { + checked += 1; + if let Some(ref actual_mux) = event.mux_id { + if actual_mux != &entry.id { + violations += 1; + println!( + " VIOLATION: pubkey {} should route to '{}' but routed to '{}'", + &pk[..20.min(pk.len())], + entry.id, + actual_mux + ); + } + } + } + } + } + } + + println!("\n=== Result ==="); + if violations > 0 { + println!("FAIL: {violations} routing violation(s) out of {checked} checked"); + std::process::exit(1); + } else if checked == 0 { + println!("WARN: No mux events matched to config pubkeys. Events may not have proposer_pubkey fields."); + std::process::exit(0); + } else { + println!("PASS: All {checked} mux routing decisions are correct"); + std::process::exit(0); + } +} + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +struct MuxEntry { + id: String, + relay_identity: String, + validator_pubkeys: Vec, +} + +struct CbEvent { + message: String, + slot: Option, + validator: Option, + relay_id: Option, + mux_id: Option, +} + +// --------------------------------------------------------------------------- +// Config parsing +// --------------------------------------------------------------------------- + +fn parse_mux_config(path: &str) -> Result>, Box> { + let raw = std::fs::read_to_string(path)?; + + let template = if path.ends_with(".yml") || path.ends_with(".yaml") { + let parsed: serde_yaml::Value = serde_yaml::from_str(&raw)?; + parsed + .get("mev_params") + .and_then(|p| p.get("commit_boost_config")) + .and_then(|c| c.as_str()) + .ok_or("No mev_params.commit_boost_config found")? + .to_string() + } else { + raw + }; + + if !template.contains("[[mux]]") { + return Ok(None); + } + + let mut entries = Vec::new(); + let mut lines = template.lines().peekable(); + + while let Some(line) = lines.next() { + if line.trim() == "[[mux]]" { + let entry = parse_mux_section(&mut lines); + entries.push(entry); + } + } + + if entries.is_empty() { + return Ok(None); + } + + Ok(Some(entries)) +} + +fn parse_mux_section<'a>(lines: &mut std::iter::Peekable>) -> MuxEntry { + let mut id = None; + let mut pubkeys = None; + + loop { + let is_header = lines.peek().map(|l| l.trim().starts_with("[[")).unwrap_or(false); + if is_header { + let header = lines.peek().unwrap().trim().to_string(); + if header.starts_with("[[mux.relays]]") { + let _ = lines.next(); + continue; + } + break; + } + + let Some(line) = lines.next() else { break }; + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + if let Some((key, val)) = trimmed.split_once('=') { + let key = key.trim(); + let val = val.trim(); + match key { + "id" => id = Some(val.trim_matches('"').to_string()), + "validator_pubkeys" => pubkeys = Some(parse_pubkey_array(val, lines)), + _ => {} + } + } + } + + let id = id.unwrap_or_default(); + let relay_identity = if let Some(pos) = id.rfind("to_") { + let ident = id[pos + 3..].trim().to_string(); + if !ident.is_empty() { ident } else { id.clone() } + } else { + id.clone() + }; + let pubkeys = pubkeys.unwrap_or_default(); + + MuxEntry { id, relay_identity, validator_pubkeys: pubkeys } +} + +fn parse_pubkey_array(rest: &str, lines: &mut std::iter::Peekable) -> Vec { + let mut accum = rest.to_string(); + if !accum.trim_end().ends_with(']') { + loop { + let Some(next) = lines.next() else { break }; + accum.push('\n'); + accum.push_str(next); + if next.trim().ends_with(']') { break; } + } + } + let raw = accum.trim(); + let start = raw.find('[').unwrap_or(0); + let end = raw.rfind(']').unwrap_or(raw.len()); + raw[start + 1..end] + .split(',') + .map(|s| s.trim().trim_matches('"').to_string()) + .filter(|s| !s.is_empty()) + .collect() +} + +// --------------------------------------------------------------------------- +// Service discovery +// --------------------------------------------------------------------------- + +fn discover_services(enclave: &str) -> Result, Box> { + let output = Command::new("kurtosis") + .args(["enclave", "inspect", "--full-uuids", enclave]) + .output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("kurtosis enclave inspect failed: {stderr}").into()); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let mut services = Vec::new(); + + for line in stdout.lines() { + let lower = line.to_lowercase(); + if lower.contains("commit-boost") && lower.contains("running") { + // Extract service name (first column) + if let Some(name) = line.split_whitespace().next() { + services.push(name.to_string()); + } + } + } + + Ok(services) +} + +// --------------------------------------------------------------------------- +// Log fetching +// --------------------------------------------------------------------------- + +fn fetch_logs(enclave: &str, service: &str) -> Result> { + let output = Command::new("kurtosis") + .args(["service", "logs", enclave, service, "-n", "200000"]) + .output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("kurtosis service logs failed: {stderr}").into()); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + + // Filter to relevant lines + let result: String = stdout + .lines() + .filter(|line| { + line.contains("using mux config") + || line.contains("received new header") + || line.contains("auction winner") + || line.contains("received unblinded block") + }) + .collect::>() + .join("\n"); + + Ok(result) +} + +// --------------------------------------------------------------------------- +// Log parsing +// --------------------------------------------------------------------------- + +fn strip_ansi_codes(s: &str) -> String { + let mut result = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\x1b' { + if chars.peek() == Some(&'[') { + chars.next(); + while let Some(&ch) = chars.peek() { + chars.next(); + if ch.is_ascii_alphabetic() { break; } + } + } + } else { + result.push(c); + } + } + result +} + +fn parse_line(line: &str) -> Option { + let line = line.trim(); + if line.is_empty() { return None; } + + // Strip kurtosis prefix: "[service-name] rest" + let line = if line.starts_with('[') { + if let Some(pos) = line.find(']') { + line[pos + 1..].trim_start() + } else { line } + } else { line }; + + // Strip ANSI escape codes + let line = strip_ansi_codes(&line); + + // Find message after "LEVEL : " or "LEVEL " + let after_level: String = if let Some(pos) = line.find(" : ") { + line[pos + 3..].to_string() + } else { + let levels = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR"]; + let mut found = line.clone(); + for lvl in &levels { + if let Some(pos) = line.find(&format!(" {} ", lvl)) { + found = line[pos + lvl.len() + 2..].to_string(); + break; + } + if line.starts_with(lvl) { + found = line[lvl.len()..].trim_start().to_string(); + break; + } + } + found + }; + + // Find message/key boundary: first " key=" where key is a valid identifier + let mut message_end = after_level.len(); + let bytes = after_level.as_bytes(); + for i in 0..bytes.len() { + if bytes[i] == b' ' { + let rest = &after_level[i + 1..]; + if rest.is_empty() { continue; } + let first = rest.as_bytes()[0]; + if first.is_ascii_alphabetic() || first == b'_' { + if let Some(eq_pos) = rest.find('=') { + let key = &rest[..eq_pos]; + if key.chars().all(|c| c.is_alphanumeric() || c == '_') { + let after_eq = &rest[eq_pos + 1..]; + if !after_eq.is_empty() { + message_end = i; + break; + } + } + } + } + } + } + + let message = after_level[..message_end].trim().to_string(); + let kv_part = &after_level[message_end..]; + + let mut slot = None; + let mut validator = None; + let mut relay_id = None; + let mut mux_id = None; + + for kv in kv_part.split_whitespace() { + if let Some((key, val)) = kv.split_once('=') { + let val = val.trim_matches('"'); + match key { + "slot" => { slot = val.parse().ok(); } + "validator" | "pubkey" => { validator = Some(val.to_string()); } + "relay_id" => { relay_id = Some(val.to_string()); } + "mux_id" => { mux_id = Some(val.to_string()); } + _ => {} + } + } + } + + Some(CbEvent { message, slot, validator, relay_id, mux_id }) +} diff --git a/src/checks/chain_health.rs b/src/checks/chain_health.rs index 56478a7..41a83ac 100644 --- a/src/checks/chain_health.rs +++ b/src/checks/chain_health.rs @@ -29,10 +29,18 @@ pub async fn check_missed_slots( end_slot: u64, threshold: f64, ) -> CheckResult { - let total = end_slot.saturating_sub(start_slot); - if total == 0 { + if start_slot > end_slot { return CheckResult::fail("missed_slots", 2, "Invalid slot range"); } + // Single-slot window: nothing meaningful to check + if start_slot == end_slot { + return CheckResult::skip( + "missed_slots", + 2, + format!("Single-slot window (slot {}), skipping missed slot check", start_slot), + ); + } + let total = end_slot - start_slot; let mut missed = 0u64; for slot in start_slot..end_slot { diff --git a/src/checks/mux_routing.rs b/src/checks/mux_routing.rs index f02a0ce..c71e526 100644 --- a/src/checks/mux_routing.rs +++ b/src/checks/mux_routing.rs @@ -1,22 +1,60 @@ //! MUX routing verification check. //! //! Extracts `[[mux]]` sections from a Commit-Boost config TOML file. -//! If the config contains mux rules, automatically verifies that every -//! delivered payload conforms — zero cross-contamination between relays. +//! If the config contains mux rules, verifies that CB's PBS service +//! correctly routes getHeader requests according to the mux config. +//! +//! Verification works by fetching CB PBS container logs and parsing +//! structured INFO lines. Each line records key=value pairs that are +//! cross-referenced against the mux configuration. +//! +//! Relevant CB log lines (from commit-boost-client crates): +//! "using mux config" — mux_id, relays, pubkey (DEBUG) +//! "received new header" — relay_id, slot, validator, value_eth, block_hash (INFO) +//! "auction winner" — relay_id, value_eth, block_hash (INFO) +//! "new request" (submit_blinded_blocks) — slot, validator (INFO) +//! "received unblinded block (v1/v2)" — (INFO) +//! "CRITICAL: no payload received" — block_hash (ERROR) +//! //! If no mux sections are found, the check is skipped. use std::collections::{HashMap, HashSet}; -use crate::checks::{CheckResult, CheckStatus}; -use crate::relay::RelayClient; +use crate::checks::CheckResult; +use tracing::{info, warn}; -/// A single parsed mux entry with the relay index resolved from the template. +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/// A single parsed mux entry from the CB config. pub struct MuxEntry { pub id: String, - pub relay_index: usize, + pub relay_identity: String, pub validator_pubkeys: Vec, } +/// Parsed event from a CB INFO/DEBUG log line. +#[derive(Debug, Clone)] +pub struct CbEvent { + /// The log message text (e.g., "using mux config", "received new header") + pub message: String, + /// Key=value pairs extracted from the log line. + pub fields: HashMap, + /// The slot number if present in the fields. + pub slot: Option, + /// The validator pubkey if present. + pub validator: Option, + /// The relay_id if present. + pub relay_id: Option, + /// The mux_id if present. + pub mux_id: Option, +} + +// --------------------------------------------------------------------------- +// Config parsing (unchanged from before) +// --------------------------------------------------------------------------- + /// Parse a Commit-Boost config file and extract mux routing entries. /// /// Supports two file formats: @@ -43,7 +81,6 @@ pub fn extract_mux_from_config(path: &str) -> eyre::Result> parse_mux_from_toml_template(&template) } -/// Extract the `commit_boost_config` field from a Kurtosis YAML config. fn extract_commit_boost_config_from_yaml(raw: &str) -> eyre::Result { use serde_yaml::Value as YamlValue; @@ -63,16 +100,10 @@ fn extract_commit_boost_config_from_yaml(raw: &str) -> eyre::Result { Ok(template.to_string()) } -/// Quick inspection: does the raw text look like a TOML with `[[mux]]`? fn has_mux_sections(text: &str) -> bool { - text.lines().any(|l| l.trim() == "[[mux]]" || l.trim() == "[[mux]]") + text.lines().any(|l| l.trim() == "[[mux]]") } -/// Parse `[[mux]]` sections from a Commit-Boost config TOML template. -/// -/// The template may contain Go template expressions like `{{ index .Relays 0 }}` -/// in string values. This works around that by doing a line-by-line parse -/// rather than using a TOML parser (which would choke on template syntax). fn parse_mux_from_toml_template(template: &str) -> eyre::Result>> { if !has_mux_sections(template) { return Ok(None); @@ -83,16 +114,13 @@ fn parse_mux_from_toml_template(template: &str) -> eyre::Result e, - Ok(None) => continue, // parse_one_mux_section advanced but returned nothing + Ok(None) => continue, Err(e) => return Err(eyre::eyre!("Failed to parse mux section: {e}")), }; @@ -106,19 +134,23 @@ fn parse_mux_from_toml_template(template: &str) -> eyre::Result String { + if let Some(pos) = id.rfind("to_") { + let ident = id[pos + 3..].trim().to_string(); + if !ident.is_empty() { + return ident; + } + } + id.to_string() +} + fn parse_one_mux_section<'a>( lines: &mut std::iter::Peekable>, ) -> eyre::Result> { let mut id: Option = None; let mut pubkeys: Option> = None; - let mut relay_index: Option = None; loop { - // Peek at what's next without consuming it let is_section_header = lines .peek() .map(|l| l.trim().starts_with("[[")) @@ -127,16 +159,10 @@ fn parse_one_mux_section<'a>( if is_section_header { let header = lines.peek().unwrap().trim().to_string(); if header.starts_with("[[mux.relays]]") { - // Consume the [[mux.relays]] header and parse its body let _ = lines.next(); - let idx = parse_mux_relay_body(lines)?; - if relay_index.is_none() { - relay_index = idx; - } + let _ = parse_mux_relay_body(lines)?; continue; } - // Any other [[ section — stop without consuming. - // The caller's main loop will pick up the header. break; } @@ -145,12 +171,10 @@ fn parse_one_mux_section<'a>( }; let trimmed = line.trim(); - // Skip comments and blank lines if trimmed.is_empty() || trimmed.starts_with('#') { continue; } - // key = "value" if let Some((key, raw_val)) = parse_key_value(trimmed) { match key { "id" => { @@ -159,34 +183,27 @@ fn parse_one_mux_section<'a>( "validator_pubkeys" => { pubkeys = Some(parse_pubkey_array(raw_val, lines)?); } - _ => {} // Ignore other keys + _ => {} } } } let id = id.ok_or_else(|| eyre::eyre!("[[mux]] section missing 'id' field"))?; + let relay_identity = relay_identity_from_mux_id(&id); let pubkeys = pubkeys .ok_or_else(|| eyre::eyre!("[[mux]] section '{id}' missing 'validator_pubkeys'"))?; - let relay_index = relay_index.ok_or_else(|| { - eyre::eyre!("[[mux]] section '{id}' missing [[mux.relays]] with url template") - })?; Ok(Some(MuxEntry { id, - relay_index, + relay_identity, validator_pubkeys: pubkeys, })) } -/// Parse the body of a [[mux.relays]] sub-section (after its header was -/// consumed) to extract the relay index from `url = "{{ index .Relays N }}"`. -/// -/// Stops at the next `[[` section header without consuming it. fn parse_mux_relay_body( lines: &mut std::iter::Peekable, ) -> eyre::Result> { loop { - // Stop at next section header without consuming if lines .peek() .map(|l| l.trim().starts_with("[[")) @@ -207,10 +224,7 @@ fn parse_mux_relay_body( if let Some((key, raw_val)) = parse_key_value(trimmed) { if key == "url" { let val = raw_val.trim_matches('"'); - if let Some(idx) = parse_relay_index_from_template(val) { - return Ok(Some(idx)); - } - return Ok(None); + return Ok(parse_relay_index_from_template(val)); } } } @@ -218,18 +232,13 @@ fn parse_mux_relay_body( Ok(None) } -/// Extract the relay index from a Go template expression like `{{ index .Relays 0 }}`. fn parse_relay_index_from_template(val: &str) -> Option { let val = val.trim(); - // Match: {{ index .Relays N }} or {{ index .Relays N}} - // The N is a decimal integer let stripped = val .trim_start_matches("{{") .trim_end_matches("}}") .trim(); - // Now stripped should be something like: index .Relays 0 let parts: Vec<&str> = stripped.split_whitespace().collect(); - // Expected: ["index", ".Relays", "N"] if parts.len() >= 3 && parts[0] == "index" && parts[1] == ".Relays" { parts[2].parse::().ok() } else { @@ -237,7 +246,6 @@ fn parse_relay_index_from_template(val: &str) -> Option { } } -/// Parse a simple key = "value" pair. fn parse_key_value(s: &str) -> Option<(&str, &str)> { let eq_pos = s.find('=')?; let key = s[..eq_pos].trim(); @@ -245,20 +253,12 @@ fn parse_key_value(s: &str) -> Option<(&str, &str)> { Some((key, raw_val)) } -/// Parse a TOML-style array literal: `["0x...", "0x...", ...]` -/// -/// Handles both single-line and multi-line formats. -/// `rest` is what follows the `=` sign. If it starts with `[`, the array -/// is inline. If not, the array starts on the next line(s). fn parse_pubkey_array( rest: &str, lines: &mut std::iter::Peekable, ) -> eyre::Result> { let mut accum = rest.to_string(); - // Collect lines until we find the closing `]`. - // Works for both single-line (`rest` already contains `]`) and - // multi-line (rest is just `[`, or rest is the start up to `= [`). if !accum.trim_end().ends_with(']') { loop { let Some(next) = lines.next() else { @@ -272,10 +272,7 @@ fn parse_pubkey_array( } } - // Now parse the array from accum let raw = accum.trim(); - - // Find the opening and closing brackets let start = raw.find('[').ok_or_else(|| { eyre::eyre!("Could not find opening '[' in pubkey array: {raw:.50}...") })?; @@ -284,15 +281,10 @@ fn parse_pubkey_array( })?; let inner = &raw[start + 1..end]; - let mut pubkeys = Vec::new(); for item in inner.split(',') { let item = item.trim().trim_matches('"').trim(); - if item.is_empty() { - continue; - } - // Strip Go template expressions if any leaked in - if item.contains("{{") || item.contains("}}") { + if item.is_empty() || item.contains("{{") || item.contains("}}") { continue; } pubkeys.push(item.to_string()); @@ -301,22 +293,243 @@ fn parse_pubkey_array( Ok(pubkeys) } -/// Normalize a public key string: strip `0x` prefix, lowercase. fn normalize_pubkey(pk: &str) -> String { pk.trim_start_matches("0x") .trim_start_matches("0X") .to_lowercase() } -/// Run the mux routing check against the relay data. +// --------------------------------------------------------------------------- +// CB log parsing +// --------------------------------------------------------------------------- + +/// Strip ANSI escape codes from a string. +/// +/// ANSI codes look like `\x1b[32m` (color) or `\x1b[0m` (reset). +/// This removes them so we can parse the actual text content. +fn strip_ansi_codes(s: &str) -> String { + let mut result = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\x1b' { + // Skip the '[' and everything until we hit a letter (m, H, J, etc.) + if chars.peek() == Some(&'[') { + chars.next(); // skip '[' + while let Some(&ch) = chars.peek() { + chars.next(); + if ch.is_ascii_alphabetic() { + break; + } + } + } + } else { + result.push(c); + } + } + result +} + +/// Parse a CB tracing log line into a `CbEvent`. +/// +/// CB uses `tracing` with a compact format: +/// `timestamp LEVEL : message key=value key=value ...` +/// +/// We extract the message and all key=value pairs. +pub fn parse_cb_log_line(line: &str) -> Option { + let line = line.trim(); + if line.is_empty() { + return None; + } + + // Strip kurtosis prefix: "[service-name] rest" + let line = if line.starts_with('[') { + if let Some(pos) = line.find(']') { + line[pos + 1..].trim_start() + } else { + line + } + } else { + line + }; + + // Strip ANSI escape codes (e.g., "\x1b[32m" for green text) + let line = strip_ansi_codes(line); + // Continue parsing with the cleaned line (now owned String) + // All subsequent code uses `line` as a String, not &str + + // Find the message portion after "LEVEL : " or "LEVEL ". + let after_level: String = if let Some(pos) = line.find(" : ") { + line[pos + 3..].to_string() + } else { + let levels = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR"]; + let mut found = line.clone(); + for lvl in &levels { + if let Some(pos) = line.find(&format!(" {} ", lvl)) { + found = line[pos + lvl.len() + 2..].to_string(); + break; + } + if line.starts_with(lvl) { + found = line[lvl.len()..].trim_start().to_string(); + break; + } + } + found + }; + + // Find the message/key boundary by scanning for " key=" where key is + // a valid identifier (alphanumeric + underscore, starting with alpha). + // This is more reliable than a fixed list of known keys. + let mut message_end = after_level.len(); + let bytes = after_level.as_bytes(); + for i in 0..bytes.len() { + if bytes[i] == b' ' { + // Check if what follows is "key=" where key starts with alpha/underscore + let rest = &after_level[i + 1..]; + if rest.is_empty() { + continue; + } + let first = rest.as_bytes()[0]; + if first.is_ascii_alphabetic() || first == b'_' { + // Find the '=' after the key + if let Some(eq_pos) = rest.find('=') { + let key = &rest[..eq_pos]; + // Key must be all alphanumeric/underscore + if key.chars().all(|c| c.is_alphanumeric() || c == '_') { + // Value after '=' must not be empty (or must be quote) + let after_eq = &rest[eq_pos + 1..]; + if !after_eq.is_empty() { + message_end = i; + break; + } + } + } + } + } + } + + let message = after_level[..message_end].trim().to_string(); + let kv_part = &after_level[message_end..].to_string(); + + let mut fields = HashMap::new(); + let mut slot = None; + let mut validator = None; + let mut relay_id = None; + let mut mux_id = None; + + for kv in kv_part.split_whitespace() { + if let Some((key, val)) = kv.split_once('=') { + let val = val.trim_matches('"').to_string(); + fields.insert(key.to_string(), val.clone()); + + match key { + "slot" => { slot = val.parse().ok(); } + "validator" | "pubkey" => { validator = Some(normalize_pubkey(&val)); } + "relay_id" => { relay_id = Some(val); } + "mux_id" => { mux_id = Some(val); } + _ => {} + } + } + } + + Some(CbEvent { + message, + fields, + slot, + validator, + relay_id, + mux_id, + }) +} + + + +// --------------------------------------------------------------------------- +// Log fetching +// --------------------------------------------------------------------------- + +/// Fetch logs from a Kurtosis service, filtered to relevant mux/pbs lines. +/// +/// Fetches all logs and filters client-side. The `--regex-match` flag is +/// tried first as an optimization, but some kurtosis versions ignore it. +pub fn fetch_service_logs(enclave: &str, service: &str) -> eyre::Result { + info!( + "mux check: fetching logs from service '{service}' (enclave={enclave})..." + ); + + let output = std::process::Command::new("kurtosis") + .args([ + "service", "logs", enclave, service, + "-n", "200000", + ]) + .output() + .map_err(|e| eyre::eyre!("Failed to run 'kurtosis service logs': {e}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(eyre::eyre!( + "kurtosis service logs {enclave} {service} failed (rc={:?}): {}", + output.status.code(), + stderr.trim() + )); + } + + // Combine stdout and stderr — kurtosis writes to either depending on version. + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let all_logs = format!("{}\n{}", stdout, stderr); + + // Filter to relevant lines client-side. + let result: String = all_logs + .lines() + .filter(|line| { + line.contains("using mux config") + || line.contains("received new header") + || line.contains("auction winner") + || line.contains("received unblinded block") + || line.contains("CRITICAL: no payload") + }) + .collect::>() + .join("\n"); + + if result.is_empty() { + let sample: String = all_logs + .lines() + .filter(|l| !l.trim().is_empty()) + .take(3) + .collect::>() + .join("\n"); + warn!( + "mux check: service '{service}' returned no relevant log lines. Total: {} bytes stdout, {} bytes stderr. Sample:\n{}", + stdout.len(), + stderr.len(), + sample + ); + } else { + info!( + "mux check: service '{service}' returned {} relevant log line(s)", + result.lines().count() + ); + } + + Ok(result) +} + +// --------------------------------------------------------------------------- +// Verification +// --------------------------------------------------------------------------- + +/// Run the mux routing check by parsing CB PBS service logs. /// -/// Returns SKIP if entries is empty (no mux config). -/// Returns PASS/WARN/FAIL based on cross-contamination findings. +/// Fetches logs from each CB PBS service in the enclave, parses structured +/// events, and verifies that the mux routing decisions match the mux config. +/// +/// PASS: All logged routing decisions match the mux config. +/// FAIL: A pubkey was routed to the wrong mux/relay. +/// WARN: No relevant log lines found (no getHeader requests occurred). pub async fn check_mux_routing( - relays: &[RelayClient], + enclave: &str, + cb_service_names: &[String], entries: &[MuxEntry], - start_slot: u64, - end_slot: u64, ) -> CheckResult { if entries.is_empty() { return CheckResult::skip( @@ -326,100 +539,130 @@ pub async fn check_mux_routing( ); } - if entries.len() > relays.len() { - return CheckResult::fail( - "mux.routing", - 1, - format!( - "Mux config has {} entries but only {} relay(s) discovered", - entries.len(), - relays.len() - ), - ); - } - - // Validate relay indices are in range - for entry in entries { - if entry.relay_index >= relays.len() { - return CheckResult::fail( - "mux.routing", - 1, - format!( - "Mux '{}' references relay_index {} but only {} relay(s) discovered", - entry.id, - entry.relay_index, - relays.len() - ), - ); - } - } - - // Build: normalized_pubkey -> expected_relay_index - let mut expected_relay: HashMap = HashMap::new(); + // Build: normalized_pubkey → expected_mux_id + let mut expected_mux: HashMap = HashMap::new(); for entry in entries { for pk in &entry.validator_pubkeys { - let nk = normalize_pubkey(pk); - expected_relay.insert(nk, entry.relay_index); + expected_mux.insert(normalize_pubkey(pk), entry.id.clone()); } } - let total_mapped_pubkeys = expected_relay.len(); - // Query each relay's delivered payloads in the observation window - let mut violations: Vec = Vec::new(); - let mut seen_pubkeys: HashMap> = HashMap::new(); - - for (relay_idx, relay) in relays.iter().enumerate() { - let Ok(entries) = relay.get_payloads_delivered(start_slot, end_slot).await else { - tracing::warn!( - "mux check: relay[{relay_idx}] {} payload query failed", - relay.base_url() - ); - continue; + // Fetch and parse logs from each CB PBS service. + let mut all_events: Vec = Vec::new(); + + for service_name in cb_service_names { + let logs = match fetch_service_logs(enclave, service_name) { + Ok(l) => l, + Err(e) => { + warn!( + "mux check: failed to fetch logs from service '{service_name}': {e}. \ + Skipping this service." + ); + continue; + } }; - tracing::info!( - "mux check: relay[{relay_idx}] {} returned {} delivered payload(s)", - relay.base_url(), - entries.len() - ); - - for payload in &entries { - let pk_hex = payload.proposer_pubkey.to_string(); - let pk_norm = normalize_pubkey(&pk_hex); - - if let Some(&expected_idx) = expected_relay.get(&pk_norm) { - if expected_idx != relay_idx { - violations.push(serde_json::json!({ - "slot": payload.slot, - "proposer_pubkey": format!("0x{pk_norm}"), - "relay_actual": relay_idx, - "relay_expected": expected_idx, - })); - } + if logs.is_empty() { + continue; + } - seen_pubkeys - .entry(pk_norm) - .or_default() - .insert(payload.slot); + for line in logs.lines() { + if let Some(event) = parse_cb_log_line(line) { + all_events.push(event); } } } - // Mapped pubkeys that never appeared in any relay's deliveries - let mut unseen: Vec = Vec::new(); - for pk_norm in expected_relay.keys() { - if !seen_pubkeys.contains_key(pk_norm) { - unseen.push(format!("0x{pk_norm}")); + // Filter to events relevant to mux verification. + let mux_events: Vec<&CbEvent> = all_events + .iter() + .filter(|e| { + e.message.starts_with("using mux") + || e.message.starts_with("received new header") + || e.message.starts_with("auction winner") + || e.message.starts_with("received unblinded block") + || e.message.contains("CRITICAL: no payload") + }) + .collect(); + + let total_events = mux_events.len(); + + let data = serde_json::json!({ + "total_mux_entries": entries.len(), + "total_log_events": total_events, + "pubkeys_verified": 0, + "violations": [], + "violation_count": 0, + "mux_entries_seen": [], + }); + + if total_events == 0 { + return CheckResult::warn( + "mux.routing", + 1, + format!( + "No mux-related log lines found in any CB PBS service. \ + No getHeader requests were recorded — mux config is valid \ + but routing could not be verified at runtime. muxes=[{}]", + entries.iter().map(|e| e.id.as_str()).collect::>().join(", ") + ), + ).with_data(data); + } + + // Verify: for each "using mux config" event, does the pubkey match? + let mut violations: Vec = Vec::new(); + let mut pubkeys_verified: HashSet = HashSet::new(); + let mut mux_entries_seen: HashSet = HashSet::new(); + + for event in &mux_events { + if let Some(ref mux_id) = event.mux_id { + mux_entries_seen.insert(mux_id.clone()); + } + + if let Some(ref pk_norm) = event.validator { + pubkeys_verified.insert(pk_norm.clone()); + + if let Some(expected_mux_id) = expected_mux.get(pk_norm) { + if let Some(ref actual_mux_id) = event.mux_id { + if actual_mux_id != expected_mux_id { + let expected_relay = entries + .iter() + .find(|e| e.id == *expected_mux_id) + .map(|e| e.relay_identity.as_str()) + .unwrap_or("?"); + let actual_relay = entries + .iter() + .find(|e| e.id == *actual_mux_id) + .map(|e| e.relay_identity.as_str()) + .unwrap_or("?"); + + violations.push(serde_json::json!({ + "slot": event.slot, + "proposer_pubkey": format!("0x{pk_norm}"), + "routed_to_mux": actual_mux_id, + "routed_to_relay": actual_relay, + "expected_mux": expected_mux_id, + "expected_relay": expected_relay, + })); + + warn!( + "mux check: MISROUTING — pubkey 0x{pk_norm}.. should route to \ + '{expected_mux_id}' ({expected_relay}) but was routed to \ + '{actual_mux_id}' ({actual_relay})" + ); + } + } + } } } let data = serde_json::json!({ - "total_mapped_pubkeys": total_mapped_pubkeys, + "total_mux_entries": entries.len(), + "total_log_events": total_events, + "pubkeys_verified": pubkeys_verified.len(), "violations": violations, "violation_count": violations.len(), - "unseen_pubkeys": unseen, - "unseen_count": unseen.len(), - "total_deliveries_checked": seen_pubkeys.values().map(|s| s.len()).sum::(), + "mux_entries_seen": mux_entries_seen.iter().cloned().collect::>(), }); let mux_ids: Vec<&str> = entries.iter().map(|e| e.id.as_str()).collect(); @@ -430,32 +673,20 @@ pub async fn check_mux_routing( "mux.routing", 1, format!( - "{} mux routing violation(s): pubkey(s) delivered to the wrong relay. {}", + "{} mux routing violation(s): CB PBS routed a pubkey to the wrong mux/relay. {}", violations.len(), mux_detail, ), ) .with_data(data) - } else if !unseen.is_empty() { - CheckResult::warn( - "mux.routing", - 1, - format!( - "No routing violations ✓. {} mux-mapped pubkey(s) had zero deliveries \ - in the observation window (they may not have proposed). {}", - unseen.len(), - mux_detail, - ), - ) - .with_data(data) } else { CheckResult::pass( "mux.routing", 1, format!( - "All {} mux-mapped pubkey(s) delivered exclusively to their assigned relays. \ - Zero routing violations. {}", - total_mapped_pubkeys, + "All {} mux routing decision(s) verified ✓ CB PBS correctly routed \ + every getHeader request according to mux config. {}", + total_events, mux_detail, ), ) @@ -463,10 +694,22 @@ pub async fn check_mux_routing( } } +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + #[cfg(test)] mod tests { use super::*; + #[test] + fn test_relay_identity_from_mux_id() { + assert_eq!(relay_identity_from_mux_id("node_0_to_helix"), "helix"); + assert_eq!(relay_identity_from_mux_id("node_1_to_flashbots"), "flashbots"); + assert_eq!(relay_identity_from_mux_id("my_mux_entry"), "my_mux_entry"); + assert_eq!(relay_identity_from_mux_id("to_"), "to_"); + } + #[test] fn test_parse_relay_index_from_template() { assert_eq!( @@ -477,20 +720,7 @@ mod tests { parse_relay_index_from_template("{{ index .Relays 1 }}"), Some(1) ); - assert_eq!( - parse_relay_index_from_template("{{ index .Relays 42 }}"), - Some(42) - ); - assert_eq!( - parse_relay_index_from_template("{{ index .Relays 0}}"), - Some(0) - ); - // Not a template - assert_eq!( - parse_relay_index_from_template("http://relay:18550"), - None - ); - // Range variable reference + assert_eq!(parse_relay_index_from_template("http://relay:18550"), None); assert_eq!(parse_relay_index_from_template("{{ $relay }}"), None); } @@ -502,18 +732,8 @@ mod tests { #[test] fn test_normalize_pubkey() { - assert_eq!( - normalize_pubkey("0xABC123"), - "abc123" - ); - assert_eq!( - normalize_pubkey("abc123"), - "abc123" - ); - assert_eq!( - normalize_pubkey("0XABC123"), - "abc123" - ); + assert_eq!(normalize_pubkey("0xABC123"), "abc123"); + assert_eq!(normalize_pubkey("abc123"), "abc123"); } #[test] @@ -566,10 +786,10 @@ url = "{{ index .Relays 1 }}" let entries = result.unwrap(); assert_eq!(entries.len(), 2); assert_eq!(entries[0].id, "node_0_to_helix"); - assert_eq!(entries[0].relay_index, 0); + assert_eq!(entries[0].relay_identity, "helix"); assert_eq!(entries[0].validator_pubkeys.len(), 2); assert_eq!(entries[1].id, "node_1_to_flashbots"); - assert_eq!(entries[1].relay_index, 1); + assert_eq!(entries[1].relay_identity, "flashbots"); assert_eq!(entries[1].validator_pubkeys.len(), 1); } @@ -607,4 +827,151 @@ additional_services: assert!(template.contains("[[mux]]")); assert!(template.contains("0xabc")); } + + // --- CB log parsing tests --- + + #[test] + fn test_parse_cb_log_line_using_mux_config() { + let line = r#"2026-05-06T18:50:43.014799Z DEBUG : using mux config mux_id="node_1_to_flashbots" relays=1 pubkey=0xb2ad1574eaca33f1555308e24b27a095d24aed8f4af5302ea2c6ba2e50936d25ffea7047be94065eac630693c7f86757 method=/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} req_id=85e4778a-9144-4cde-80d8-d11a5078f760 slot=160 parent_hash=0xcdee44e74bab2f2ee522dadccadc2cbdf67f8cfa96b5b10ab893becb6cb16bb7 validator=0xb2ad1574eaca33f1555308e24b27a095d24aed8f4af5302ea2c6ba2e50936d25ffea7047be94065eac630693c7f86757"#; + + let event = parse_cb_log_line(line).expect("should parse"); + assert_eq!(event.message, "using mux config"); + assert_eq!(event.mux_id, Some("node_1_to_flashbots".to_string())); + assert_eq!(event.slot, Some(160)); + assert!(event.validator.is_some()); + assert_eq!(event.validator.unwrap(), "b2ad1574eaca33f1555308e24b27a095d24aed8f4af5302ea2c6ba2e50936d25ffea7047be94065eac630693c7f86757"); + } + + #[test] + fn test_parse_cb_log_line_with_kurtosis_prefix() { + // This is the actual format that kurtosis service logs returns + let line = r#"[commit-boost-1-lighthouse-geth] 2026-05-07T01:24:00.002761Z DEBUG : using mux config mux_id="node_0_to_helix" relays=1 pubkey=0x98213294b82bc66ee39e95a678472fb41df846ec2863c5be53e1fd56b6ff0fe1bfd5b2bd8c534dd97acbe597ad119cc7 method=/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} req_id=8e5020cb-a893-42b3-a2f5-8f4c3f400c9e slot=521 parent_hash=0x969f22b336da6810b4cb9e31837b9d8e26f0a9c3277d150587742bb049561031 validator=0x98213294b82bc66ee39e95a678472fb41df846ec2863c5be53e1fd56b6ff0fe1bfd5b2bd8c534dd97acbe597ad119cc7"#; + + let event = parse_cb_log_line(line).expect("should parse kurtosis-prefixed line"); + assert_eq!(event.message, "using mux config"); + assert_eq!(event.mux_id, Some("node_0_to_helix".to_string())); + assert_eq!(event.slot, Some(521)); + assert!(event.validator.is_some()); + assert_eq!(event.relay_id, None); // no relay_id in this line + } + + #[test] + fn test_parse_cb_log_line_received_new_header_with_prefix() { + let line = r#"[commit-boost-1-lighthouse-geth] 2026-05-07T01:24:00.009013Z INFO : received new header relay_id="mux_helix" header_size_bytes=2891 latency=6.1415ms version=Fulu value_eth="0.050439063999832000" block_hash=0x15cd5f31333e1a8d42f0207cf1a61c65baf3d938836b07877a3a76b1cb890d11 method=/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} req_id=8e5020cb-a893-42b3-a2f5-8f4c3f400c9e slot=521"#; + + let event = parse_cb_log_line(line).expect("should parse"); + assert_eq!(event.message, "received new header"); + assert_eq!(event.relay_id, Some("mux_helix".to_string())); + assert_eq!(event.slot, Some(521)); + assert_eq!(event.fields.get("header_size_bytes"), Some(&"2891".to_string())); + assert_eq!(event.fields.get("value_eth"), Some(&"0.050439063999832000".to_string())); + } + + #[test] + fn test_parse_cb_log_line_received_new_header() { + let line = r#"2026-05-06T20:43:48.009642Z INFO : received new header relay_id="mux_helix" header_size_bytes=3099 latency=5.893291ms version=Fulu value_eth="0.042701386561497000" block_hash=0x0d7d119986cbd7b1c376056ffa703245404dc4d9d8b989f2d5ce2ee93d9354aa method=/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} req_id=6adfc13b-87ec-4564-a223-ac23af5b14eb slot=34 parent_hash=0xbd3df2244327d05919f51b0b4bbcafe655c16209573750b5ee94b39f77ff5e44 validator=0x867e89563df1501ac7dc5a369e6713cebab2aa1b676ea6d97fcb62802488866ae1223b4ed6c00718ee895d7e8e650cac"#; + + let event = parse_cb_log_line(line).expect("should parse"); + assert_eq!(event.message, "received new header"); + assert_eq!(event.relay_id, Some("mux_helix".to_string())); + assert_eq!(event.slot, Some(34)); + assert!(event.validator.is_some()); + assert_eq!(event.fields.get("header_size_bytes"), Some(&"3099".to_string())); + assert_eq!(event.fields.get("value_eth"), Some(&"0.042701386561497000".to_string())); + } + + #[test] + fn test_parse_cb_log_line_auction_winner() { + let line = r#"2026-05-06T20:43:48.010000Z INFO : auction winner relay_id="mux_helix" value_eth="0.042701386561497000" block_hash=0x0d7d119986cbd7b1c376056ffa703245404dc4d9d8b989f2d5ce2ee93d9354aa"#; + + let event = parse_cb_log_line(line).expect("should parse"); + assert_eq!(event.message, "auction winner"); + assert_eq!(event.relay_id, Some("mux_helix".to_string())); + } + + #[test] + fn test_parse_cb_log_line_non_mux_line() { + let line = "2026-05-06T20:43:48.009642Z INFO : some other message key=value"; + let event = parse_cb_log_line(line).expect("should parse"); + assert_eq!(event.message, "some other message"); + assert!(event.mux_id.is_none()); + } + + #[test] + fn test_parse_cb_log_line_empty() { + assert!(parse_cb_log_line("").is_none()); + assert!(parse_cb_log_line(" ").is_none()); + } +} + +#[cfg(test)] +mod log_file_tests { + use super::*; + + #[test] + fn test_parse_cb_log_line_from_file() { + // Test parsing the actual log format from kurtosis service logs + // These lines have ANSI escape codes and [service-name] prefix + let lines = vec![ + r#"[16eac416a3014ec191173b9e95cc11a6] 2026-05-07T04:28:26.004744Z DEBUG : using mux config mux_id="node_1_to_flashbots" relays=1 pubkey=0x8ca49f0c method=/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} req_id=90252b32 slot=2 validator=0x8ca49f0c"#, + r#"[commit-boost-1-lighthouse-geth] 2026-05-07T01:24:00.002761Z DEBUG : using mux config mux_id="node_0_to_helix" relays=1 pubkey=0x98213294 slot=521 validator=0x98213294"#, + ]; + + for line in &lines { + let event = parse_cb_log_line(line).expect(&format!("should parse: {}", &line[..80])); + assert!(event.message.starts_with("using mux"), "message should start with 'using mux', got: {:?}", event.message); + assert!(event.mux_id.is_some(), "mux_id should be Some"); + assert!(event.slot.is_some(), "slot should be Some"); + assert!(event.validator.is_some(), "validator should be Some"); + } + } + + #[test] + fn test_parse_cb_log_line_with_ansi_codes() { + // Lines from kurtosis service logs have ANSI escape codes for coloring + let line = "[16eac416a3014ec191173b9e95cc11a6] \x1b[2m2026-05-07T04:28:26.004744Z\x1b[0m \x1b[34mDEBUG\x1b[0m \x1b[1m\x1b[0m: using mux config \x1b[3mmux_id\x1b[0m\x1b[2m=\x1b[0m\"node_1_to_flashbots\" \x1b[3mrelays\x1b[0m\x1b[2m=\x1b[0m1 \x1b[3mpubkey\x1b[0m\x1b[2m=\x1b[0m0x8ca49f0c \x1b[2m\x1b[3mmethod\x1b[0m\x1b[2m=\x1b[0m/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} \x1b[3mreq_id\x1b[0m\x1b[2m=\x1b[0m90252b32 \x1b[3mslot\x1b[0m\x1b[2m=\x1b[0m2 \x1b[3mvalidator\x1b[0m\x1b[2m=\x1b[0m0x8ca49f0c\x1b[0m"; + + let event = parse_cb_log_line(line).expect("should parse line with ANSI codes"); + // The message should contain "using mux" (may have trailing ANSI codes) + assert!(event.message.contains("using mux"), "message should contain 'using mux', got: {:?}", event.message); + // mux_id should be parsed correctly despite ANSI codes + assert_eq!(event.mux_id, Some("node_1_to_flashbots".to_string())); + assert_eq!(event.slot, Some(2)); + assert!(event.validator.is_some()); + } + + #[test] + fn test_parse_received_new_header_with_ansi() { + let line = "[16eac416a3014ec191173b9e95cc11a6] \x1b[2m2026-05-07T04:28:26.009013Z\x1b[0m \x1b[32mINFO\x1b[0m \x1b[1m\x1b[0m: received new header \x1b[3mrelay_id\x1b[0m\x1b[2m=\x1b[0m\"mux_helix\" \x1b[3mheader_size_bytes\x1b[0m\x1b[2m=\x1b[0m2891 \x1b[3mlatency\x1b[0m\x1b[2m=\x1b[0m6.1415ms \x1b[3mversion\x1b[0m\x1b[2m=\x1b[0mFulu \x1b[3mvalue_eth\x1b[0m\x1b[2m=\x1b[0m\"0.050439063999832000\" \x1b[2m\x1b[3mmethod\x1b[0m\x1b[2m=\x1b[0m/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} \x1b[3mreq_id\x1b[0m\x1b[2m=\x1b[0m8e5020cb \x1b[3mslot\x1b[0m\x1b[2m=\x1b[0m521 \x1b[3mparent_hash\x1b[0m\x1b[2m=\x1b[0m0x969f22b3 \x1b[3mvalidator\x1b[0m\x1b[2m=\x1b[0m0x98213294\x1b[0m"; + + let event = parse_cb_log_line(line).expect("should parse"); + assert!(event.message.contains("received new header"), "message: {:?}", event.message); + assert_eq!(event.relay_id, Some("mux_helix".to_string())); + assert_eq!(event.slot, Some(521)); + } + + #[test] + fn test_mux_event_filter() { + // Test that the filter used in check_mux_routing matches parsed events + let lines = vec![ + "2026-05-07T04:28:26.004744Z DEBUG : using mux config mux_id=\"node_1_to_flashbots\" relays=1 pubkey=0x8ca49f0c slot=2 validator=0x8ca49f0c", + "2026-05-07T04:28:26.009013Z INFO : received new header relay_id=\"mux_helix\" header_size_bytes=2891 slot=521 validator=0x98213294", + "2026-05-07T04:28:26.011040Z INFO : auction winner relay_id=\"mux_helix\" value_eth=\"0.050439063999832000\" block_hash=0x15cd5f31 slot=521", + "2026-05-07T04:28:26.011056Z INFO : received header value_eth=\"0.050439063999832000\" block_hash=0x15cd5f31 slot=521", + ]; + + let events: Vec = lines.iter().filter_map(|l| parse_cb_log_line(l)).collect(); + assert_eq!(events.len(), 4, "all 4 lines should parse"); + + let mux_events: Vec<&CbEvent> = events + .iter() + .filter(|e| { + e.message.starts_with("using mux") + || e.message.starts_with("received new header") + || e.message.starts_with("auction winner") + }) + .collect(); + + assert_eq!(mux_events.len(), 3, "3 of 4 events should be mux-related (not 'received header')"); + } } diff --git a/src/discovery.rs b/src/discovery.rs index eb92bab..0759a94 100644 --- a/src/discovery.rs +++ b/src/discovery.rs @@ -20,11 +20,32 @@ pub struct PostMortemRecord { pub value: String, } +/// Derive a relay identity from the Kurtosis service name. +/// +/// Returns a short string like "helix", "flashbots", or "mev-rs" +/// that can be matched against mux entry IDs. +pub fn relay_identity(service_name: &str) -> Option { + let lower = service_name.to_lowercase(); + if lower.contains("helix") { + Some("helix".to_string()) + } else if lower.contains("mev-rs") { + Some("mev-rs".to_string()) + } else if lower.contains("relay") { + // Generic mev-boost relay (used by flashbots) + Some("flashbots".to_string()) + } else { + None + } +} + /// Discovered services from a Kurtosis enclave. #[derive(Debug, Default)] pub struct EnclaveServices { pub beacon_urls: Vec, pub relay_urls: Vec, + /// Parallel to relay_urls: identity string per relay + /// ("helix", "flashbots", "mev-rs", etc.) + pub relay_identities: Vec, pub cb_pbs_urls: Vec, pub cb_metrics_urls: Vec, pub cb_service_names: Vec, @@ -311,8 +332,10 @@ pub fn discover(enclave: &str) -> Result { .or_else(|| find_port("http")) .or_else(|| find_port("endpoint")); if let Some(url) = url { - info!("Relay API: {} -> {url}", svc.name); + let identity = relay_identity(&svc.name).unwrap_or_else(|| "unknown".to_string()); + info!("Relay API: {} -> {url} (identity={identity})", svc.name); result.relay_urls.push(url); + result.relay_identities.push(identity); } else { warn!("Relay '{}': no http/endpoint port", svc.name); } @@ -522,6 +545,19 @@ fn is_relay_api_service(name: &str) -> bool { #[cfg(test)] mod tests { + use super::relay_identity; + + #[test] + fn test_relay_identity() { + assert_eq!(relay_identity("helix-relay").as_deref(), Some("helix")); + assert_eq!(relay_identity("Helix-Relay").as_deref(), Some("helix")); + assert_eq!(relay_identity("mev-relay-api").as_deref(), Some("flashbots")); + assert_eq!(relay_identity("mev-rs-relay").as_deref(), Some("mev-rs")); + // Non-relay services: function should not be called for these + // in practice (is_relay_api_service filters them), but they + // won't match anything meaningful. + assert_eq!(relay_identity("prometheus"), None); + } use super::*; #[test] diff --git a/src/main.rs b/src/main.rs index d296e8f..8b47c60 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,35 +31,74 @@ use report::{ObservationWindow, VerificationReport}; const SLOTS_PER_EPOCH: u64 = 32; /// Verify Commit-Boost MEV pipeline in a Kurtosis devnet. +/// +/// Two modes of operation: +/// +/// 1. Attached: Run alongside a testnet launched by `run-and-verify.sh`. +/// The enclave is specified with --enclave and the verifier waits for +/// readiness, observes, and checks. +/// +/// 2. Standalone: Point the verifier at a running enclave with --enclave. +/// It checks whatever data is available and reports whether the pipeline +/// is healthy. Use --config to also verify mux routing rules. +/// +/// Examples: +/// # Attached mode (launches testnet + verifies) +/// ./scripts/run-and-verify.sh --config configs/cb-mux.yml +/// +/// # Standalone: quick health check (no observation window) +/// cb-verify --enclave CB-Testnet --min-epochs 0 +/// +/// # Standalone: full verification with mux checks +/// cb-verify --enclave CB-Testnet --config configs/cb-mux.yml +/// +/// # Standalone: show raw CB PBS logs for debugging +/// cb-verify --enclave CB-Testnet --show-logs +/// +/// # Standalone, just check current health (no observation window) +/// cb-verify --enclave CB-Testnet --min-epochs 0 #[derive(Parser, Debug)] #[command(name = "cb-verify", version, about)] struct Cli { - /// Kurtosis enclave name + /// Kurtosis enclave name. #[arg(long)] - enclave: String, + enclave: Option, - /// Observation window in epochs + /// Path to the Kurtosis config file (YAML). + /// + /// Used to extract the embedded Commit-Boost config for mux + /// verification. If no [[mux]] sections are found, the mux check + /// is skipped. + /// + /// The enclave name must be provided separately via --enclave. + #[arg(long)] + config: Option, + + /// Observation window in epochs. Set to 0 to skip the observation + /// window and run checks immediately against current state. #[arg(long, default_value_t = 2)] min_epochs: u64, - /// Wait until this epoch before starting checks + /// Wait until this epoch before starting checks. /// (default 7: genesis + validator activation + relay registration + builder warm-up) + /// If the enclave hasn't reached this epoch, the verifier will wait + /// up to --timeout seconds. #[arg(long, default_value_t = 7)] target_epoch: u64, - /// Max seconds to wait for devnet readiness + /// Max seconds to wait for devnet readiness. #[arg(long, default_value_t = 3600)] timeout: u64, - /// Minimum MEV delivery rate threshold + /// Minimum MEV delivery rate threshold. #[arg(long, default_value_t = 0.30)] mev_threshold: f64, - /// Output JSON report instead of terminal colors + /// Output JSON report instead of terminal colors. #[arg(long)] json: bool, - /// Enable debug logging + /// Enable debug logging. #[arg(short, long)] verbose: bool, @@ -80,18 +119,12 @@ struct Cli { #[arg(long)] live_metrics: bool, - /// Path to the Commit-Boost config file (TOML or Kurtosis YAML). - /// - /// When set, parses the config for [[mux]] sections and automatically - /// verifies that all delivered payloads conform to the mux routing - /// rules: validator pubkeys are delivered exclusively to their assigned - /// relay. Fails if any pubkey appears on the wrong relay. + /// Print raw CB PBS service logs to stdout for debugging. /// - /// If no [[mux]] sections are found in the config, the check is skipped. - /// Supports .toml (raw CB config) and .yml/.yaml (Kurtosis config with - /// embedded mev_params.commit_boost_config). + /// Fetches the last N log lines from each CB PBS service and prints them + /// in a human-readable format. Does not run any verification checks. #[arg(long)] - cb_config: Option, + show_logs: bool, /// Directory to save JSON report files. Requires --json. /// @@ -124,6 +157,16 @@ async fn main() -> Result<()> { std::process::exit(code); } +/// Resolve the enclave name and CB config from CLI args. +/// The enclave name is required. The config is optional and used for +/// mux verification. +fn resolve_enclave_and_config(cli: &Cli) -> Result<(String, Option)> { + let enclave = cli.enclave.clone().ok_or_else(|| { + eyre::eyre!("Must provide --enclave to specify a running enclave") + })?; + Ok((enclave, cli.config.clone())) +} + async fn run_verification(cli: &Cli) -> i32 { let now = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); @@ -134,13 +177,25 @@ async fn run_verification(cli: &Cli) -> i32 { } }; + // Step 0: Resolve enclave name and CB config + let (enclave_name, cb_config) = match resolve_enclave_and_config(cli) { + Ok(result) => result, + Err(e) => { + error!("{e}"); + let report = make_error_report("unknown", &now, &format!("{e}")); + report::print_report(&report, cli.json); + save_report(&report); + return 2; + } + }; + // Step 1: Discover services - info!("Discovering services in enclave '{}'...", cli.enclave); - let services = match discovery::discover(&cli.enclave) { + info!("Discovering services in enclave '{}'...", enclave_name); + let services = match discovery::discover(&enclave_name) { Ok(s) => s, Err(e) => { error!("Service discovery failed: {e}"); - let report = make_error_report(&cli.enclave, &now, &format!("Discovery failed: {e}")); + let report = make_error_report(&enclave_name, &now, &format!("Discovery failed: {e}")); report::print_report(&report, cli.json); save_report(&report); return 2; @@ -149,7 +204,7 @@ async fn run_verification(cli: &Cli) -> i32 { if services.beacon_urls.is_empty() { error!("No beacon nodes found in enclave"); - let report = make_error_report(&cli.enclave, &now, "No beacon nodes found"); + let report = make_error_report(&enclave_name, &now, "No beacon nodes found"); report::print_report(&report, cli.json); save_report(&report); return 2; @@ -163,6 +218,11 @@ async fn run_verification(cli: &Cli) -> i32 { info!("Relays: {:?}", services.relay_urls); info!("CB metrics: {}", metrics_url.unwrap_or("not available")); + // --show-logs mode: print raw CB PBS logs and exit + if cli.show_logs { + return show_cb_logs(&enclave_name, &services.cb_service_names, &now, cli.json, &save_report); + } + if relays.is_empty() { warn!("No relay URLs found -- relay checks will fail"); } @@ -170,7 +230,7 @@ async fn run_verification(cli: &Cli) -> i32 { // Step 2: Wait for readiness if !wait_for_readiness(&beacon, cli.target_epoch, cli.timeout).await { let report = make_error_report( - &cli.enclave, + &enclave_name, &now, &format!("Devnet did not stabilize within {}s", cli.timeout), ); @@ -235,7 +295,7 @@ async fn run_verification(cli: &Cli) -> i32 { if all_are_relays && relay_died { info!("Relay Data API unreachable — attempting post-mortem via Postgres..."); - let postmortem = discovery::query_mev_relay_postgres(&cli.enclave); + let postmortem = discovery::query_mev_relay_postgres(&enclave_name); if !postmortem.is_empty() { info!( "Post-mortem: found {} payload(s) in relay Postgres before crash:", @@ -258,7 +318,7 @@ async fn run_verification(cli: &Cli) -> i32 { } else { error!("Post-mortem: no delivery records found in relay Postgres."); let report = make_error_report( - &cli.enclave, + &enclave_name, &now, &format!( "Preflight failed ({} of {} services): {}. Relay API unreachable \ @@ -267,7 +327,7 @@ async fn run_verification(cli: &Cli) -> i32 { dead_at_preflight.len(), targets.len(), summary.join(", "), - cli.enclave + &enclave_name ), ); report::print_report(&report, cli.json); @@ -282,7 +342,7 @@ async fn run_verification(cli: &Cli) -> i32 { summary ); let report = make_error_report( - &cli.enclave, + &enclave_name, &now, &format!( "Preflight failed ({} of {} services): {}. Kurtosis port mapping may be \ @@ -290,7 +350,7 @@ async fn run_verification(cli: &Cli) -> i32 { dead_at_preflight.len(), targets.len(), summary.join(", "), - cli.enclave + &enclave_name ), ); report::print_report(&report, cli.json); @@ -300,50 +360,72 @@ async fn run_verification(cli: &Cli) -> i32 { } info!(" All {} service(s) reachable", targets.len()); - // Step 3: Observe for min_epochs, with periodic health checks across - // every critical service. + // Step 3: Observe for min_epochs (if > 0), with periodic health checks. + // + // When min_epochs is 0, skip the observation window entirely and run + // checks immediately against the current state. This is useful for + // standalone verification of a running enclave. // // Kurtosis happily leaves stopped containers in place -- when that happens // the service stops accepting connections but the enclave still looks // "up". Probing every ~30s means we fail fast instead of waiting out the // whole window and surfacing a ghost error at the end. - let sps = beacon.get_seconds_per_slot().await; - let obs_timeout = cli.min_epochs * SLOTS_PER_EPOCH * sps + 120; - let window = match observe_epochs( - &beacon, - &health_client, - &targets, - cli.min_epochs, - obs_timeout, - ObserveLiveOpts { - metrics_url, - live_metrics: cli.live_metrics, - json_output: cli.json, - }, - ) - .await - { - ObserveOutcome::Done(w) => w, - ObserveOutcome::ServiceDied { label, detail } => { - error!("{label} died during observation window -- aborting"); - let report = make_error_report( - &cli.enclave, - &now, - &format!( - "{label} went offline mid-observation: {detail}. Check: docker ps -a ; kurtosis enclave inspect {}", - cli.enclave - ), - ); - report::print_report(&report, cli.json); - save_report(&report); - return 2; + let window = if cli.min_epochs > 0 { + let sps = beacon.get_seconds_per_slot().await; + let obs_timeout = cli.min_epochs * SLOTS_PER_EPOCH * sps + 120; + match observe_epochs( + &beacon, + &health_client, + &targets, + cli.min_epochs, + obs_timeout, + ObserveLiveOpts { + metrics_url, + live_metrics: cli.live_metrics, + json_output: cli.json, + }, + ) + .await + { + ObserveOutcome::Done(w) => w, + ObserveOutcome::ServiceDied { label, detail } => { + error!("{label} died during observation window -- aborting"); + let report = make_error_report( + &enclave_name, + &now, + &format!( + "{label} went offline mid-observation: {detail}. Check: docker ps -a ; kurtosis enclave inspect {}", + enclave_name + ), + ); + report::print_report(&report, cli.json); + save_report(&report); + return 2; + } + ObserveOutcome::Timeout => { + let report = + make_error_report(&enclave_name, &now, "Failed to complete observation window"); + report::print_report(&report, cli.json); + save_report(&report); + return 2; + } } - ObserveOutcome::Timeout => { - let report = - make_error_report(&cli.enclave, &now, "Failed to complete observation window"); - report::print_report(&report, cli.json); - save_report(&report); - return 2; + } else { + // No observation window — use current slot as both start and end + let current_slot = match beacon.get_head_slot().await { + Ok(slot) => slot, + Err(e) => { + error!("Failed to get current slot: {e}"); + let report = make_error_report(&enclave_name, &now, &format!("Failed to get current slot: {e}")); + report::print_report(&report, cli.json); + save_report(&report); + return 2; + } + }; + info!("Skipping observation window (min_epochs=0), using current slot {}", current_slot); + ObservationWindow { + start_slot: current_slot, + end_slot: current_slot, } }; @@ -356,7 +438,7 @@ async fn run_verification(cli: &Cli) -> i32 { &beacon, window.start_slot, window.end_slot, - &cli.enclave, + &enclave_name, ) .await, ); @@ -408,15 +490,15 @@ async fn run_verification(cli: &Cli) -> i32 { checks::cb_metrics::run_metrics_checks( &http_client, metrics_url, - Some(cli.enclave.as_str()), + Some(enclave_name.as_str()), &services.cb_service_names, cli.strict, ) .await, ); - // MUX routing check (optional — requires --cb-config) - if let Some(ref cb_path) = cli.cb_config { + // MUX routing check (optional — requires config with [[mux]] sections) + if let Some(ref cb_path) = cb_config { info!("Checking for [[mux]] sections in CB config: {cb_path}..."); match checks::mux_routing::extract_mux_from_config(cb_path) { Ok(Some(mux_entries)) => { @@ -426,10 +508,9 @@ async fn run_verification(cli: &Cli) -> i32 { ); all_checks.push( checks::mux_routing::check_mux_routing( - &relays, + &enclave_name, + &services.cb_service_names, &mux_entries, - window.start_slot, - window.end_slot, ) .await, ); @@ -455,7 +536,7 @@ async fn run_verification(cli: &Cli) -> i32 { .any(|c| c.tier == 1 && c.status == CheckStatus::Fail); let report = VerificationReport { - enclave: cli.enclave.clone(), + enclave: enclave_name.clone(), timestamp: now, observation_window: Some(window), result: if tier1_failed { @@ -515,9 +596,13 @@ async fn wait_for_readiness(beacon: &BeaconClient, target_epoch: u64, timeout: u if let (Some(_), Some(fin)) = (head, finalized) && fin >= 2 - && current_epoch >= target_epoch + && current_epoch + 1 >= target_epoch { - info!("Devnet is ready."); + if current_epoch >= target_epoch { + info!("Devnet is ready."); + } else { + info!("Devnet is close enough (epoch {current_epoch}, target {target_epoch}). Proceeding..."); + } return true; } @@ -655,6 +740,78 @@ async fn observe_epochs( } } +/// Fetch and print raw CB PBS service logs for debugging. +fn show_cb_logs( + enclave_name: &str, + cb_service_names: &[String], + now: &str, + json_mode: bool, + save_report: &dyn Fn(&VerificationReport), +) -> i32 { + use crate::checks::mux_routing::{parse_cb_log_line, fetch_service_logs}; + + println!("\n=== CB PBS Service Logs ==="); + println!("Enclave: {enclave_name}"); + println!("Services: {}\n", cb_service_names.join(", ")); + + let mut total_events = 0; + let mut parsed_events = 0; + + for service_name in cb_service_names { + println!("--- {service_name} ---"); + match fetch_service_logs(enclave_name, service_name) { + Ok(logs) => { + if logs.is_empty() { + println!(" (no relevant log lines)"); + continue; + } + for line in logs.lines() { + total_events += 1; + if let Some(event) = parse_cb_log_line(line) { + parsed_events += 1; + print!(" [{}] {}", event.message, event.slot.map(|s| format!("slot={}", s)).unwrap_or_default()); + if let Some(ref mux) = event.mux_id { + print!(" mux={}", mux); + } + if let Some(ref relay) = event.relay_id { + print!(" relay={}", relay); + } + if let Some(ref val) = event.validator { + let short = if val.len() > 20 { &val[..20] } else { val }; + print!(" val={}...", short); + } + println!(); + } else { + // Print raw line if parsing failed + let short = if line.len() > 120 { &line[..120] } else { line }; + println!(" [RAW] {}...", short); + } + } + } + Err(e) => { + println!(" ERROR: {e}"); + } + } + } + + println!("\nTotal: {} log lines, {} parsed successfully", total_events, parsed_events); + + let report = VerificationReport { + enclave: enclave_name.to_string(), + timestamp: now.to_string(), + observation_window: None, + result: CheckStatus::Pass, + checks: vec![CheckResult::pass( + "logs", + 1, + format!("Fetched {} log lines from {} service(s)", total_events, cb_service_names.len()), + )], + }; + report::print_report(&report, json_mode); + save_report(&report); + 0 +} + fn make_error_report(enclave: &str, timestamp: &str, detail: &str) -> VerificationReport { VerificationReport { enclave: enclave.to_string(), diff --git a/src/orchestrator.rs b/src/orchestrator.rs new file mode 100644 index 0000000..459a357 --- /dev/null +++ b/src/orchestrator.rs @@ -0,0 +1,869 @@ +//! cb-orchestrator: Run multiple Commit-Boost config scenarios concurrently. +//! +//! Replaces `just test-all` with a concurrent pipeline: +//! +//! Launch enclaves ──► Wait for readiness ──► Observe ──► Check ──► Tear down +//! │ │ │ │ │ +//! └────────────────────┴───────────────────┴──────────┴──────────┘ +//! All enclaves run through the pipeline concurrently (bounded by --jobs) +//! +//! Each enclave is independent. While one is observing, another can be launching. +//! The bottleneck is the observation window (~2 epochs ≈ 12 min), so with --jobs=4 +//! you get roughly 4× throughput vs sequential `just test-all`. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +use clap::Parser; +use eyre::{bail, Context, Result}; +use serde::Serialize; +use tokio::sync::Semaphore; +use tokio::task::JoinSet; +use tracing::{error, info, warn}; + +// For colored output in print_batch_summary +use colored::Colorize; + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +#[derive(Parser, Debug)] +#[command(name = "cb-orchestrator", version, about)] +struct Cli { + /// Config files to run (YAML Kurtosis configs). + /// + /// Accepts individual files or directories (all *.yml in directory). + /// If omitted, defaults to configs/generated/*.yml. + #[arg(value_name = "CONFIG")] + configs: Vec, + + /// Max concurrent enclaves (default: 2). + /// + /// Each enclave uses ~2-4 GB RAM and 2-4 CPU cores. A 16-core/32GB + /// machine can comfortably run 4-6 concurrent enclaves. + #[arg(long, default_value_t = 2)] + jobs: usize, + + /// Kurtosis package path or ref. + #[arg(long, default_value = "./ethereum-package")] + package: String, + + /// Observation window in epochs. + #[arg(long, default_value_t = 2)] + min_epochs: u64, + + /// Wait until this epoch before starting observation. + #[arg(long, default_value_t = 7)] + target_epoch: u64, + + /// Readiness timeout in seconds. + #[arg(long, default_value_t = 3600)] + timeout: u64, + + /// Save JSON reports to this directory. + #[arg(long)] + results_dir: Option, + + /// Keep enclaves running after checks (don't tear down). + #[arg(long)] + keep: bool, + + /// Strict mode: promote WARN to FAIL. + #[arg(long)] + strict: bool, + + /// Live metrics polling during observation. + #[arg(long)] + live_metrics: bool, + + /// Verbose logging. + #[arg(short, long)] + verbose: bool, +} + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/// The lifecycle state of a single enclave run. +#[derive(Debug, Clone, PartialEq)] +enum EnclaveState { + /// `kurtosis run` has been launched, waiting for containers to start. + Launching, + /// Containers are up, waiting for beacon to reach target_epoch. + WaitingForReadiness, + /// Beacon is ready, observing for min_epochs. + Observing, + /// Running cb-verify checks. + Checking, + /// All checks complete. + Done, + /// Failed at some point. + Failed(String), +} + +/// Per-enclave status tracked by the orchestrator. +#[derive(Debug, Clone)] +struct EnclaveStatus { + name: String, + config: PathBuf, + state: EnclaveState, + /// Set when the enclave process has been launched. + launched_at: Option, + /// Set when the enclave becomes ready for observation. + ready_at: Option, + /// Set when observation completes. + observed_at: Option, + /// Set when checks complete. + checked_at: Option, + /// Check results (populated after Done). + check_result: Option, +} + +/// Summarized check result for the final report. +#[derive(Debug, Clone, Serialize)] +struct CheckSummary { + enclave: String, + config: String, + result: String, + passed: usize, + failed: usize, + warnings: usize, + skipped: usize, + duration_secs: u64, +} + +/// Final batch report. +#[derive(Debug, Serialize)] +struct BatchReport { + timestamp: String, + total: usize, + passed: usize, + failed: usize, + results: Vec, +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +#[tokio::main] +async fn main() -> Result<()> { + color_eyre::install()?; + let cli = Cli::parse(); + + // Initialize tracing + let filter = if cli.verbose { + "debug,hyper=info,reqwest=info,rustls=info" + } else { + "info" + }; + tracing_subscriber::fmt() + .with_env_filter(filter) + .with_target(false) + .init(); + + // Resolve config files + let configs = resolve_configs(&cli.configs)?; + if configs.is_empty() { + bail!("No config files found. Pass config files or directories as arguments."); + } + + info!( + "Orchestrator: {} config(s), {} concurrent job(s)", + configs.len(), + cli.jobs + ); + + // Create results dir if needed + if let Some(ref dir) = cli.results_dir { + std::fs::create_dir_all(dir)?; + } + + // Build enclave names from config filenames + let enclaves: Vec = configs + .iter() + .map(|config| { + let name = enclave_name(config); + EnclaveStatus { + name, + config: config.clone(), + state: EnclaveState::Launching, + launched_at: None, + ready_at: None, + observed_at: None, + checked_at: None, + check_result: None, + } + }) + .collect(); + + // Clean up any stale enclaves with the same names + for enc in &enclaves { + info!("Cleaning stale enclave '{}' (if any)...", enc.name); + let _ = Command::new("kurtosis") + .args(["enclave", "rm", "-f", &enc.name]) + .output(); + } + + // Semaphore to bound concurrency + let semaphore = std::sync::Arc::new(Semaphore::new(cli.jobs)); + + // Spawn all enclave pipelines concurrently + let mut join_set = JoinSet::new(); + + for (idx, enc) in enclaves.into_iter().enumerate() { + let sem = semaphore.clone(); + let package = cli.package.clone(); + let results_dir = cli.results_dir.clone(); + let min_epochs = cli.min_epochs; + let target_epoch = cli.target_epoch; + let timeout = cli.timeout; + let keep = cli.keep; + let strict = cli.strict; + let live_metrics = cli.live_metrics; + let verbose = cli.verbose; + + join_set.spawn(async move { + let _permit = sem.acquire().await.expect("semaphore closed"); + let result = run_enclave_pipeline( + enc, + &package, + min_epochs, + target_epoch, + timeout, + keep, + strict, + live_metrics, + verbose, + results_dir.as_deref(), + ) + .await; + (idx, result) + }); + } + + // Collect results + let mut results: Vec<(usize, Result)> = Vec::new(); + while let Some(res) = join_set.join_next().await { + match res { + Ok((idx, result)) => results.push((idx, result)), + Err(e) => { + error!("Task panicked: {e}"); + results.push(( + usize::MAX, + Err(eyre::eyre!("Task panicked: {e}")), + )); + } + } + } + + // Sort by original index for deterministic output + results.sort_by_key(|(idx, _)| *idx); + + // Build summaries + let mut summaries: Vec = Vec::new(); + let mut total_passed = 0usize; + let mut total_failed = 0usize; + + for (_, result) in &results { + match result { + Ok(enc) => { + if let Some(ref summary) = enc.check_result { + if summary.result == "PASS" { + total_passed += 1; + } else { + total_failed += 1; + } + summaries.push(summary.clone()); + } else { + total_failed += 1; + summaries.push(CheckSummary { + enclave: enc.name.clone(), + config: enc.config.display().to_string(), + result: "FAILED".to_string(), + passed: 0, + failed: 0, + warnings: 0, + skipped: 0, + duration_secs: 0, + }); + } + } + Err(e) => { + total_failed += 1; + summaries.push(CheckSummary { + enclave: "unknown".to_string(), + config: "unknown".to_string(), + result: format!("ERROR: {e}"), + passed: 0, + failed: 0, + warnings: 0, + skipped: 0, + duration_secs: 0, + }); + } + } + } + + // Print batch summary + let batch = BatchReport { + timestamp: chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + total: summaries.len(), + passed: total_passed, + failed: total_failed, + results: summaries, + }; + + print_batch_summary(&batch); + + // Save batch report if requested + if let Some(ref dir) = cli.results_dir { + let report_path = dir.join("batch-report.json"); + match serde_json::to_string_pretty(&batch) { + Ok(json) => { + if let Err(e) = std::fs::write(&report_path, &json) { + warn!("Failed to write batch report: {e}"); + } else { + info!("Batch report saved to {}", report_path.display()); + } + } + Err(e) => warn!("Failed to serialize batch report: {e}"), + } + } + + // Exit code: 0 if all passed, 1 if any failed + if total_failed > 0 { + std::process::exit(1); + } + std::process::exit(0); +} + +// --------------------------------------------------------------------------- +// Per-enclave pipeline +// --------------------------------------------------------------------------- + +async fn run_enclave_pipeline( + mut enc: EnclaveStatus, + package: &str, + min_epochs: u64, + target_epoch: u64, + timeout: u64, + keep: bool, + strict: bool, + live_metrics: bool, + verbose: bool, + results_dir: Option<&Path>, +) -> Result { + let start = Instant::now(); + + // Phase 1: Launch + info!("[{}] Launching enclave with config {}...", enc.name, enc.config.display()); + enc.state = EnclaveState::Launching; + enc.launched_at = Some(Instant::now()); + + if let Err(e) = launch_enclave(&enc.name, &enc.config, package).await { + let msg = format!("Launch failed: {e}"); + error!("[{}] {}", enc.name, msg); + enc.state = EnclaveState::Failed(msg.clone()); + // Try to clean up + if !keep { let _ = teardown_enclave(&enc.name); } + bail!(msg); + } + + // Phase 2: Wait for readiness + info!("[{}] Waiting for readiness (target epoch {target_epoch})...", enc.name); + enc.state = EnclaveState::WaitingForReadiness; + + if let Err(e) = wait_for_enclave_readiness(&enc.name, target_epoch, timeout).await { + let msg = format!("Readiness timeout: {e}"); + error!("[{}] {}", enc.name, msg); + enc.state = EnclaveState::Failed(msg.clone()); + if !keep { let _ = teardown_enclave(&enc.name); } + bail!(msg); + } + enc.ready_at = Some(Instant::now()); + info!( + "[{}] Enclave ready after {:?}", + enc.name, + enc.ready_at.unwrap().duration_since(enc.launched_at.unwrap()) + ); + + // Phase 3: Observe + info!("[{}] Observing for {min_epochs} epoch(s)...", enc.name); + enc.state = EnclaveState::Observing; + + if let Err(e) = observe_enclave(&enc.name, min_epochs, target_epoch).await { + let msg = format!("Observation failed: {e}"); + error!("[{}] {}", enc.name, msg); + enc.state = EnclaveState::Failed(msg.clone()); + if !keep { let _ = teardown_enclave(&enc.name); } + bail!(msg); + } + enc.observed_at = Some(Instant::now()); + + // Phase 4: Run checks + info!("[{}] Running checks...", enc.name); + enc.state = EnclaveState::Checking; + + let check_result = run_checks( + &enc.name, + &enc.config, + results_dir, + strict, + live_metrics, + verbose, + ) + .await; + + enc.checked_at = Some(Instant::now()); + + match check_result { + Ok(summary) => { + let result_str = summary.result.clone(); + enc.check_result = Some(summary); + enc.state = EnclaveState::Done; + info!("[{}] Checks complete: {} (total {:?})", enc.name, result_str, enc.checked_at.unwrap().duration_since(start)); + } + Err(e) => { + let msg = format!("Check execution failed: {e}"); + error!("[{}] {}", enc.name, msg); + enc.state = EnclaveState::Failed(msg); + } + } + + // Phase 5: Tear down + if !keep { + info!("[{}] Tearing down enclave...", enc.name); + if let Err(e) = teardown_enclave(&enc.name) { + warn!("[{}] Teardown error (non-fatal): {e}", enc.name); + } + } else { + info!("[{}] Keeping enclave running (--keep)", enc.name); + } + + Ok(enc) +} + +// --------------------------------------------------------------------------- +// Phase implementations +// --------------------------------------------------------------------------- + +/// Phase 1: Launch a Kurtosis enclave. +async fn launch_enclave(name: &str, config: &Path, package: &str) -> Result<()> { + let output = tokio::process::Command::new("kurtosis") + .args([ + "run", + package, + "--enclave", + name, + "--args-file", + &config.display().to_string(), + "--image-download", + "always", + ]) + .output() + .await + .wrap_err("Failed to run kurtosis")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + bail!("kurtosis run failed: {}", stderr.trim()); + } + + Ok(()) +} + +/// Phase 2: Wait for the enclave's beacon to reach the target epoch. +async fn wait_for_enclave_readiness( + name: &str, + target_epoch: u64, + timeout_secs: u64, +) -> Result<()> { + let start = Instant::now(); + let timeout = Duration::from_secs(timeout_secs); + let poll_interval = Duration::from_secs(10); + + // Discover the beacon URL + let beacon_url = discover_beacon_url(name).await?; + + loop { + if start.elapsed() >= timeout { + bail!("Timeout waiting for enclave readiness after {timeout_secs}s"); + } + + // Query beacon head slot via the standard Beacon API + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build()?; + + let url = format!("{beacon_url}/eth/v1/beacon/headers/head"); + match client.get(&url).send().await { + Ok(resp) => { + if let Ok(json) = resp.json::().await { + if let Some(slot) = json + .get("data") + .and_then(|d| d.get("header")) + .and_then(|h| h.get("message")) + .and_then(|m| m.get("slot")) + .and_then(|s| s.as_str()) + .and_then(|s| s.parse::().ok()) + { + let epoch = slot / 32; + if epoch >= target_epoch { + return Ok(()); + } + tracing::debug!( + "[{}] Beacon at epoch {epoch}, waiting for {target_epoch}...", + name + ); + } + } + } + Err(e) => { + tracing::debug!("[{}] Beacon not reachable yet: {e}", name); + } + } + + tokio::time::sleep(poll_interval).await; + } +} + +/// Phase 3: Observe the enclave for min_epochs. +/// +/// Polls the beacon head slot and waits until min_epochs have passed since +/// the enclave became ready. Also does periodic health probes. +async fn observe_enclave(name: &str, min_epochs: u64, target_epoch: u64) -> Result<()> { + let beacon_url = discover_beacon_url(name).await?; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build()?; + + let start_slot = target_epoch * 32; + let target_slot = start_slot + (min_epochs * 32); + let poll_interval = Duration::from_secs(5); + + info!( + "[{}] Observing: slot {start_slot} -> {target_slot} ({min_epochs} epochs)", + name + ); + + let start = Instant::now(); + let timeout = Duration::from_secs(min_epochs * 32 * 12 + 120); // ~12s per slot + buffer + + loop { + if start.elapsed() >= timeout { + bail!("Observation timeout"); + } + + let url = format!("{beacon_url}/eth/v1/beacon/headers/head"); + match client.get(&url).send().await { + Ok(resp) => { + if let Ok(json) = resp.json::().await { + if let Some(slot) = json + .get("data") + .and_then(|d| d.get("header")) + .and_then(|h| h.get("message")) + .and_then(|m| m.get("slot")) + .and_then(|s| s.as_str()) + .and_then(|s| s.parse::().ok()) + { + if slot >= target_slot { + info!("[{}] Observation complete: slot {start_slot} -> {slot}", name); + return Ok(()); + } + } + } + } + Err(e) => { + warn!("[{}] Health probe failed during observation: {e}", name); + } + } + + tokio::time::sleep(poll_interval).await; + } +} + +/// Phase 4: Run cb-verify against the enclave. +async fn run_checks( + name: &str, + config: &Path, + results_dir: Option<&Path>, + strict: bool, + live_metrics: bool, + verbose: bool, +) -> Result { + // Find the cb-verify binary (built from the same crate) + let manifest_path = std::env::var("CARGO_MANIFEST_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| PathBuf::from(".")); + let binary_path = manifest_path.join("target/release/cb-verify"); + + if !binary_path.exists() { + bail!("cb-verify binary not found at {}. Run 'cargo build --release' first.", binary_path.display()); + } + + let mut cmd = tokio::process::Command::new(&binary_path); + cmd.arg("--enclave").arg(name); + cmd.arg("--cb-config").arg(config); + cmd.arg("--json"); + cmd.arg("--timeout").arg("3600"); + cmd.arg("--min-epochs").arg("0"); // Already observed + cmd.arg("--target-epoch").arg("0"); // Already ready + + if let Some(dir) = results_dir { + cmd.arg("--output-dir").arg(dir); + } + if strict { + cmd.arg("--strict"); + } + if live_metrics { + cmd.arg("--live-metrics"); + } + if verbose { + cmd.arg("-v"); + } + + let output = cmd + .output() + .await + .wrap_err("Failed to run cb-verify")?; + + // Parse the JSON report from stdout + let stdout = String::from_utf8_lossy(&output.stdout); + + // Find the JSON blob (cb-verify may print non-JSON lines before it) + let json_start = stdout + .find("{\n") + .or_else(|| stdout.find("{\"")) + .unwrap_or(0); + let json_str = &stdout[json_start..]; + + let report: serde_json::Value = serde_json::from_str(json_str) + .wrap_err("Failed to parse cb-verify JSON output")?; + + let result = report + .get("result") + .and_then(|r| r.as_str()) + .unwrap_or("unknown") + .to_string(); + + let checks = report + .get("checks") + .and_then(|c| c.as_array()) + .map(|c| c.as_slice()) + .unwrap_or(&[]); + + let passed = checks + .iter() + .filter(|c| c.get("status").and_then(|s| s.as_str()) == Some("Pass")) + .count(); + let failed = checks + .iter() + .filter(|c| c.get("status").and_then(|s| s.as_str()) == Some("Fail")) + .count(); + let warnings = checks + .iter() + .filter(|c| c.get("status").and_then(|s| s.as_str()) == Some("Warn")) + .count(); + let skipped = checks + .iter() + .filter(|c| c.get("status").and_then(|s| s.as_str()) == Some("Skip")) + .count(); + + Ok(CheckSummary { + enclave: name.to_string(), + config: config.display().to_string(), + result, + passed, + failed, + warnings, + skipped, + duration_secs: 0, // Will be filled in by caller + }) +} + +/// Phase 5: Tear down an enclave. +fn teardown_enclave(name: &str) -> Result<()> { + let output = Command::new("kurtosis") + .args(["enclave", "rm", "-f", name]) + .output() + .wrap_err("Failed to run kurtosis enclave rm")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + bail!("kurtosis enclave rm failed: {}", stderr.trim()); + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Discover the beacon HTTP URL for an enclave by querying kurtosis port print. +async fn discover_beacon_url(enclave: &str) -> Result { + // Try common beacon service names + let beacon_names = ["cl-1-lighthouse", "cl-1-prysm", "cl-1-teku", "cl-1-nimbus", "cl-1-lodestar"]; + + for name_prefix in &beacon_names { + // Try to find the full service name + let output = tokio::process::Command::new("kurtosis") + .args(["enclave", "inspect", "--full-uuids", enclave]) + .output() + .await + .wrap_err("kurtosis enclave inspect failed")?; + + let stdout = String::from_utf8_lossy(&output.stdout); + for line in stdout.lines() { + let lower = line.to_lowercase(); + if lower.contains(name_prefix) && lower.contains("running") { + // Found a running beacon service, get its HTTP port + let service_name = line + .split_whitespace() + .next() + .unwrap_or("") + .trim() + .to_string(); + + if service_name.is_empty() { + continue; + } + + // Try to get the HTTP port URL + let port_output = tokio::process::Command::new("kurtosis") + .args(["port", "print", enclave, &service_name, "http"]) + .output() + .await; + + if let Ok(port_out) = port_output { + let url = String::from_utf8_lossy(&port_out.stdout).trim().to_string(); + if !port_out.status.success() || url.is_empty() { + // Try "cl-http" as port name + let port_output2 = tokio::process::Command::new("kurtosis") + .args(["port", "print", enclave, &service_name, "cl-http"]) + .output() + .await; + if let Ok(port_out2) = port_output2 { + let url2 = + String::from_utf8_lossy(&port_out2.stdout).trim().to_string(); + if !url2.is_empty() { + return Ok(url2); + } + } + continue; + } + return Ok(url); + } + } + } + } + + // Fallback: try to use the enclave's default beacon port + bail!("Could not discover beacon URL for enclave '{enclave}'") +} + +/// Derive an enclave name from a config filename. +fn enclave_name(config: &Path) -> String { + let stem = config + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown"); + + // Strip common prefixes like "cb-" for cleaner names + let name = stem.strip_prefix("cb-").unwrap_or(stem); + format!("CB-{name}") +} + +/// Resolve config file paths: expand directories to *.yml files. +fn resolve_configs(inputs: &[PathBuf]) -> Result> { + let mut configs = Vec::new(); + + if inputs.is_empty() { + // Default: configs/generated/*.yml + let default_dir = PathBuf::from("configs/generated"); + if default_dir.is_dir() { + for entry in std::fs::read_dir(&default_dir)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) == Some("yml") + || path.extension().and_then(|e| e.to_str()) == Some("yaml") + { + configs.push(path); + } + } + } + configs.sort(); + return Ok(configs); + } + + for input in inputs { + if input.is_dir() { + for entry in std::fs::read_dir(input)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) == Some("yml") + || path.extension().and_then(|e| e.to_str()) == Some("yaml") + { + configs.push(path); + } + } + } else if input.is_file() { + configs.push(input.clone()); + } else { + warn!("Skipping non-existent path: {}", input.display()); + } + } + + configs.sort(); + Ok(configs) +} + +// --------------------------------------------------------------------------- +// Output +// --------------------------------------------------------------------------- + +fn print_batch_summary(batch: &BatchReport) { + println!(); + println!("╔══════════════════════════════════════════════════════════════╗"); + println!("║ Batch Verification Report ║"); + println!("╠══════════════════════════════════════════════════════════════╣"); + println!( + "║ Time: {:48} ║", + batch.timestamp + ); + println!( + "║ Total: {:48} ║", + batch.total + ); + println!( + "║ Passed: {:48} ║", + batch.passed.to_string().green() + ); + println!( + "║ Failed: {:48} ║", + batch.failed.to_string().red() + ); + println!("╠══════════════════════════════════════════════════════════════╣"); + + for result in &batch.results { + let status_icon = match result.result.as_str() { + "PASS" => "✓".green(), + "FAIL" => "✗".red(), + _ => "?".yellow(), + }; + let name = Path::new(&result.config) + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or(&result.config); + println!( + "║ {} {:20} {:6} ({}p / {}f / {}w / {}s) ║", + status_icon, name, result.result, result.passed, result.failed, result.warnings, result.skipped + ); + } + + println!("╚══════════════════════════════════════════════════════════════╝"); +} diff --git a/src/relay.rs b/src/relay.rs index fff9427..ddbd24c 100644 --- a/src/relay.rs +++ b/src/relay.rs @@ -56,33 +56,28 @@ impl RelayClient { /// /// Returns payloads delivered in the given slot range. /// - /// The relay API supports a `cursor` param (not in the standard alloy query type) - /// which acts as the upper bound slot. We pass it as a raw query param alongside - /// the typed query fields. + /// The relay API's `cursor` param is an opaque pagination token / DB ID, not a slot + /// filter — passing a slot number produces unreliable results. Instead we request a + /// generous count (no cursor) and let the client-side filter narrow to the window. pub async fn get_payloads_delivered( &self, start_slot: u64, end_slot: u64, ) -> Result> { - let limit = end_slot.saturating_sub(start_slot) + 1; - let resp: Vec = self .client .get(format!( "{}/relay/v1/data/bidtraces/proposer_payload_delivered", self.base_url )) - .query(&[ - ("cursor", end_slot.to_string()), - ("limit", limit.to_string()), - ]) + .query(&[("limit", "10000")]) .send() .await? .error_for_status()? .json() .await?; - // Filter to our slot range + // Filter strictly to our slot range Ok(resp .into_iter() .filter(|p| p.slot >= start_slot && p.slot <= end_slot) From 831de5dbc17c0f207bed9d9c953be54cc8ecd928 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 11:47:54 -0700 Subject: [PATCH 06/20] working run --- Cargo.toml | 4 + src/bin/test_relay.rs | 226 +++++++++++++++++++++++++++++++++++ src/checks/relay_pipeline.rs | 50 +++++--- src/relay.rs | 67 ++++++++--- 4 files changed, 313 insertions(+), 34 deletions(-) create mode 100644 src/bin/test_relay.rs diff --git a/Cargo.toml b/Cargo.toml index 645634c..87b31ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,10 @@ path = "src/orchestrator.rs" name = "test-mux" path = "src/bin/test_mux.rs" +[[bin]] +name = "test-relay" +path = "src/bin/test_relay.rs" + [dependencies] alloy-primitives = "^1.3.1" diff --git a/src/bin/test_relay.rs b/src/bin/test_relay.rs new file mode 100644 index 0000000..17e6876 --- /dev/null +++ b/src/bin/test_relay.rs @@ -0,0 +1,226 @@ +//! Quick relay API diagnostic. +//! +//! Tests all relay data API endpoints with proper slot filtering. +//! No observation window needed — just query the relay directly. +//! +//! Usage: +//! cargo run --release --bin test_relay -- [pubkey] +//! +//! Examples: +//! cargo run --release --bin test_relay -- http://127.0.0.1:59945 128 160 +//! cargo run --release --bin test_relay -- http://127.0.0.1:59945 128 160 0x889dbdf3bd68af1f6fd84cb6173b1fa1f7c5e6ba63297dc1e2f45cd1a82bb6231ba832adc5228143c5cff3ef0b1caae2 + +use std::time::Duration; + +#[tokio::main] +async fn main() { + let args: Vec = std::env::args().collect(); + if args.len() < 4 { + eprintln!("Usage: {} [pubkey]", args[0]); + eprintln!("Example: {} http://127.0.0.1:59945 128 160", args[0]); + std::process::exit(1); + } + + let relay_url = &args[1]; + let start_slot: u64 = args[2].parse().expect("invalid start_slot"); + let end_slot: u64 = args[3].parse().expect("invalid end_slot"); + let pubkey = args.get(4).map(|s| s.as_str()); + + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .expect("failed to build HTTP client"); + + let base = relay_url.trim_end_matches('/'); + + // === 0. Check if relay has ANY delivered payloads at all === + println!("=== 0. Latest delivered payloads (no slot filter) ==="); + { + let url = format!("{base}/relay/v1/data/bidtraces/proposer_payload_delivered"); + let req = client.get(&url).query(&[("limit", "5")]); + match send::>(req).await { + Ok(payloads) => { + println!(" Total payloads available: {}", payloads.len()); + for p in &payloads { + println!(" slot={} value={} proposer={}...", p.slot, p.value, &p.proposer_pubkey[..20.min(p.proposer_pubkey.len())]); + } + } + Err(e) => println!(" FAIL: {e}"), + } + } + + // === 1. Delivered payloads filtered by slot === + println!("=== 1. Delivered payloads (slot {start_slot}..={end_slot}) ==="); + { + let url = format!("{base}/relay/v1/data/bidtraces/proposer_payload_delivered"); + let mut all = Vec::new(); + let mut cursor: Option = None; + let mut page = 0; + let limit_str = String::from("200"); + let start_slot_str = start_slot.to_string(); + + loop { + page += 1; + let mut params: Vec<(&str, String)> = vec![ + ("limit", limit_str.clone()), + ("slot", start_slot_str.clone()), + ]; + if let Some(ref c) = cursor { + params.push(("cursor", c.clone())); + } + + let req = client.get(&url).query(¶ms); + match send::>(req).await { + Ok(payloads) => { + if payloads.is_empty() { + println!(" Page {page}: empty, done"); + break; + } + let min_slot = payloads.iter().filter_map(|p| p.slot.parse::().ok()).min().unwrap_or(0); + let max_slot = payloads.iter().filter_map(|p| p.slot.parse::().ok()).max().unwrap_or(0); + let page_len = payloads.len(); + let in_range: Vec<_> = payloads.into_iter().filter(|p| { + let s: u64 = p.slot.parse().unwrap_or(0); + s >= start_slot && s <= end_slot + }).collect(); + let count = in_range.len(); + all.extend(in_range); + println!(" Page {page}: {page_len} payloads (slots {min_slot}..={max_slot}), {count} in range, {} total", + all.len()); + if min_slot < start_slot { break; } + cursor = all.last().map(|p: &PayloadDelivered| p.block_number.clone()); + } + Err(e) => { + println!(" FAIL: {e}"); + break; + } + } + if page >= 50 { break; } + } + println!(" Total delivered in range: {}", all.len()); + for p in all.iter().take(5) { + let pk_short = if p.proposer_pubkey.len() > 20 { &p.proposer_pubkey[..20] } else { &p.proposer_pubkey }; + println!(" slot={} value={} proposer={}...", p.slot, p.value, pk_short); + } + } + + // === 2. Builder blocks received filtered by slot === + println!("\n=== 2. Builder blocks received (slot {start_slot}..={end_slot}) ==="); + { + let url = format!("{base}/relay/v1/data/bidtraces/builder_blocks_received"); + let mut all = Vec::new(); + + // Query each slot individually (the API supports slot filter) + for slot in start_slot..=end_slot { + let slot_str = slot.to_string(); + let limit_str = "200".to_string(); + let req = client.get(&url).query(&[("slot", &slot_str), ("limit", &limit_str)]); + match send::>(req).await { + Ok(blocks) => { + if !blocks.is_empty() { + println!(" Slot {slot}: {} blocks", blocks.len()); + for b in &blocks { + let val_short = if b.value.len() > 12 { &b.value[..12] } else { &b.value }; + println!(" builder={}... value={val_short}", &b.builder_pubkey[..20.min(b.builder_pubkey.len())]); + } + all.extend(blocks); + } + } + Err(e) => { + println!(" Slot {slot}: FAIL: {e}"); + } + } + } + println!(" Total builder blocks in range: {}", all.len()); + } + + // === 3. Validator registration check === + println!("\n=== 3. Validator registration ==="); + if let Some(pk) = pubkey { + let url = format!("{base}/relay/v1/data/validator_registration"); + let req = client.get(&url).query(&[("pubkey", pk)]); + match send::(req).await { + Ok(reg) => { + println!(" Registered: YES"); + if let Some(msg) = reg.get("message") { + println!(" Fee recipient: {}", msg.get("fee_recipient").and_then(|v| v.as_str()).unwrap_or("?")); + println!(" Gas limit: {}", msg.get("gas_limit").and_then(|v| v.as_str()).unwrap_or("?")); + println!(" Timestamp: {}", msg.get("timestamp").and_then(|v| v.as_str()).unwrap_or("?")); + } + } + Err(e) => { + println!(" Registered: NO ({e})"); + } + } + } else { + println!(" (skipped — no pubkey provided)"); + } + + // === 4. Summary === + println!("\n=== Summary ==="); + println!("Relay: {base}"); + println!("Slot range: {start_slot}..={end_slot} ({} slots)", end_slot - start_slot + 1); + println!("All queries completed successfully"); +} + +async fn send(req: reqwest::RequestBuilder) -> Result { + let resp = req.send().await.map_err(|e| format!("HTTP error: {e}"))?; + let status = resp.status(); + if !status.is_success() { + let body = resp.text().await.unwrap_or_default(); + return Err(format!("HTTP {}: {}", status, body)); + } + resp.json::().await.map_err(|e| format!("JSON error: {e}")) +} + +#[derive(serde::Deserialize, Debug, Clone)] +struct PayloadDelivered { + slot: String, + block_hash: String, + value: String, + #[serde(default)] + proposer_pubkey: String, + #[serde(default)] + builder_pubkey: String, + #[serde(default)] + block_number: String, + #[serde(default)] + parent_hash: String, + #[serde(default)] + proposer_fee_recipient: String, + #[serde(default)] + gas_limit: String, + #[serde(default)] + gas_used: String, + #[serde(default)] + num_tx: String, +} + +#[derive(serde::Deserialize, Debug, Clone)] +struct BuilderBlock { + slot: String, + block_hash: String, + value: String, + #[serde(default)] + builder_pubkey: String, + #[serde(default)] + proposer_pubkey: String, + #[serde(default)] + block_number: String, + #[serde(default)] + parent_hash: String, + #[serde(default)] + proposer_fee_recipient: String, + #[serde(default)] + gas_limit: String, + #[serde(default)] + gas_used: String, + #[serde(default)] + num_tx: String, + #[serde(default)] + timestamp: String, + #[serde(default)] + timestamp_ms: String, +} + +// Add this as test 0 at the start of main(), before the slot range tests diff --git a/src/checks/relay_pipeline.rs b/src/checks/relay_pipeline.rs index e10e08c..1e58a4c 100644 --- a/src/checks/relay_pipeline.rs +++ b/src/checks/relay_pipeline.rs @@ -103,23 +103,42 @@ pub async fn check_payloads_delivered_multi( /// Check MEV delivery rate: relay payloads vs on-chain blocks. pub async fn check_mev_delivery_rate( - relay: &RelayClient, + relays: &[RelayClient], beacon: &BeaconClient, start_slot: u64, end_slot: u64, threshold: f64, ) -> CheckResult { - // Get delivered payload block hashes - let delivered = match relay.get_payloads_delivered(start_slot, end_slot).await { - Ok(p) => p, - Err(e) => { - return CheckResult::fail( - "relay.mev_delivery_rate", - 2, - format!("Error querying delivered payloads: {e}"), - ); + // Get delivered payload block hashes — try each relay until one succeeds. + // Some relays (e.g., mev-boost-relay) don't expose the data API. + let mut delivered = Vec::new(); + let mut last_error = None; + for relay in relays { + match relay.get_payloads_delivered(start_slot, end_slot).await { + Ok(p) => { + delivered = p; + break; + } + Err(e) => { + tracing::warn!( + "Relay {} doesn't support data API ({}), trying next...", + relay.base_url(), + e + ); + last_error = Some(e); + } } - }; + } + if delivered.is_empty() && last_error.is_some() { + return CheckResult::skip( + "relay.mev_delivery_rate", + 2, + format!( + "No relay supports the data API. Last error: {}", + last_error.unwrap() + ), + ); + } let delivered_hashes: std::collections::HashSet<_> = delivered.iter().map(|p| p.block_hash).collect(); @@ -358,11 +377,10 @@ pub async fn run_relay_checks( // Aggregated: best-of status; reports combined delivery stats. { let mut mv_results: Vec = Vec::new(); - for relay in &live { - mv_results.push( - check_mev_delivery_rate(relay, beacon, start_slot, end_slot, mev_threshold).await, - ); - } + // Try all relays for delivery data — some may not support the data API + mv_results.push( + check_mev_delivery_rate(&live, beacon, start_slot, end_slot, mev_threshold).await, + ); let any_pass = mv_results.iter().any(|r| r.status == CheckStatus::Pass); let best_status = if any_pass { CheckStatus::Pass diff --git a/src/relay.rs b/src/relay.rs index ddbd24c..f5ec668 100644 --- a/src/relay.rs +++ b/src/relay.rs @@ -56,32 +56,63 @@ impl RelayClient { /// /// Returns payloads delivered in the given slot range. /// - /// The relay API's `cursor` param is an opaque pagination token / DB ID, not a slot - /// filter — passing a slot number produces unreliable results. Instead we request a - /// generous count (no cursor) and let the client-side filter narrow to the window. + /// The relay enforces a maximum limit of 200. We paginate using `cursor` + /// (which is an opaque DB ID from the last item's `block_number` field) until + /// we've fetched all payloads in the slot range or the relay returns no more results. pub async fn get_payloads_delivered( &self, start_slot: u64, end_slot: u64, ) -> Result> { - let resp: Vec = self - .client - .get(format!( + let mut all = Vec::new(); + let mut cursor: Option = None; + let max_pages = 50; // safety: 50 × 200 = 10,000 payloads max + + for _ in 0..max_pages { + let url = format!( "{}/relay/v1/data/bidtraces/proposer_payload_delivered", self.base_url - )) - .query(&[("limit", "10000")]) - .send() - .await? - .error_for_status()? - .json() - .await?; + ); + let mut req = self.client.get(&url).query(&[("limit", "200")]); + if let Some(ref c) = cursor { + req = req.query(&[("cursor", c)]); + } + let resp: Vec = req + .send() + .await? + .error_for_status()? + .json() + .await?; + + if resp.is_empty() { + break; + } + + // Check if we've gone past our slot range + let min_slot = resp.iter().map(|p| p.slot).min().unwrap_or(0); + let _max_slot = resp.iter().map(|p| p.slot).max().unwrap_or(0); + + // Filter to our slot range + for p in &resp { + if p.slot >= start_slot && p.slot <= end_slot { + all.push(p.clone()); + } + } + + // If the oldest result is before our range, we can stop + if min_slot < start_slot { + break; + } + + // Use the last item's block_number as cursor for pagination + if let Some(last) = resp.last() { + cursor = Some(last.block_number.to_string()); + } else { + break; + } + } - // Filter strictly to our slot range - Ok(resp - .into_iter() - .filter(|p| p.slot >= start_slot && p.slot <= end_slot) - .collect()) + Ok(all) } /// GET /relay/v1/data/bidtraces/builder_blocks_received?slot={slot} From 993cad8f8be587affca4b01bff91c33faa9e015b Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 12:25:10 -0700 Subject: [PATCH 07/20] cleanup old configs / assertoor --- .env.example | 6 +- README.md | 243 ++---------------- configs/assertoor-pbs.yml | 91 ------- configs/basic-pbs.yml | 102 -------- configs/pbs-metrics.yml | 63 ----- configs/pbs-validation-modes.yml | 81 ------ configs/reference/kurtosis-config.yml | 101 -------- configs/reference/kurtosis-custom-config.yml | 43 ---- configs/reference/kurtosis-mux-test.yml | 50 ---- configs/reference/kurtosis-ssz-lighthouse.yml | 30 --- configs/reference/kurtosis-ssz-matrix.yml | 60 ----- configs/reference/kurtosis-ssz-prysm.yml | 32 --- scripts/generate_kurtosis_configs.py | 17 +- 13 files changed, 20 insertions(+), 899 deletions(-) delete mode 100644 configs/assertoor-pbs.yml delete mode 100644 configs/basic-pbs.yml delete mode 100644 configs/pbs-metrics.yml delete mode 100644 configs/pbs-validation-modes.yml delete mode 100644 configs/reference/kurtosis-config.yml delete mode 100644 configs/reference/kurtosis-custom-config.yml delete mode 100644 configs/reference/kurtosis-mux-test.yml delete mode 100644 configs/reference/kurtosis-ssz-lighthouse.yml delete mode 100644 configs/reference/kurtosis-ssz-matrix.yml delete mode 100644 configs/reference/kurtosis-ssz-prysm.yml diff --git a/.env.example b/.env.example index d1000be..e494300 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,6 @@ # Default: ethpandaops/mev-boost-relay:main # MEV_RELAY_IMAGE=ethpandaops/mev-boost-relay:main -# Commit-Boost PBS image (your local PBS build) -# Default: commit-boost/pbs:kurtosis -# MEV_BOOST_IMAGE=commit-boost/pbs:kurtosis +# Commit-Boost PBS image (sub latest for your local build) +# Default: commit-boost/commit-boost:latest +# MEV_BOOST_IMAGE=commit-boost/commit-boost:latest diff --git a/README.md b/README.md index 5eccbd4..84d28cb 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,6 @@ Automated verification for [Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) Kurtosis devnets. Spins up a local Ethereum testnet with Commit-Boost as the MEV sidecar, waits for the MEV pipeline to stabilize, verifies each stage of the pipeline, and reports pass/fail. -## What is this? - -When Commit-Boost proxies MEV between validators and relays, several things need to work in sequence: validators register with the relay, the builder submits blocks, the relay serves headers through CB, and the delivered payloads land on chain. If any stage breaks, the rest falls apart silently. - -This tool verifies each stage independently and tells you exactly what broke. - -``` -Validator -> CB (get_header) -> Relay -> Builder -Validator -> CB (submit_block) -> Relay -> Payload on chain - ^ - we verify this whole flow -``` - ## Prerequisites - [Kurtosis CLI](https://docs.kurtosis.com/install) (>= 0.90) @@ -37,48 +24,31 @@ cp .env.example .env | Variable | Default | Purpose | |---|---|---| -| `HELIX_RELAY_IMAGE` | `helix-relay:kurtosis` | Custom Helix relay | +| `HELIX_RELAY_IMAGE` | `helix-relay:kurtosis` | Custom Helix relay image | | `MEV_RELAY_IMAGE` | `ethpandaops/mev-boost-relay:main` | mev-boost relay (multi-relay scenarios) | -| `MEV_BOOST_IMAGE` | `commit-boost/pbs:kurtosis` | Commit-Boost PBS sidecar | +| `MEV_BOOST_IMAGE` | `commit-boost/commit-boost:latest` | Commit-Boost latest image | | `BUILDER_CL_IMAGE` | `sigp/lighthouse:latest` | Builder consensus client | | `BUILDER_EL_IMAGE` | `ethpandaops/reth-rbuilder:develop` | Builder execution client | The `.env` file is read automatically by `generate_kurtosis_configs.py`. It is gitignored — do not commit it. Use `.env.example` as the reference. -## Quick start - -```bash -# Run with the default config (2 Lighthouse/Geth nodes) -# (builds automatically on first run via cargo) -./scripts/run-and-verify.sh +## Kurtosis setup / gotchas +The repo contains a forked `ethereum-package` as a submodule (don't forget to call `git submodule update`). The fork generalizes a lot of the hardcoded patterns in the upstream repo, allowing us to run configurations like commit-boost + helix that weren't possible before. Once [this PR](https://github.com/ethpandaops/ethereum-package/pull/1384) is merged we can deprecate the fork but until then the commands use this as the default Kurtosis package. -# Takes ~55 minutes. You'll see: -# [PASS] chain_finality - Finalized epoch: 7 -# [PASS] relay.payloads_delivered_multi - Delivered 64 payload(s) -# [PASS] payload_hash_match - 64 matched, 0 mismatched -# Result: PASS (8 passed, 0 failed, 0 warnings) -``` +### Kurtosis configs +Kurtosis uses a default Commit-Boost config that can be overridden by inlining it into the kurtosis config, see `example-kurtosis-config.yml`. This is how every test is expected to run. -### Testing a local CB build +To simplify things, the `generate_kurtosis_configs.py` script generates different test scenarios using the values from your `.env` file. These make good references if you need to adjust your Commit-Boost settings. ```bash -# 1. Configure images in .env (if using custom builds) -# 2. In the commit-boost-client repo, build the PBS Docker image: -just build-pbs kurtosis -# 3. Run the verifier: -./scripts/run-and-verify.sh --config configs/basic-pbs.yml +# Creates pre-generated kurtosis config files from .env params +just generate-configs ``` -### Testing against a specific ethereum-package - -If you have a local checkout of the [ethereum-package](https://github.com/ethpandaops/ethereum-package) (e.g., with the [custom CB config PR](https://github.com/ethpandaops/ethereum-package/pull/1355)): - -```bash -./scripts/run-and-verify.sh --package ../ethereum-package --keep -v -``` +## Quick start -The `--keep` flag leaves the enclave running so you can inspect it after. The `-v` flag enables debug logging. +REDOOOOO (use just cmds) ## What it checks @@ -104,201 +74,20 @@ The `--keep` flag leaves the enclave running so you can inspect it after. The `- | `cb_header_values` | Relay header bid values | > 0 | | `cb_get_header_success` | Successful get_header responses | > 0 | -Tier 2 metric checks (latency, errors, header values, get_header success) require CB metrics. See [Metrics limitations](#metrics-limitations) below. +Tier 2 metric checks (latency, errors, header values, get_header success) require CB metrics. ### Tier 3: Extended checks (config-dependent) +talk about muxes -| Check | When it runs | -|---|---| -| `relay.validator_registrations` | When validator pubkeys are provided | - -Mux routing, SSZ encoding, and signer health checks are planned but not yet implemented. - -## Config presets - -The `configs/` directory contains ready-to-use Kurtosis config files: - -| Config | Nodes | What it exercises | -|---|---|---| -| [`basic-pbs.yml`](configs/basic-pbs.yml) | 2x Lighthouse/Geth | Default PBS pipeline. Header selection, relay fan-out. Start here. | -| [`pbs-metrics.yml`](configs/pbs-metrics.yml) | 2x Lighthouse/Geth | PBS + `[metrics]` enabled + Prometheus. Exercises Tier 2 checks. | -| [`pbs-validation-modes.yml`](configs/pbs-validation-modes.yml) | 3x Lighthouse/Geth | Mux with different `header_validation_mode` settings (None vs Standard). | -| [`assertoor-pbs.yml`](configs/assertoor-pbs.yml) | 2x Lighthouse/Geth | CI integration. Same checks run inside assertoor via `run_shell`. | - -Most presets use `commit_boost_config` to inject a full CB config as inline TOML (`basic-pbs.yml` uses the default template). Template variables (`{{ .Network }}`, `{{ .Port }}`, `{{ .Relays }}`, `{{ .Timestamp }}`) are injected by the ethereum-package at launch. - -Additional reference configs for SSZ testing, client matrix testing, and mux experiments are in `configs/reference/`. - -### Writing your own config - -Start from `basic-pbs.yml` and modify the `commit_boost_config` block. The config is standard CB TOML with template variable injection. Key things to know: - -- `{{ .Port }}` becomes the PBS listen port (18550) -- `{{ .Relays }}` is the list of relay URLs discovered by the ethereum-package -- `{{ .Network }}` is the path to the network config inside the container -- `{{ .Timestamp }}` is the genesis time -- `spamoor` in `additional_services` is required (generates transactions so the builder has something to build) ## CLI reference -### cb-verify - -``` -cargo run --release -- --enclave CB-Testnet [OPTIONS] - -Options: - --enclave NAME Kurtosis enclave name (required) - --min-epochs N Observation window in epochs (default: 2) - --target-epoch N Wait until this epoch before checks (default: 7) - --timeout SECS Readiness timeout in seconds (default: 1500) - --mev-threshold PCT Min MEV delivery rate, 0.0-1.0 (default: 0.30) - --json Output JSON report - -v, --verbose Debug logging -``` +REDO -### run-and-verify.sh - -``` -./scripts/run-and-verify.sh [OPTIONS] - -Options: - --config FILE Kurtosis config (default: configs/basic-pbs.yml) - --enclave NAME Enclave name (default: CB-Testnet) - --package PATH ethereum-package: local path or GitHub ref - --keep Don't tear down the enclave on exit - --json JSON output - --timeout SECS Readiness timeout (default: 1500) - --min-epochs N Observation window (default: 2) - -v, --verbose Debug logging -``` ## How it works -1. **Discovery**: Finds beacon, relay, and CB services via `kurtosis enclave inspect` -2. **Readiness**: Polls beacon API until synced, finalizing, and past epoch 7 (no hardcoded sleeps) -3. **Observation**: Watches 2 more epochs of steady-state activity -4. **Checks**: Queries beacon API, relay data API, and (optionally) CB metrics -5. **Report**: Colored terminal output or JSON (`--json`). Exit code 0/1/2. - -### Exit codes - -- `0` All Tier 1 checks passed -- `1` One or more Tier 1 checks failed -- `2` Setup failure (enclave not found, timeout, etc.) - -### Timing - -With the default `seconds_per_slot: 12` (matching mainnet), expect ~55 minutes total: -- ~45 min for the devnet to reach epoch 7 and finalize -- ~8 min for the 2-epoch observation window - -## Metrics limitations - -CB's metrics server only starts when the `CB_METRICS_PORT` environment variable is set. This env var is normally injected by `cb docker init` when generating docker-compose files, but upstream `ethpandaops/ethereum-package` doesn't set it. The TOML `[metrics]` block in `commit_boost_config` alone is ignored by PBS standalone -- see `crates/metrics/src/provider.rs` in commit-boost-client. - -### Fixed in local ethereum-package fork - -The `JasonVranek/ethereum-package` fork patches this in `src/mev/commit-boost/mev_boost/mev_boost_launcher.star`: -- Adds `metrics: 9090/tcp` to `USED_PORTS` -- Adds `CB_METRICS_PORT: "9090"` to `env_vars` - -Run against the local fork to enable metrics: - -```bash -./scripts/run-and-verify.sh --package ../ethereum-package --config configs/assertoor-pbs.yml -``` - -With the fork, the `cb_*_matrix` checks populate with per-endpoint, per-relay status code counts, and the `cb_relay_latency` p95 histogram check activates. Using upstream ethereum-package, these tier 2 checks SKIP. All tier 1 checks (chain health, relay pipeline, payload matching) work either way. - -## Assertoor integration (CI) - -For CI pipelines, the verification checks run inside [assertoor](https://github.com/ethpandaops/assertoor) as an in-enclave service. This avoids the need for a separate verification container or external polling. - -The test definition lives at [`assertoor/cb-mev-pipeline.yaml`](assertoor/cb-mev-pipeline.yaml). It uses assertoor's native tasks for chain readiness (finality, sync) and `run_shell` tasks with `curl`+`jq` for relay and payload checks. The assertoor container has both tools installed. - -### How it works - -1. The kurtosis config (`configs/assertoor-pbs.yml`) adds assertoor to `additional_services` -2. `assertoor_params.tests` references the test YAML via raw GitHub URL -3. Assertoor fetches the test at startup and runs it inside the enclave -4. The [kurtosis-assertoor-github-action](https://github.com/ethpandaops/kurtosis-assertoor-github-action) polls assertoor's API and reports pass/fail - -### GitHub Actions example - -```yaml -jobs: - cb-mev-pipeline: - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - uses: ethpandaops/kurtosis-assertoor-github-action@v1 - with: - ethereum_package_args: configs/assertoor-pbs.yml -``` - -### Service discovery inside the enclave - -The test YAML defaults to standard kurtosis DNS names: -- Beacon: `http://cl-1-lighthouse-geth:4000` -- Relay: `http://mev-relay-api:9062` - -If your enclave uses different CL/EL types, override via `testConfig` in `assertoor_params.tests`: - -```yaml -assertoor_params: - tests: - - file: "https://raw.githubusercontent.com/jvranek/cb-testing/main/assertoor/cb-mev-pipeline.yaml" - testConfig: - beaconUrl: "http://cl-1-prysm-geth:3500" -``` - -### Local assertoor testing - -To test the assertoor flow locally without pushing to GitHub: - -```bash -# Serve the test YAML locally -cd assertoor && python3 -m http.server 8888 & - -# Point assertoor at the local URL (edit assertoor-pbs.yml temporarily) -# Change the test URL to: http://host.docker.internal:8888/cb-mev-pipeline.yaml -kurtosis run github.com/ethpandaops/ethereum-package \ - --enclave CB-Testnet --args-file configs/assertoor-pbs.yml - -# Check assertoor status -ASSERTOOR=$(kurtosis port print CB-Testnet assertoor http) -curl -s $ASSERTOOR/api/v1/test_runs | jq '.[0].status' -``` - -For iterating on checks locally, the standalone verifier (`./scripts/run-and-verify.sh`) is faster. +REDOOOOO ## Project layout - -``` -src/ - main.rs CLI entry point and orchestrator (clap) - beacon.rs Beacon API client (alloy types) - relay.rs Relay Data API client (alloy types) - discovery.rs Kurtosis service/port discovery - metrics.rs Prometheus metrics fetching - report.rs Terminal (ANSI) and JSON output formatting - checks/ - mod.rs CheckResult, CheckStatus types - chain_health.rs Finality, sync, missed slots, CB service status - relay_pipeline.rs Builder blocks, delivered payloads, MEV delivery rate - payload_matching.rs Cross-ref relay payloads with on-chain blocks - cb_metrics.rs CB Prometheus metric assertions -assertoor/ - cb-mev-pipeline.yaml Assertoor test definition for CI -configs/ - basic-pbs.yml Default PBS preset - pbs-metrics.yml PBS + metrics preset - pbs-validation-modes.yml Mux validation mode preset - assertoor-pbs.yml CI preset with assertoor - reference/ Additional configs for SSZ, client matrix, etc. -scripts/ - run-and-verify.sh Full lifecycle: launch, verify, tear down -Cargo.toml Rust project config -PLAN.md Design doc and roadmap -``` +REDOOOOOOOOOOOOOOOOO diff --git a/configs/assertoor-pbs.yml b/configs/assertoor-pbs.yml deleted file mode 100644 index 5647388..0000000 --- a/configs/assertoor-pbs.yml +++ /dev/null @@ -1,91 +0,0 @@ -# cb-testnet-verifier preset: Assertoor PBS Pipeline + Metrics -# -# Runs CB MEV pipeline checks inside assertoor for CI integration, AND -# enables CB's Prometheus metrics endpoint so cb-verify's status-code -# matrix checks populate when run against the same enclave. -# -# Two verifiers, one enclave, same invariants. Run assertoor for CI-blessed -# pass/fail, then run cb-verify against the same enclave to get the full -# status-code matrix + latency histograms. -# -# Usage (local): -# kurtosis run github.com/ethpandaops/ethereum-package \ -# --enclave CB-Testnet --args-file configs/assertoor-pbs.yml -# -# # Check assertoor status: -# ASSERTOOR=$(kurtosis port print CB-Testnet assertoor http) -# curl -s $ASSERTOOR/api/v1/test_runs | jq '.[0].status' -# -# # Run cb-verify against the same enclave (metrics now populated): -# ./scripts/run-and-verify.sh --config configs/assertoor-pbs.yml -# -# Usage (CI with GitHub Action): -# uses: ethpandaops/kurtosis-assertoor-github-action@v1 -# with: -# ethereum_package_args: configs/assertoor-pbs.yml - -participants: - - el_type: geth - cl_type: lighthouse - count: 2 - -additional_services: - - spamoor - - dora - - assertoor - - prometheus - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - # Enable CB metrics so cb-verify's status-code matrix populates. - # The kurtosis ethereum-package doesn't publish port 9090 to the host - # by default, so cb-verify falls back to `kurtosis service exec` + curl - # inside the container to scrape. See checks/cb_metrics.rs. - commit_boost_config: | - chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } - - [pbs] - host = "0.0.0.0" - port = {{ .Port }} - timeout_get_header_ms = 950 - timeout_get_payload_ms = 4000 - late_in_slot_time_ms = 2000 - - {{ range $index, $relay := .Relays }} - [[relays]] - id = "mev_relay_{{$index}}" - url = "{{ $relay }}" - {{- end }} - - [metrics] - host = "0.0.0.0" - port = 9090 - - [logs.stdout] - level = "debug" - - [logs.file] - enabled = false - -assertoor_params: - run_stability_check: false - run_block_proposal_check: false - tests: - # For CI: use the raw GitHub URL (uncomment and update branch/tag): - # - file: "https://raw.githubusercontent.com/jvranek/cb-testing/main/assertoor/cb-mev-pipeline.yaml" - # For local testing: serve the file and use host.docker.internal: - - file: "http://host.docker.internal:8888/assertoor/cb-mev-pipeline.yaml" - -network_params: - network: kurtosis - seconds_per_slot: 12 - num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: - "giant issue aisle success illegal bike spike - question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy - very lucky have athlete" diff --git a/configs/basic-pbs.yml b/configs/basic-pbs.yml deleted file mode 100644 index 8b4537e..0000000 --- a/configs/basic-pbs.yml +++ /dev/null @@ -1,102 +0,0 @@ -# cb-testnet-verifier preset: Basic PBS Pipeline -# -# Tests the core MEV pipeline through Commit-Boost. -# -# Image pinning rationale (from the CB repo's working kurtosis config): -# - mev_boost_image = commit-boost/pbs:kurtosis -# CB-internal kurtosis build. Must be built locally from the -# commit-boost-client repo (e.g. `just kurtosis-image` or equivalent) -# before running. The ghcr :latest tag has historically been unstable. -# - mev_relay_image and mev_builder_image pinned to the pair known to -# co-operate. Rolling tags drift; this pair is what the CB repo runs. -# -# Usage: -# ./scripts/run-and-verify.sh --config configs/basic-pbs.yml - -participants: - - el_type: geth - cl_type: lighthouse - -additional_services: - - dora - - spamoor -mev_type: commit-boost - -mev_params: - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_boost_image: commit-boost/pbs:kurtosis - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - -network_params: - network: kurtosis - network_id: "3151908" - deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" - seconds_per_slot: 12 - slot_duration_ms: 12000 - num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: - "giant issue aisle success illegal bike spike - question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy - very lucky have athlete" - preregistered_validator_count: 0 - additional_mnemonics: [] - genesis_delay: 20 - genesis_time: 0 - genesis_gaslimit: 60000000 - max_per_epoch_activation_churn_limit: 8 - churn_limit_quotient: 65536 - confirmation_byzantine_threshold: 25 - ejection_balance: 16000000000 - eth1_follow_distance: 2048 - min_validator_withdrawability_delay: 256 - shard_committee_period: 256 - attestation_due_bps_gloas: 2500 - aggregate_due_bps_gloas: 5000 - sync_message_due_bps_gloas: 2500 - contribution_due_bps_gloas: 5000 - payload_attestation_due_bps: 7500 - view_freeze_cutoff_bps: 7500 - inclusion_list_submission_due_bps: 6667 - proposer_inclusion_list_cutoff_bps: 9167 - deneb_fork_epoch: 0 - electra_fork_epoch: 0 - fulu_fork_epoch: 0 - gloas_fork_epoch: 18446744073709551615 - network_sync_base_url: https://snapshots.ethpandaops.io/ - force_snapshot_sync: false - samples_per_slot: 8 - custody_requirement: 4 - max_blobs_per_block_electra: 9 - max_request_blocks_deneb: 128 - target_blobs_per_block_electra: 6 - base_fee_update_fraction_electra: 5007716 - additional_preloaded_contracts: {} - devnet_repo: ethpandaops - prefunded_accounts: {} - bpo_1_epoch: 0 - bpo_1_max_blobs: 15 - bpo_1_target_blobs: 10 - bpo_1_base_fee_update_fraction: 8346193 - bpo_2_epoch: 18446744073709551615 - bpo_2_max_blobs: 21 - bpo_2_target_blobs: 14 - bpo_2_base_fee_update_fraction: 11684671 - bpo_3_epoch: 18446744073709551615 - bpo_3_max_blobs: 0 - bpo_3_target_blobs: 0 - bpo_3_base_fee_update_fraction: 0 - bpo_4_epoch: 18446744073709551615 - bpo_4_max_blobs: 0 - bpo_4_target_blobs: 0 - bpo_4_base_fee_update_fraction: 0 - bpo_5_epoch: 18446744073709551615 - bpo_5_max_blobs: 0 - bpo_5_target_blobs: 0 - bpo_5_base_fee_update_fraction: 0 - withdrawal_type: "0x00" - withdrawal_address: "0x8943545177806ED17B9F23F0a21ee5948eCaa776" - validator_balance: 32 - min_epochs_for_data_column_sidecars_requests: 4096 - builder_count: 0 - builder_balance: 100 diff --git a/configs/pbs-metrics.yml b/configs/pbs-metrics.yml deleted file mode 100644 index 748e103..0000000 --- a/configs/pbs-metrics.yml +++ /dev/null @@ -1,63 +0,0 @@ -# cb-testnet-verifier preset: PBS + Metrics -# -# Same as basic-pbs but with CB metrics enabled and Prometheus in the enclave. -# Adds [metrics] block to the CB config via commit_boost_config inline TOML. -# -# NOTE: Tier 2 metric checks will still SKIP until the CB metrics env var -# issue is resolved (CB needs CB_METRICS_PORT env var to start the metrics -# server, which kurtosis doesn't set). See PLAN.md for details. -# -# Usage: -# ./scripts/run-and-verify.sh --config configs/pbs-metrics.yml - -participants: - - el_type: geth - cl_type: lighthouse - count: 2 - -additional_services: - - spamoor - - dora - - prometheus - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - commit_boost_config: | - chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } - - [pbs] - host = "0.0.0.0" - port = {{ .Port }} - timeout_get_header_ms = 950 - timeout_get_payload_ms = 4000 - late_in_slot_time_ms = 2000 - - {{ range $index, $relay := .Relays }} - [[relays]] - id = "mev_relay_{{$index}}" - url = "{{ $relay }}" - {{- end }} - - [metrics] - host = "0.0.0.0" - port = 9090 - - [logs.stdout] - level = "debug" - - [logs.file] - enabled = false - -network_params: - network: kurtosis - seconds_per_slot: 12 - num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: - "giant issue aisle success illegal bike spike - question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy - very lucky have athlete" diff --git a/configs/pbs-validation-modes.yml b/configs/pbs-validation-modes.yml deleted file mode 100644 index c6e4fb4..0000000 --- a/configs/pbs-validation-modes.yml +++ /dev/null @@ -1,81 +0,0 @@ -# cb-testnet-verifier preset: Header Validation Modes -# -# Tests Commit-Boost's header validation modes using mux routing. -# The default path uses "standard" mode (full decode + signature verification). -# A mux route overrides to "none" mode (light path, minimal decoding). -# -# To use this config effectively, you need to populate the mux's -# validator_pubkeys with real pubkeys from the devnet. Steps: -# -# 1. Start the enclave with this config (mux pubkeys empty = no routing) -# 2. Query validator pubkeys: -# curl -s http:///eth/v1/beacon/states/head/validators \ -# | jq '.data[:5] | .[].validator.pubkey' -# 3. Paste those pubkeys into the [[mux]] validator_pubkeys list -# 4. Restart the enclave with the updated config -# -# Usage: -# ./scripts/run-and-verify.sh --config configs/pbs-validation-modes.yml --keep - -participants: - - el_type: geth - cl_type: lighthouse - count: 3 - -additional_services: - - spamoor - - dora - - prometheus - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - commit_boost_config: | - chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } - - [pbs] - host = "0.0.0.0" - port = {{ .Port }} - timeout_get_header_ms = 950 - timeout_get_payload_ms = 4000 - header_validation_mode = "standard" - - {{ range $index, $relay := .Relays }} - [[relays]] - id = "relay_{{$index}}" - url = "{{ $relay }}" - {{- end }} - - # Mux: route specific validators through "none" mode (light path). - # Replace the placeholder pubkey with real validator pubkeys from your devnet. - [[mux]] - id = "light_path" - validator_pubkeys = [] - timeout_get_header_ms = 500 - header_validation_mode = "none" - [[mux.relays]] - id = "mux_relay_0" - url = "{{ index .Relays 0 }}" - - [metrics] - host = "0.0.0.0" - port = 9090 - - [logs.stdout] - level = "debug" - - [logs.file] - enabled = false - -network_params: - network: kurtosis - seconds_per_slot: 12 - num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: - "giant issue aisle success illegal bike spike - question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy - very lucky have athlete" diff --git a/configs/reference/kurtosis-config.yml b/configs/reference/kurtosis-config.yml deleted file mode 100644 index 9658f7e..0000000 --- a/configs/reference/kurtosis-config.yml +++ /dev/null @@ -1,101 +0,0 @@ -# ELs: geth, nethermind, erigon, besu, reth, ethrex -# CLs: nimbus, lighthouse, lodestar, teku, prysm, and grandine -participants: - - el_type: geth - cl_type: nimbus - - - el_type: nethermind - cl_type: lighthouse - - - el_type: erigon - cl_type: lodestar - - - el_type: besu - cl_type: teku - - - el_type: reth - cl_type: prysm - - - el_type: ethrex - cl_type: grandine - -additional_services: - - dora - - spamoor -mev_type: commit-boost - -mev_params: - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_boost_image: commit-boost/pbs:kurtosis - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - -network_params: - network: kurtosis - network_id: "3151908" - deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" - seconds_per_slot: 12 - slot_duration_ms: 12000 - num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: - "giant issue aisle success illegal bike spike - question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy - very lucky have athlete" - preregistered_validator_count: 0 - additional_mnemonics: [] - genesis_delay: 20 - genesis_time: 0 - genesis_gaslimit: 60000000 - max_per_epoch_activation_churn_limit: 8 - churn_limit_quotient: 65536 - ejection_balance: 16000000000 - eth1_follow_distance: 2048 - min_validator_withdrawability_delay: 256 - shard_committee_period: 256 - attestation_due_bps_gloas: 2500 - aggregate_due_bps_gloas: 5000 - sync_message_due_bps_gloas: 2500 - contribution_due_bps_gloas: 5000 - payload_attestation_due_bps: 7500 - view_freeze_cutoff_bps: 7500 - inclusion_list_submission_due_bps: 6667 - proposer_inclusion_list_cutoff_bps: 9167 - deneb_fork_epoch: 0 - electra_fork_epoch: 0 - fulu_fork_epoch: 0 - gloas_fork_epoch: 18446744073709551615 - network_sync_base_url: https://snapshots.ethpandaops.io/ - force_snapshot_sync: false - samples_per_slot: 8 - custody_requirement: 4 - max_blobs_per_block_electra: 9 - max_request_blocks_deneb: 128 - target_blobs_per_block_electra: 6 - base_fee_update_fraction_electra: 5007716 - additional_preloaded_contracts: {} - devnet_repo: ethpandaops - prefunded_accounts: {} - bpo_1_epoch: 0 - bpo_1_max_blobs: 15 - bpo_1_target_blobs: 10 - bpo_1_base_fee_update_fraction: 8346193 - bpo_2_epoch: 18446744073709551615 - bpo_2_max_blobs: 21 - bpo_2_target_blobs: 14 - bpo_2_base_fee_update_fraction: 11684671 - bpo_3_epoch: 18446744073709551615 - bpo_3_max_blobs: 0 - bpo_3_target_blobs: 0 - bpo_3_base_fee_update_fraction: 0 - bpo_4_epoch: 18446744073709551615 - bpo_4_max_blobs: 0 - bpo_4_target_blobs: 0 - bpo_4_base_fee_update_fraction: 0 - bpo_5_epoch: 18446744073709551615 - bpo_5_max_blobs: 0 - bpo_5_target_blobs: 0 - bpo_5_base_fee_update_fraction: 0 - withdrawal_type: "0x00" - withdrawal_address: "0x8943545177806ED17B9F23F0a21ee5948eCaa776" - validator_balance: 32 - min_epochs_for_data_column_sidecars_requests: 4096 diff --git a/configs/reference/kurtosis-custom-config.yml b/configs/reference/kurtosis-custom-config.yml deleted file mode 100644 index 634d66e..0000000 --- a/configs/reference/kurtosis-custom-config.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Test config: exercises commit_boost_config inline override. -# Adds skip_sigverify and custom timeouts that the default template doesn't expose. -# -# Run against local ethereum-package with the feature branch: -# just kurtosis-restart /path/to/ethereum-package provisioning/kurtosis-custom-config.yml - -participants: - - el_type: geth - cl_type: lighthouse - count: 2 - -additional_services: - - dora - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - commit_boost_config: | - chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } - - [pbs] - host = "0.0.0.0" - port = {{ .Port }} - skip_sigverify = true - timeout_get_header_ms = 800 - timeout_get_payload_ms = 3000 - late_in_slot_time_ms = 1500 - - {{ range $index, $relay := .Relays }} - [[relays]] - id = "mev_relay_{{$index}}" - url = "{{ $relay }}" - {{- end }} - - [logs.stdout] - level = "debug" - - [logs.file] - enabled = false diff --git a/configs/reference/kurtosis-mux-test.yml b/configs/reference/kurtosis-mux-test.yml deleted file mode 100644 index 7543274..0000000 --- a/configs/reference/kurtosis-mux-test.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Test config: mux routing with single relay backing both entries -# Tests that CB parses [[mux]] correctly -# Both relay entries point at the same flashbots relay with different IDs. - -participants: - - el_type: geth - cl_type: lighthouse - count: 2 - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop - commit_boost_config: | - chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } - - [pbs] - host = "0.0.0.0" - port = {{ .Port }} - skip_sigverify = true - timeout_get_header_ms = 950 - - {{ range $index, $relay := .Relays }} - [[relays]] - id = "relay_{{$index}}" - url = "{{ $relay }}" - {{- end }} - - [[mux]] - id = "fast_path" - validator_pubkeys = [ - "0x8000000000000000000000000000000000000000000000000000000000000000000000000000000000000000deadbeef", - ] - timeout_get_header_ms = 500 - late_in_slot_time_ms = 1000 - [[mux.relays]] - id = "mux_relay" - url = "{{ index .Relays 0 }}" - - [logs.stdout] - level = "debug" - - [logs.file] - enabled = false - -additional_services: - - dora diff --git a/configs/reference/kurtosis-ssz-lighthouse.yml b/configs/reference/kurtosis-ssz-lighthouse.yml deleted file mode 100644 index 21b9076..0000000 --- a/configs/reference/kurtosis-ssz-lighthouse.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Kurtosis config: SSZ get_header testing with Lighthouse -# Lighthouse sends Accept: application/octet-stream;q=1.0,application/json;q=0.9 by default. -# No extra flags needed. Use --builder-disable-ssz to force JSON-only. -# -# Prerequisites: -# just build-pbs kurtosis -# -# Run: -# kurtosis run github.com/ethpandaops/ethereum-package \ -# --enclave CB-Testnet \ -# --args-file provisioning/kurtosis-ssz-lighthouse.yml -# -# Verify SSZ is flowing: -# kurtosis service logs CB-Testnet commit-boost-001-lighthouse-geth 2>&1 | grep -i "content_type\|ssz\|encoding" - -participants: - - el_type: geth - cl_type: lighthouse - count: 2 - -additional_services: - - dora - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop diff --git a/configs/reference/kurtosis-ssz-matrix.yml b/configs/reference/kurtosis-ssz-matrix.yml deleted file mode 100644 index 50a1edd..0000000 --- a/configs/reference/kurtosis-ssz-matrix.yml +++ /dev/null @@ -1,60 +0,0 @@ -# Kurtosis config: SSZ encoding matrix across all CL clients -# Tests SSZ get_header behavior with every major CL implementation. -# -# SSZ builder API support by client (as of 2026-04): -# -# Lighthouse: SSZ preferred by default. --builder-disable-ssz to force JSON. -# Teku: SSZ preferred by default. No flag. Hardcoded. -# Nimbus: SSZ preferred by default. No flag. Hardcoded. -# Lodestar: SSZ preferred by default. No flag. Adaptive with 415 fallback. -# Prysm: JSON by default. Requires --enable-builder-ssz flag. -# Grandine: Unknown / not audited. -# -# All clients except Prysm send Accept: application/octet-stream with higher -# priority than application/json. Prysm is the only holdout requiring explicit opt-in. -# -# This config runs all SSZ-capable CLs to verify CB handles the full matrix. -# -# Prerequisites: -# just build-pbs kurtosis -# -# Run: -# kurtosis run github.com/ethpandaops/ethereum-package \ -# --enclave CB-Testnet \ -# --args-file provisioning/kurtosis-ssz-matrix.yml -# -# Verify per-client encoding: -# for svc in $(kurtosis enclave inspect CB-Testnet 2>&1 | grep commit-boost | awk '{print $1}'); do -# echo "=== $svc ==="; kurtosis service logs CB-Testnet $svc 2>&1 | grep "content_type" | tail -3 -# done - -participants: - # SSZ by default (no flags needed) - - el_type: geth - cl_type: lighthouse - - - el_type: nethermind - cl_type: teku - - - el_type: reth - cl_type: nimbus - - - el_type: erigon - cl_type: lodestar - - # SSZ requires explicit flag - - el_type: besu - cl_type: prysm - cl_extra_params: - - "--enable-builder-ssz" - -additional_services: - - dora - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop diff --git a/configs/reference/kurtosis-ssz-prysm.yml b/configs/reference/kurtosis-ssz-prysm.yml deleted file mode 100644 index 62f7e3a..0000000 --- a/configs/reference/kurtosis-ssz-prysm.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Kurtosis config: SSZ get_header testing with Prysm -# Prysm is the ONLY major CL that requires a flag for SSZ builder API support. -# Without --enable-builder-ssz, Prysm sends Accept: application/json only. -# -# Prerequisites: -# just build-pbs kurtosis -# -# Run: -# kurtosis run github.com/ethpandaops/ethereum-package \ -# --enclave CB-Testnet \ -# --args-file provisioning/kurtosis-ssz-prysm.yml -# -# Verify SSZ is flowing: -# kurtosis service logs CB-Testnet commit-boost-001-prysm-geth 2>&1 | grep -i "content_type\|ssz\|encoding" - -participants: - - el_type: geth - cl_type: prysm - cl_extra_params: - - "--enable-builder-ssz" - count: 2 - -additional_services: - - dora - -mev_type: commit-boost - -mev_params: - mev_boost_image: commit-boost/pbs:kurtosis - mev_relay_image: ethpandaops/mev-boost-relay:main - mev_builder_cl_image: sigp/lighthouse:latest - mev_builder_image: ethpandaops/reth-rbuilder:develop diff --git a/scripts/generate_kurtosis_configs.py b/scripts/generate_kurtosis_configs.py index 4225a31..0d4e394 100644 --- a/scripts/generate_kurtosis_configs.py +++ b/scripts/generate_kurtosis_configs.py @@ -64,8 +64,7 @@ def load_env(): additional_services: - dora - spamoor - - prometheus - - assertoor""" + - prometheus""" COMMON_NETWORK_PARAMS = ( "network_params:\n" @@ -81,14 +80,6 @@ def load_env(): ' very lucky have athlete"' ) -ASSERTOOR_PARAMS = ( - "assertoor_params:\n" - ' run_stability_check: false\n' - ' run_block_proposal_check: false\n' - " tests:\n" - ' file: "http://host.docker.internal:8888/assertoor/cb-mev-pipeline.yaml"' -) - MUX_NETWORK_PARAMS = ( "network_params:\n" ' network: kurtosis\n' @@ -278,7 +269,6 @@ def generate_basic(): "mev_type: custom", mev_params, COMMON_NETWORK_PARAMS, - ASSERTOOR_PARAMS, ]) + "\n" @@ -306,7 +296,6 @@ def generate_multiple_relays(): "mev_type: custom", mev_params, COMMON_NETWORK_PARAMS, - ASSERTOOR_PARAMS, ]) + "\n" @@ -333,7 +322,6 @@ def generate_skip_sigverify(): "mev_type: custom", mev_params, COMMON_NETWORK_PARAMS, - ASSERTOOR_PARAMS, ]) + "\n" @@ -369,7 +357,6 @@ def generate_timing_games(): "mev_type: custom", mev_params, COMMON_NETWORK_PARAMS, - ASSERTOOR_PARAMS, ]) + "\n" @@ -403,7 +390,6 @@ def generate_extra_validation(): "mev_type: custom", mev_params, COMMON_NETWORK_PARAMS, - ASSERTOOR_PARAMS, ]) + "\n" @@ -432,7 +418,6 @@ def generate_mux(pubkeys_node0, pubkeys_node1): "mev_type: custom", mev_params, MUX_NETWORK_PARAMS, - ASSERTOOR_PARAMS, ]) + "\n" From 9b02219f6b34834b5e0dbbcb77cf77cccea86f7a Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 17:12:43 -0700 Subject: [PATCH 08/20] working prometheus --- scripts/generate_kurtosis_configs.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/generate_kurtosis_configs.py b/scripts/generate_kurtosis_configs.py index 0d4e394..15209d2 100644 --- a/scripts/generate_kurtosis_configs.py +++ b/scripts/generate_kurtosis_configs.py @@ -142,6 +142,12 @@ def build_cb_toml_basic(timeout_get_header_ms, timeout_get_payload_ms, lines.insert(insert_idx, line) insert_idx += 1 + lines.append("") + lines.append("") + lines.append("[metrics]") + lines.append("enabled = true") + lines.append('host = "0.0.0.0"') + lines.append("start_port = 9090") lines.append("") lines.append("{{ range $index, $relay := .Relays }}") lines.append("[[relays]]") @@ -183,6 +189,11 @@ def build_cb_toml_mux(pubkeys_node0, pubkeys_node1): 'url = "{{ $relay }}"', "{{- end }}", "", + "[metrics]", + "enabled = true", + 'host = "0.0.0.0"', + "start_port = 9090", + "", "[[mux]]", 'id = "node_0_to_helix"', f"validator_pubkeys = {node0_list}", @@ -199,6 +210,7 @@ def build_cb_toml_mux(pubkeys_node0, pubkeys_node1): 'id = "mux_flashbots"', 'url = "{{ index .Relays 1 }}"', "", + "[logs.stdout]", 'level = "debug"', "", From 8d69ea66e8f9771f4677c2b6558b3f1392bd0a6c Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 17:18:52 -0700 Subject: [PATCH 09/20] enable metrics for cb in kurtosis --- ethereum-package | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum-package b/ethereum-package index bff8b58..022951e 160000 --- a/ethereum-package +++ b/ethereum-package @@ -1 +1 @@ -Subproject commit bff8b58edd239693f5ffc48e7d4967bd465cd825 +Subproject commit 022951ed270f2d0843a4750720dd97a5989ace2e From 4ebe8539ea9298ee63b55134abf2b9460804186d Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 17:19:22 -0700 Subject: [PATCH 10/20] add example config --- configs/example-kurtosis-config.yml | 64 +++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 configs/example-kurtosis-config.yml diff --git a/configs/example-kurtosis-config.yml b/configs/example-kurtosis-config.yml new file mode 100644 index 0000000..1b3acd7 --- /dev/null +++ b/configs/example-kurtosis-config.yml @@ -0,0 +1,64 @@ + +# cb-basic: Single relay (helix) with default Commit-Boost config. +# +# Tests the core MEV pipeline through Commit-Boost with a single Helix +# relay as the only relay endpoint. + +participants: + - el_type: geth + cl_type: lighthouse + +additional_services: + - dora + - spamoor + - prometheus + +mev_type: custom + +mev_params: + mev_relay: helix + mev_sidecar: commit-boost + mev_builder: flashbots + + helix_relay_image: helix-relay:kurtosis + mev_boost_image: commit-boost/pbs:kurtosis + mev_builder_image: ethpandaops/reth-rbuilder:develop + mev_builder_cl_image: sigp/lighthouse:latest + + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + timeout_get_header_ms = 950 + timeout_get_payload_ms = 4000 + late_in_slot_time_ms = 2000 + + [metrics] + host = "0.0.0.0" + port = 9090 + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled = false + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" From 421719bdfa2ee8202e0f7ecd1795377c88761af6 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 17:53:24 -0700 Subject: [PATCH 11/20] update readme --- README.md | 162 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 143 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 84d28cb..2e0379b 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ cp .env.example .env |---|---|---| | `HELIX_RELAY_IMAGE` | `helix-relay:kurtosis` | Custom Helix relay image | | `MEV_RELAY_IMAGE` | `ethpandaops/mev-boost-relay:main` | mev-boost relay (multi-relay scenarios) | -| `MEV_BOOST_IMAGE` | `commit-boost/commit-boost:latest` | Commit-Boost latest image | +| `MEV_BOOST_IMAGE` | `commit-boost/pbs:kurtosis` | Commit-Boost PBS image | | `BUILDER_CL_IMAGE` | `sigp/lighthouse:latest` | Builder consensus client | | `BUILDER_EL_IMAGE` | `ethpandaops/reth-rbuilder:develop` | Builder execution client | @@ -34,21 +34,60 @@ The `.env` file is read automatically by `generate_kurtosis_configs.py`. It is gitignored — do not commit it. Use `.env.example` as the reference. ## Kurtosis setup / gotchas -The repo contains a forked `ethereum-package` as a submodule (don't forget to call `git submodule update`). The fork generalizes a lot of the hardcoded patterns in the upstream repo, allowing us to run configurations like commit-boost + helix that weren't possible before. Once [this PR](https://github.com/ethpandaops/ethereum-package/pull/1384) is merged we can deprecate the fork but until then the commands use this as the default Kurtosis package. + +The repo contains a forked `ethereum-package` as a submodule: + +```bash +git submodule update --init +``` + +The fork generalizes hardcoded patterns from upstream, enabling configs like commit-boost + helix that weren't possible before. Once [this PR](https://github.com/ethpandaops/ethereum-package/pull/1384) merges we can deprecate the fork. ### Kurtosis configs -Kurtosis uses a default Commit-Boost config that can be overridden by inlining it into the kurtosis config, see `example-kurtosis-config.yml`. This is how every test is expected to run. -To simplify things, the `generate_kurtosis_configs.py` script generates different test scenarios using the values from your `.env` file. These make good references if you need to adjust your Commit-Boost settings. +Kurtosis uses a default Commit-Boost config that can be overridden by inlining it into the kurtosis config — see `configs/example-kurtosis-config.yml`. Every generated test config uses this pattern. + +`generate_kurtosis_configs.py` generates test scenarios from `.env`: ```bash -# Creates pre-generated kurtosis config files from .env params just generate-configs ``` +Six scenarios are generated: + +| Config | What it tests | +|---|---| +| `cb-basic.yml` | Single relay (helix), default CB config | +| `cb-multiple-relays.yml` | Two relays (helix + flashbots), aggregated bidding | +| `cb-mux.yml` | Mux routing — 128 validators to helix, 128 to flashbots | +| `cb-skip-sigverify.yml` | Fast path with BLS signature verification disabled | +| `cb-timing-games.yml` | Aggressive per-relay timing overrides for late bidding | +| `cb-extra-validation.yml` | Extra get_header validation via local EL RPC | + ## Quick start -REDOOOOO (use just cmds) +```bash +# Generate configs from .env +just generate-configs + +# Launch a testnet + verify (attached mode) +just testnet configs/generated/cb-mux.yml + +# Verify a running enclave (standalone, no observation window) +just verify-now CB-Testnet + +# Verify with mux routing checks +just verify-with-config configs/generated/cb-mux.yml CB-Testnet + +# Show raw CB PBS logs for debugging +just show-logs CB-Testnet + +# Quick mux routing diagnostic +just test-mux CB-Testnet configs/generated/cb-mux.yml + +# Test relay API endpoints +cargo run --release --bin test-relay -- http://127.0.0.1:PORT 128 160 +``` ## What it checks @@ -59,8 +98,9 @@ REDOOOOO (use just cmds) | `chain_finality` | Beacon chain has finalized (epoch >= 2) | | `sync_status` | Beacon node is not syncing | | `cb_running` | Commit-Boost services are running in the enclave | -| `relay.payloads_delivered_multi` | The relay delivered payloads to proposers | -| `payload_hash_match` | Every delivered payload's block_hash matches on chain | +| `relay.payloads_delivered_multi` | Relays delivered payloads to proposers | +| `payload_hash_match` | Delivered payload block_hashes match on-chain | +| `mux.routing` | Validator pubkeys routed to correct relay (config-dependent) | ### Tier 2: Quality metrics (should pass) @@ -69,25 +109,109 @@ REDOOOOO (use just cmds) | `missed_slots` | Missed slot rate in observation window | < 10% | | `relay.builder_blocks_received` | Builder submitted blocks to relay | > 0 | | `relay.mev_delivery_rate` | Slots using relay-built blocks vs local | >= 30% | -| `cb_relay_latency` | Mean get_header latency | < 500ms | -| `cb_relay_errors` | Relay 5xx error count | == 0 | -| `cb_header_values` | Relay header bid values | > 0 | -| `cb_get_header_success` | Successful get_header responses | > 0 | +| `relay.validator_registrations` | All validators registered on relay | 100% | -Tier 2 metric checks (latency, errors, header values, get_header success) require CB metrics. - -### Tier 3: Extended checks (config-dependent) -talk about muxes +### Tier 3: CB metrics +| Check | What it verifies | +|---|---| +| `cb_get_header_matrix` | get_header status codes from relay vs beacon side | +| `cb_register_validator_matrix` | register_validator status codes | +| `cb_submit_blinded_block_matrix` | submit_blinded_block status codes | +| `cb_status_matrix` | status check responses | +| `cb_v2_fallback` | v1→v2 fallback behavior | +| `cb_relay_latency` | get_header latency histogram (p95) | ## CLI reference -REDO +### cb-verify + +``` +cb-verify [OPTIONS] + +Options: + --enclave Kurtosis enclave name (required) + --config Kurtosis YAML config (for mux verification) + --min-epochs Observation window in epochs [default: 2] + --target-epoch Wait before starting checks [default: 7] + --timeout Readiness timeout [default: 3600] + --mev-threshold Min MEV delivery rate [default: 0.30] + --json Output JSON report + --verbose Debug logging + --strict Promote WARN to FAIL + --live-metrics Poll :9090/metrics during observation + --show-logs Print raw CB PBS logs, no checks + --output-dir Save JSON reports (requires --json) +``` + +### test-mux + +``` +test-mux + +Fetches CB PBS logs, parses mux events, verifies routing against config. +No observation window. Completes in seconds. +``` +### test-relay + +``` +test-relay [pubkey] + +Tests relay data API endpoints with slot filtering. +Verifies delivered payloads, builder blocks, validator registration. +``` ## How it works -REDOOOOO +**Attached mode** (`just testnet`): +1. `run-and-verify.sh` launches Kurtosis enclave with the chosen config +2. `cb-verify` waits for chain readiness (target epoch, finalization) +3. Observes for `min_epochs` while polling health +4. Runs all tier checks, outputs report + +**Standalone mode** (`just verify-now`): +1. Discovers services in running enclave via `kurtosis enclave inspect` +2. Skips observation window (`--min-epochs 0`) +3. Runs checks against current state + +**Mux verification** (`test-mux` or `--config`): +1. Parses Kurtosis YAML → extracts `commit_boost_config` → finds `[[mux]]` sections +2. Fetches CB PBS logs via `kurtosis service logs` +3. Parses log lines (ANSI-aware) for `using mux config` events +4. Cross-references proposer pubkeys against mux mapping +5. FAIL if any pubkey appears on wrong relay + +**Metrics** (tier 3): +1. Discovers metrics URL from `kurtosis port print` (port 9090) +2. Falls back to `kurtosis exec` into CB container +3. Parses Prometheus text format +4. Checks status code matrices, latency histograms ## Project layout -REDOOOOOOOOOOOOOOOOO + +``` +cb-testing/ + Cargo.toml # Workspace: cb-verify, test-mux, test-relay + justfile # Build/test/launch commands + README.md + .env.example # Docker image overrides + scripts/ + run-and-verify.sh # Attached mode launcher + generate_kurtosis_configs.py # Config generator + configs/ + generated/ # Pre-generated test scenarios + example-kurtosis-config.yml + src/ + main.rs # cb-verify binary + checks/ + chain_health.rs # Finality, sync, missed slots + relay_pipeline.rs # Delivery, registration, MEV rate + payload_matching.rs # Hash matching + mux_routing.rs # Mux config parsing, log analysis + cb_metrics.rs # Prometheus metrics checks + bin/ + test_mux.rs # Mux diagnostic binary + test_relay.rs # Relay API diagnostic binary + ethereum-package/ # Forked Kurtosis package (submodule) +``` From 1298aa1fde4356782e7a4a4fbb82024cef17c0f5 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 18:34:18 -0700 Subject: [PATCH 12/20] add mev_builder_subsidy --- configs/example-kurtosis-config.yml | 3 +++ scripts/generate_kurtosis_configs.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/configs/example-kurtosis-config.yml b/configs/example-kurtosis-config.yml index 1b3acd7..ec90da9 100644 --- a/configs/example-kurtosis-config.yml +++ b/configs/example-kurtosis-config.yml @@ -24,6 +24,8 @@ mev_params: mev_boost_image: commit-boost/pbs:kurtosis mev_builder_image: ethpandaops/reth-rbuilder:develop mev_builder_cl_image: sigp/lighthouse:latest + mev_builder_subsidy: 1 + commit_boost_config: | chain = { genesis_time_secs = {{ .Timestamp }}, path = "{{ .Network }}" } @@ -62,3 +64,4 @@ network_params: "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete" + prefunded_accounts: '{"0xb9e79d19f651a941757b35830232e7efc77e1c79": {"balance": "100000ETH"}}' diff --git a/scripts/generate_kurtosis_configs.py b/scripts/generate_kurtosis_configs.py index 15209d2..6e65e3b 100644 --- a/scripts/generate_kurtosis_configs.py +++ b/scripts/generate_kurtosis_configs.py @@ -77,7 +77,8 @@ def load_env(): " preregistered_validator_keys_mnemonic:\n" ' "giant issue aisle success illegal bike spike\n' " question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy\n" - ' very lucky have athlete"' + ' very lucky have athlete"\n' + ' prefunded_accounts: \'{"0xb9e79d19f651a941757b35830232E7EFC77E1c79": {"balance": "100000ETH"}}\'\n' ) MUX_NETWORK_PARAMS = ( @@ -91,7 +92,8 @@ def load_env(): " preregistered_validator_keys_mnemonic:\n" ' "giant issue aisle success illegal bike spike\n' " question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy\n" - ' very lucky have athlete"' + ' very lucky have athlete"\n' + " prefunded_accounts: '{\"0xb9e79d19f651a941757b35830232E7EFC77E1c79\": {\"balance\": \"100000ETH\"}}'\n" ) # --------------------------------------------------------------------------- @@ -242,6 +244,8 @@ def build_mev_params(relays, images, toml_block): for key, val in images.items(): lines.append(f" {key}: {val}") + lines.append("") + lines.append(" mev_builder_subsidy: 1") lines.append("") lines.append(" commit_boost_config: |") # Indent every non-empty TOML line by 4 spaces; keep blanks truly empty From 1fa49b6aa967b4c72ad5ae1f8ba22c1a07355cee Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 18:55:29 -0700 Subject: [PATCH 13/20] update ci --- .github/workflows/integration.yml | 58 +++++------------------------ configs/example-kurtosis-config.yml | 2 +- 2 files changed, 10 insertions(+), 50 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6b843d9..6f7623f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -28,6 +28,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + submodules: recursive - name: Setup Rust uses: dtolnay/rust-toolchain@master @@ -45,29 +47,6 @@ jobs: - name: Build cb-verify run: just build-release - - name: Checkout Commit-Boost - uses: actions/checkout@v6 - with: - repository: commit-boost/commit-boost-client - ref: ${{ github.event.inputs.cb_ref }} - path: commit-boost-client - - - name: Build CB Docker image - working-directory: commit-boost-client - run: | - docker build \ - -t commit-boost/pbs:kurtosis \ - -f Dockerfile \ - --build-arg BIN=pbs \ - . - - - name: Generate CI config - run: | - PLAYBOOK_URL="https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/assertoor/cb-mev-pipeline.yaml" - sed "s|http://host.docker.internal:8888/assertoor/cb-mev-pipeline.yaml|${PLAYBOOK_URL}|" \ - configs/assertoor-pbs.yml > configs/assertoor-ci.yml - echo "Playbook URL: $PLAYBOOK_URL" - - name: Install Kurtosis run: | echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | \ @@ -75,33 +54,15 @@ jobs: sudo apt update sudo apt install -y kurtosis-cli - - name: Launch devnet + - name: Launch devnet + verify timeout-minutes: 60 run: | - kurtosis run github.com/ethpandaops/ethereum-package \ - --enclave "$ENCLAVE" \ - --args-file configs/assertoor-ci.yml \ - --image-download always \ - --non-blocking-tasks 2>&1 | tee kurtosis-run.log - - - name: Verify - timeout-minutes: 30 - run: | - cargo run --release -- \ - --enclave "$ENCLAVE" \ - --target-epoch "${{ github.event.inputs.target_epoch }}" \ - --min-epochs "${{ github.event.inputs.min_epochs }}" \ - --timeout 3600 \ - --json > verify-report.json 2>verify-err.log - echo "exit_code=$?" >> "$GITHUB_OUTPUT" - - - name: Report - if: always() - run: | - echo "=== Verification Report ===" - if [ -f verify-report.json ]; then - python3 -m json.tool verify-report.json || cat verify-report.json - fi + ./scripts/run-and-verify.sh \ + --config configs/example-kurtosis-config.yml \ + --json \ + --live-metrics \ + --min-epochs 1 \ + --target-epoch 2 - name: Upload artifacts if: always() @@ -110,7 +71,6 @@ jobs: name: verify-${{ github.run_id }} path: | verify-report.json - verify-err.log kurtosis-run.log - name: Teardown diff --git a/configs/example-kurtosis-config.yml b/configs/example-kurtosis-config.yml index ec90da9..746399f 100644 --- a/configs/example-kurtosis-config.yml +++ b/configs/example-kurtosis-config.yml @@ -21,7 +21,7 @@ mev_params: mev_builder: flashbots helix_relay_image: helix-relay:kurtosis - mev_boost_image: commit-boost/pbs:kurtosis + mev_boost_image: ghcr.io/commit-boost/pbs:latest mev_builder_image: ethpandaops/reth-rbuilder:develop mev_builder_cl_image: sigp/lighthouse:latest mev_builder_subsidy: 1 From 4fa61b91080ccabc7ae96ad52ba1d7a42b441969 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 18:57:56 -0700 Subject: [PATCH 14/20] update ci --- .github/workflows/integration.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6f7623f..8c170f4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,6 +1,9 @@ name: Integration Test on: + schedule: + # Runs every day at 00:00 UTC + - cron: '0 0 * * *' workflow_dispatch: inputs: cb_ref: @@ -10,11 +13,11 @@ on: target_epoch: description: 'Target epoch for verification' required: false - default: '7' + default: '2' min_epochs: description: 'Observation window in epochs' required: false - default: '2' + default: '1' env: ENCLAVE: cb-ci-${{ github.run_id }} @@ -61,8 +64,8 @@ jobs: --config configs/example-kurtosis-config.yml \ --json \ --live-metrics \ - --min-epochs 1 \ - --target-epoch 2 + --min-epochs "${{ github.event.inputs.min_epochs }}" \ + --target-epoch "${{ github.event.inputs.target_epoch }}" \ - name: Upload artifacts if: always() From 2dc383bc03f982b8b7f13f6d1c807f1ce2aaa9c4 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 18:58:53 -0700 Subject: [PATCH 15/20] update ci --- .github/workflows/integration.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8c170f4..7721b20 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -6,10 +6,6 @@ on: - cron: '0 0 * * *' workflow_dispatch: inputs: - cb_ref: - description: 'Commit-Boost branch/commit to test' - required: false - default: 'main' target_epoch: description: 'Target epoch for verification' required: false From 37efae1d2ef33d6fd8598034adc22f1e1d9399f4 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 20:01:19 -0700 Subject: [PATCH 16/20] disable zk-boost --- ethereum-package | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum-package b/ethereum-package index 022951e..1dbaa38 160000 --- a/ethereum-package +++ b/ethereum-package @@ -1 +1 @@ -Subproject commit 022951ed270f2d0843a4750720dd97a5989ace2e +Subproject commit 1dbaa384cef59b122ecd3ee5831ce6978b605e26 From 5938fb8b43ecdea031a49345919f12b59c0e506f Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 20:01:26 -0700 Subject: [PATCH 17/20] fix kurtosis install --- .github/workflows/integration.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7721b20..4cac47b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -47,11 +47,14 @@ jobs: run: just build-release - name: Install Kurtosis + shell: bash run: | - echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | \ + echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" | \ sudo tee /etc/apt/sources.list.d/kurtosis.list sudo apt update sudo apt install -y kurtosis-cli + kurtosis analytics disable + echo "$(dirname $(which kurtosis))" >> $GITHUB_PATH - name: Launch devnet + verify timeout-minutes: 60 From cc1e62a84020b5c7eba86c74048f632a431b1e1b Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 20:29:45 -0700 Subject: [PATCH 18/20] update ethereum-package from main --- ethereum-package | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum-package b/ethereum-package index 1dbaa38..1ecc324 160000 --- a/ethereum-package +++ b/ethereum-package @@ -1 +1 @@ -Subproject commit 1dbaa384cef59b122ecd3ee5831ce6978b605e26 +Subproject commit 1ecc324ff248068b9b9b6d8f2330c396269a1e65 From c5afbb10f588c8950d215a31aba2e5a0faae0746 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 20:47:44 -0700 Subject: [PATCH 19/20] comment out zkboost --- ethereum-package | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum-package b/ethereum-package index 1ecc324..4844f88 160000 --- a/ethereum-package +++ b/ethereum-package @@ -1 +1 @@ -Subproject commit 1ecc324ff248068b9b9b6d8f2330c396269a1e65 +Subproject commit 4844f884cb06daab30dd2cc1693328d55168720e From d556a162167eda177ef987ed1853f1e2e85dfbfa Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 7 May 2026 20:54:57 -0700 Subject: [PATCH 20/20] use latest helix img --- configs/example-kurtosis-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/example-kurtosis-config.yml b/configs/example-kurtosis-config.yml index 746399f..7244098 100644 --- a/configs/example-kurtosis-config.yml +++ b/configs/example-kurtosis-config.yml @@ -20,7 +20,7 @@ mev_params: mev_sidecar: commit-boost mev_builder: flashbots - helix_relay_image: helix-relay:kurtosis + helix_relay_image: ghcr.io/gattaca-com/helix-relay:main mev_boost_image: ghcr.io/commit-boost/pbs:latest mev_builder_image: ethpandaops/reth-rbuilder:develop mev_builder_cl_image: sigp/lighthouse:latest