From 38f5d711e309c55a0f605e58eff80c5010f64845 Mon Sep 17 00:00:00 2001 From: beck-8 <34204218+beck-8@users.noreply.github.com> Date: Thu, 17 Sep 2026 20:59:56 +0800 Subject: [PATCH 1/2] fix(multicall3): wait for mpool nonce readiness before deploying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `forge create` intermittently failed while broadcasting the deployment: failed to look up actor state nonce: resolution lookup failed (t410fa34eb7lgf6xovtgfnsnoe6sz67ca2zjkbwi22ly): actor not found The funding step sleeps for a fixed interval and then verifies the transfers with `lotus wallet balance`, which reads the chain head. The message pool tracks its own tipset (`mp.curTs`), updated through a coalescer with a 2-6s delay (HeadChangeCoalesceMinDelay/MaxDelay), so it can still trail the head while the balance is already visible. `eth_sendRawTransaction` resolves the sender's nonce against that lagging view, and fails. Deploy Multicall3 and Deploy MockUSDFC share a parallel epoch, but MockUSDFC spends ~27s installing dependencies before it broadcasts, so only Multicall3 lands inside the window — it compiles in ~130ms and broadcasts immediately. Probe with `cast nonce --block pending`, the only query that goes through the same `getStateNonce(ctx, addr, mp.curTs)` call as the broadcast; every other block parameter silently returns 0 for a missing actor. The probe is read-only so it is retried; the deployment itself still runs exactly once. Co-Authored-By: Claude Opus 5 --- .../start/multicall3_deploy/deployment.rs | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/commands/start/multicall3_deploy/deployment.rs b/src/commands/start/multicall3_deploy/deployment.rs index 14e92652..998bf1da 100644 --- a/src/commands/start/multicall3_deploy/deployment.rs +++ b/src/commands/start/multicall3_deploy/deployment.rs @@ -8,9 +8,68 @@ use crate::commands::start::lotus_utils::get_lotus_rpc_url; use crate::docker::command_logger::run_and_log_command_strings; use crate::docker::push_bind_mount; use crate::paths::foc_devnet_multicall3_repo; +use crate::utils::retry::{retry_with_fixed_delay, DEFAULT_MAX_RETRIES, DEFAULT_RETRY_DELAY_SECS}; use std::error::Error; use tracing::{error, info}; +/// Guard the deployment against a lagging message pool. +/// +/// Funding verifies the balance against the chain head, but the pool can lag behind it, +/// and its view is the one the nonce lookup uses, so the broadcast fails with "actor not +/// found". `--block pending` routes this probe through the pool; other block parameters +/// return 0 for a missing actor and would hide the race. +fn wait_for_deployer_nonce( + deployer_eth: &str, + lotus_rpc_url: &str, + run_id: &str, + context: &super::super::step::SetupContext, +) -> Result<(), Box> { + let mut attempt = 0; + + retry_with_fixed_delay( + || { + attempt += 1; + let args: Vec = vec![ + "run".to_string(), + "--rm".to_string(), + "--name".to_string(), + format!("foc-{}-multicall3-nonce-{}", run_id, attempt), + "-u".to_string(), + "foc-user".to_string(), + "--network".to_string(), + "host".to_string(), + crate::constants::BUILDER_DOCKER_IMAGE.to_string(), + "bash".to_string(), + "-c".to_string(), + format!( + "cast nonce {} --block pending --rpc-url {}", + deployer_eth, lotus_rpc_url + ), + ]; + + let key = format!("multicall3_deployer_nonce_{}_{}", run_id, attempt); + let output = run_and_log_command_strings("docker", &args, context, &key)?; + + if output.status.success() { + Ok(()) + } else { + Err(format!( + "Message pool cannot resolve the nonce for {} yet: {}", + deployer_eth, + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()) + } + }, + DEFAULT_MAX_RETRIES, + DEFAULT_RETRY_DELAY_SECS, + "Multicall3 deployer nonce lookup", + )?; + + info!("✓ Message pool resolves the nonce for {}", deployer_eth); + Ok(()) +} + /// Deploy Multicall3 using forge create pub fn deploy_multicall3( private_key: &str, @@ -126,6 +185,9 @@ pub fn perform_deployment( // Deploy Multicall3 contract let lotus_rpc_url = get_lotus_rpc_url(context)?; let run_id = context.run_id(); + + wait_for_deployer_nonce(&multicall3_deployer_eth, &lotus_rpc_url, run_id, context)?; + let multicall3_address = deploy_multicall3(&private_key, &lotus_rpc_url, run_id, context)?; // Store in context From b09ae7e3f3f85fc955f6d91c86c7f1f0ce54f715 Mon Sep 17 00:00:00 2001 From: beck-8 <34204218+beck-8@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:51:57 +0800 Subject: [PATCH 2/2] fix(eth-acc-funding): verify mpool readiness for all funded accounts --- .../eth_acc_funding/eth_acc_funding_step.rs | 36 ++++++---- .../start/lotus_utils/account_readiness.rs | 65 +++++++++++++++++++ src/commands/start/lotus_utils/mod.rs | 4 ++ .../start/multicall3_deploy/deployment.rs | 62 ------------------ 4 files changed, 93 insertions(+), 74 deletions(-) create mode 100644 src/commands/start/lotus_utils/account_readiness.rs diff --git a/src/commands/start/eth_acc_funding/eth_acc_funding_step.rs b/src/commands/start/eth_acc_funding/eth_acc_funding_step.rs index 844fe27e..40fe1e59 100644 --- a/src/commands/start/eth_acc_funding/eth_acc_funding_step.rs +++ b/src/commands/start/eth_acc_funding/eth_acc_funding_step.rs @@ -7,6 +7,7 @@ use super::key_operations::import_faucet_key; use super::lotus_checks::{check_lotus_running, get_global_faucet_address}; use crate::commands::init::keys::load_keys; use crate::commands::start::eth_acc_funding::constants::FEVM_ACCOUNTS_PREFUNDED; +use crate::commands::start::lotus_utils::{get_lotus_rpc_url, wait_for_account_nonce}; use crate::commands::start::step::{SetupContext, Step}; use crate::docker::command_logger::log_command; use crate::docker::containers::lotus_container_name; @@ -266,12 +267,13 @@ impl ETHAccFundingStep { Ok(()) } - /// Verify account balances in parallel by querying the Lotus node - fn verify_balances_parallel( + /// Verify balances and message pool readiness for each funded account in parallel + fn verify_accounts_parallel( &self, - accounts: Vec<(String, String, u64)>, + accounts: Vec<(String, String, String, u64)>, context: &SetupContext, ) -> Result<(), Box> { + let lotus_rpc_url = get_lotus_rpc_url(context)?; let run_id = context.run_id(); let container_name = lotus_container_name(run_id); @@ -279,7 +281,8 @@ impl ETHAccFundingStep { let errors: Arc>> = Arc::new(Mutex::new(Vec::new())); let mut handles = vec![]; - for (account_name, address, expected_amount) in accounts { + for (account_name, address, eth_address, expected_amount) in accounts { + let lotus_rpc_url = lotus_rpc_url.clone(); let container = container_name.clone(); let errors_clone = Arc::clone(&errors); let context_clone = context.clone(); @@ -368,7 +371,11 @@ impl ETHAccFundingStep { &format!("Balance verification for {}", account_name), ); - // Handle retry result + let verify_result = verify_result.and_then(|()| { + wait_for_account_nonce(ð_address, &lotus_rpc_url, &context_clone) + }); + + // Handle verification result if let Err(e) = verify_result { let error_msg = format!("{}: {}", account_name, e); tracing::error!(" {}", error_msg); @@ -379,18 +386,18 @@ impl ETHAccFundingStep { handles.push(handle); } - // Wait for all balance checks to complete + // Wait for all account verification checks to complete for handle in handles { handle .join() - .map_err(|_| "Thread panicked during balance verification")?; + .map_err(|_| "Thread panicked during account verification")?; } // Check if any errors occurred let errors_vec = errors.lock().unwrap(); if !errors_vec.is_empty() { let combined_error = errors_vec.join("\n"); - return Err(format!("Balance verification failed:\n{}", combined_error).into()); + return Err(format!("Account verification failed:\n{}", combined_error).into()); } Ok(()) @@ -461,12 +468,17 @@ impl Step for ETHAccFundingStep { info!("{}: {} (ETH: {})", account_name, addr, eth_addr); - accounts_to_verify.push((account_name.to_string(), addr.to_string(), *expected_amount)); + accounts_to_verify.push(( + account_name.to_string(), + addr.to_string(), + eth_addr.to_string(), + *expected_amount, + )); } - // Verify balances in parallel - info!("Verifying account balances with Lotus node..."); - self.verify_balances_parallel(accounts_to_verify, context)?; + // Verify each account balance and message pool readiness in parallel + info!("Verifying account balances and message pool readiness with Lotus node..."); + self.verify_accounts_parallel(accounts_to_verify, context)?; info!("Ethereum account funding verified successfully!"); diff --git a/src/commands/start/lotus_utils/account_readiness.rs b/src/commands/start/lotus_utils/account_readiness.rs new file mode 100644 index 00000000..00a60dac --- /dev/null +++ b/src/commands/start/lotus_utils/account_readiness.rs @@ -0,0 +1,65 @@ +//! Read-only checks for account readiness in the Lotus message pool. + +use super::super::step::SetupContext; +use crate::docker::command_logger::run_and_log_command_strings; +use crate::utils::retry::{retry_with_fixed_delay, DEFAULT_MAX_RETRIES, DEFAULT_RETRY_DELAY_SECS}; +use std::error::Error; +use tracing::info; + +/// Wait until the message pool can resolve an account's nonce. +/// +/// A funded balance at the chain head does not mean the message pool has caught up. +/// `--block pending` queries the pool's state view, also used when broadcasting; +/// other block parameters can return zero for a missing actor and hide this race. +/// This checks actor visibility, not every condition for transaction acceptance. +pub fn wait_for_account_nonce( + eth_address: &str, + lotus_rpc_url: &str, + context: &SetupContext, +) -> Result<(), Box> { + let run_id = context.run_id(); + let mut attempt = 0; + + retry_with_fixed_delay( + || { + attempt += 1; + let args: Vec = vec![ + "run".to_string(), + "--rm".to_string(), + "--name".to_string(), + format!("foc-{}-nonce-{}-{}", run_id, eth_address, attempt), + "-u".to_string(), + "foc-user".to_string(), + "--network".to_string(), + "host".to_string(), + crate::constants::BUILDER_DOCKER_IMAGE.to_string(), + "bash".to_string(), + "-c".to_string(), + format!( + "cast nonce {} --block pending --rpc-url {}", + eth_address, lotus_rpc_url + ), + ]; + + let key = format!("account_nonce_{}_{}_{}", run_id, eth_address, attempt); + let output = run_and_log_command_strings("docker", &args, context, &key)?; + + if output.status.success() { + Ok(()) + } else { + Err(format!( + "Pending nonce lookup failed for {}: {}", + eth_address, + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()) + } + }, + DEFAULT_MAX_RETRIES, + DEFAULT_RETRY_DELAY_SECS, + &format!("Account nonce lookup for {}", eth_address), + )?; + + info!("✓ Message pool resolves the nonce for {}", eth_address); + Ok(()) +} diff --git a/src/commands/start/lotus_utils/mod.rs b/src/commands/start/lotus_utils/mod.rs index e31b74ce..d6e124a2 100644 --- a/src/commands/start/lotus_utils/mod.rs +++ b/src/commands/start/lotus_utils/mod.rs @@ -2,6 +2,10 @@ //! //! This module provides shared utilities for working with Lotus daemon. +mod account_readiness; + +pub use account_readiness::wait_for_account_nonce; + use std::error::Error; use std::fs; diff --git a/src/commands/start/multicall3_deploy/deployment.rs b/src/commands/start/multicall3_deploy/deployment.rs index 998bf1da..14e92652 100644 --- a/src/commands/start/multicall3_deploy/deployment.rs +++ b/src/commands/start/multicall3_deploy/deployment.rs @@ -8,68 +8,9 @@ use crate::commands::start::lotus_utils::get_lotus_rpc_url; use crate::docker::command_logger::run_and_log_command_strings; use crate::docker::push_bind_mount; use crate::paths::foc_devnet_multicall3_repo; -use crate::utils::retry::{retry_with_fixed_delay, DEFAULT_MAX_RETRIES, DEFAULT_RETRY_DELAY_SECS}; use std::error::Error; use tracing::{error, info}; -/// Guard the deployment against a lagging message pool. -/// -/// Funding verifies the balance against the chain head, but the pool can lag behind it, -/// and its view is the one the nonce lookup uses, so the broadcast fails with "actor not -/// found". `--block pending` routes this probe through the pool; other block parameters -/// return 0 for a missing actor and would hide the race. -fn wait_for_deployer_nonce( - deployer_eth: &str, - lotus_rpc_url: &str, - run_id: &str, - context: &super::super::step::SetupContext, -) -> Result<(), Box> { - let mut attempt = 0; - - retry_with_fixed_delay( - || { - attempt += 1; - let args: Vec = vec![ - "run".to_string(), - "--rm".to_string(), - "--name".to_string(), - format!("foc-{}-multicall3-nonce-{}", run_id, attempt), - "-u".to_string(), - "foc-user".to_string(), - "--network".to_string(), - "host".to_string(), - crate::constants::BUILDER_DOCKER_IMAGE.to_string(), - "bash".to_string(), - "-c".to_string(), - format!( - "cast nonce {} --block pending --rpc-url {}", - deployer_eth, lotus_rpc_url - ), - ]; - - let key = format!("multicall3_deployer_nonce_{}_{}", run_id, attempt); - let output = run_and_log_command_strings("docker", &args, context, &key)?; - - if output.status.success() { - Ok(()) - } else { - Err(format!( - "Message pool cannot resolve the nonce for {} yet: {}", - deployer_eth, - String::from_utf8_lossy(&output.stderr).trim() - ) - .into()) - } - }, - DEFAULT_MAX_RETRIES, - DEFAULT_RETRY_DELAY_SECS, - "Multicall3 deployer nonce lookup", - )?; - - info!("✓ Message pool resolves the nonce for {}", deployer_eth); - Ok(()) -} - /// Deploy Multicall3 using forge create pub fn deploy_multicall3( private_key: &str, @@ -185,9 +126,6 @@ pub fn perform_deployment( // Deploy Multicall3 contract let lotus_rpc_url = get_lotus_rpc_url(context)?; let run_id = context.run_id(); - - wait_for_deployer_nonce(&multicall3_deployer_eth, &lotus_rpc_url, run_id, context)?; - let multicall3_address = deploy_multicall3(&private_key, &lotus_rpc_url, run_id, context)?; // Store in context