diff --git a/.devcontainer/dev/devcontainer.json b/.devcontainer/dev/devcontainer.json index f8e9e800733..88631f248fd 100644 --- a/.devcontainer/dev/devcontainer.json +++ b/.devcontainer/dev/devcontainer.json @@ -1,6 +1,6 @@ { "name": "Development", - "image": "aztecprotocol/devbox:3.0", + "image": "aztecprotocol/devbox:3.1", "features": { // Use custom fork with noble added to list of supported distros. "./docker-in-docker": {} diff --git a/barretenberg/sol/bootstrap.sh b/barretenberg/sol/bootstrap.sh index 7a6edf633a6..17fe002e885 100755 --- a/barretenberg/sol/bootstrap.sh +++ b/barretenberg/sol/bootstrap.sh @@ -33,8 +33,9 @@ function build_sol { if ! cache_download $artifact; then rm -rf broadcast cache out - forge install - # Ensure libraries are at the correct version + # Check out the libraries at the revisions this repository records. `forge install` is not used for + # this: it syncs every submodule of the repository to foundry.lock, and exits non-zero when the lock + # names a revision that a shallow clone does not have. git submodule update --init --recursive ./lib forge fmt || true diff --git a/barretenberg/sol/foundry.toml b/barretenberg/sol/foundry.toml index 54c16203d2f..07c69f5a3c9 100644 --- a/barretenberg/sol/foundry.toml +++ b/barretenberg/sol/foundry.toml @@ -17,6 +17,12 @@ solc = "../../l1-contracts/solc-0.8.30" [fuzz] runs = 2 +[fmt] +# Foundry 1.5+ would reformat these. They are copied into the verifier templates in +# barretenberg/cpp/src/barretenberg/dsl/acir_proofs, so reformatting them changes the generated +# verifier source and with it the metadata hash in the deployed verifier's bytecode. +ignore = ["src/honk/CommitmentScheme.sol", "src/honk/Relations.sol"] + [lint] ignore = ["./lib/**"] exclude_lints = [ diff --git a/bootstrap.sh b/bootstrap.sh index 3dc1b49875e..8df09ebdd2d 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -16,7 +16,7 @@ export expected_min_node_version=24.12.0 export expected_min_zig_version=0.15.1 export expected_abs_rust_version=1.89.0 export expected_abs_wasi_version=27.0 -export expected_abs_foundry_version=1.4.1 +export expected_abs_foundry_version=1.8.5 export expected_abs_yarn_version=4.13.0 function ensure { diff --git a/build-images/bootstrap.sh b/build-images/bootstrap.sh index 14d1b1d3af7..474aec6c76c 100755 --- a/build-images/bootstrap.sh +++ b/build-images/bootstrap.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash source $(git rev-parse --show-toplevel)/ci3/source_bootstrap -version="3.0" +version="3.1" arch=$(arch) branch=${BRANCH:-$(git rev-parse --abbrev-ref HEAD)} @@ -42,14 +42,13 @@ function build_ec2 { exit 1 fi - # Request new instance. + # Request new on-demand instance. The helper writes ip/iid into the state dir and the terminate + # helper takes that same dir. SSH mode (KEY_NAME set) so we can drive the build over ssh. instance_name=build_image_$(echo -n "$branch" | tr -c 'a-zA-Z0-9-' '_')_$arch - ip_sir=$(NO_SPOT=1 aws_request_instance $instance_name $cpus $arch) - IFS=':' read -r -a parts <<< "$ip_sir" - ip="${parts[0]}" - sir="${parts[1]}" - iid="${parts[2]}" - trap 'aws_terminate_instance $iid $sir || true' EXIT + state_dir=$(mktemp -d /tmp/aws_request_instance.XXXXXX) + trap 'aws_terminate_instance $state_dir || true' EXIT + NO_SPOT=1 KEY_NAME=${KEY_NAME:-build-instance} aws_request_instance $instance_name $cpus $arch $state_dir + local ip=$(cat $state_dir/ip) ssh -F $ci3/aws/build_instance_ssh_config ubuntu@$ip " set -euo pipefail @@ -68,14 +67,11 @@ function build_ec2 { function update_manifests { for target in build devbox sysbox; do # We update the manifest to point to the latest arch specific images, pushed above. - local image=aztecprotocol/$target:$version - # Remove any old local manifest if present. - docker manifest rm $image || true - # Create new manifest and push. - docker manifest create $image \ - --amend aztecprotocol/$target:$version-amd64 \ - --amend aztecprotocol/$target:$version-arm64 - docker manifest push $image + # imagetools, not `docker manifest`: current docker pushes each arch image as an OCI index (image plus + # provenance attestation), which `docker manifest create` rejects as "is a manifest list". + docker buildx imagetools create -t aztecprotocol/$target:$version \ + aztecprotocol/$target:$version-amd64 \ + aztecprotocol/$target:$version-arm64 done } diff --git a/build-images/run.sh b/build-images/run.sh index f89d964e2f1..288937a7264 100755 --- a/build-images/run.sh +++ b/build-images/run.sh @@ -41,5 +41,5 @@ else -vdevbox-var-lib-docker:/var/lib/docker \ -v$HOME/.ssh/id_rsa:/home/aztec-dev/.ssh/id_rsa:ro \ --privileged \ - aztecprotocol/devbox:3.0 + aztecprotocol/devbox:3.1 fi diff --git a/build-images/src/Dockerfile b/build-images/src/Dockerfile index 875b10c4765..67dd8ced00c 100644 --- a/build-images/src/Dockerfile +++ b/build-images/src/Dockerfile @@ -34,7 +34,7 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --de ######################################################################################################################## # Build foundry. FROM base-build AS foundry -ENV FOUNDRY_VERSION=v1.4.1 +ENV FOUNDRY_VERSION=v1.8.5 ENV FOUNDRY_BIN_DIR="/root/.foundry/bin" ENV RUSTUP_HOME=/opt/rust/rustup ENV CARGO_HOME=/opt/rust/cargo @@ -112,10 +112,9 @@ RUN apt update && \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -# Install clang 18/20. +# Install clang 20. RUN wget https://apt.llvm.org/llvm.sh && \ chmod +x llvm.sh && \ - ./llvm.sh 18 all && \ ./llvm.sh 20 all && \ rm llvm.sh diff --git a/build-images/src/docker_daemon.json b/build-images/src/docker_daemon.json index 2deedaf0cf7..9f52ae2d67c 100644 --- a/build-images/src/docker_daemon.json +++ b/build-images/src/docker_daemon.json @@ -2,5 +2,6 @@ "default-address-pools": [ { "base": "172.20.0.0/16", "size": 24 }, { "base": "172.21.0.0/16", "size": 24 } - ] + ], + "features": { "containerd-snapshotter": false } } diff --git a/ci3/aws/ami_id_amd64 b/ci3/aws/ami_id_amd64 index a837bf6c3d6..43b3d727378 100644 --- a/ci3/aws/ami_id_amd64 +++ b/ci3/aws/ami_id_amd64 @@ -1 +1 @@ -ami-067627aa971a1dcbb +ami-0b92d1f5e1e85dd99 diff --git a/ci3/aws/ami_id_arm64 b/ci3/aws/ami_id_arm64 index 040b84c6025..a34ef19e926 100644 --- a/ci3/aws/ami_id_arm64 +++ b/ci3/aws/ami_id_arm64 @@ -1 +1 @@ -ami-099a6d94f71a679d0 +ami-0bd6dbacd3fb445bc diff --git a/ci3/aws/ami_update.sh b/ci3/aws/ami_update.sh index 6d65076c45d..1ff6434e33f 100755 --- a/ci3/aws/ami_update.sh +++ b/ci3/aws/ami_update.sh @@ -7,11 +7,11 @@ arch=${ARCH:-$(arch)} # Trap function to terminate our running instance when the script exits. function on_exit { - [ "${NO_TERMINATE:-0}" -eq 0 ] && aws_terminate_instance $iid $sir + [ "${NO_TERMINATE:-0}" -eq 0 ] && aws_terminate_instance $state_dir } -if [ ! -f $HOME/.aws/build_instance_credentials ]; then - echo "You need the build instance credentials located at: $HOME/.aws/build_instance_credentials" +if ! aws sts get-caller-identity > /dev/null 2>&1; then + echo "AWS CLI credentials for the CI account are required (aws sts get-caller-identity failed)." exit 1 fi @@ -27,13 +27,12 @@ case "$arch" in exit 1 esac -# Request new instance (ami: ubuntu 24.04 LTS). -ip_sir=$(AMI=$ami aws_request_instance ami_update_$arch 4 $arch) -parts=(${ip_sir//:/ }) -ip="${parts[0]}" -sir="${parts[1]}" -iid="${parts[2]}" +# Request new instance (ami: ubuntu 24.04 LTS). SSH mode: the setup below is driven over ssh. +state_dir=$(mktemp -d /tmp/aws_request_instance.XXXXXX) trap on_exit EXIT +AMI=$ami KEY_NAME=${KEY_NAME:-build-instance} aws_request_instance ami_update_$arch 4 $arch $state_dir +ip=$(cat $state_dir/ip) +iid=$(cat $state_dir/iid | tr -d '\n\r' | xargs) echo "Instance ip: $ip" @@ -55,8 +54,8 @@ ssh $ssh_args -F build_instance_ssh_config ubuntu@$ip 'export CRS_FORMAT=uncompr # Pull devbox onto host, and build into docker-in-docker volume. ssh $ssh_args -F build_instance_ssh_config ubuntu@$ip " - docker run --privileged --rm -v bootstrap_ci_local_docker:/var/lib/docker aztecprotocol/devbox:3.0 bash -c \" - docker pull aztecprotocol/build:3.0 + docker run --privileged --rm -v bootstrap_ci_local_docker:/var/lib/docker aztecprotocol/devbox:3.1 bash -c \" + docker pull aztecprotocol/build:3.1 \" " diff --git a/ci3/bootstrap_ec2 b/ci3/bootstrap_ec2 index 703d0cacf61..402a5847998 100755 --- a/ci3/bootstrap_ec2 +++ b/ci3/bootstrap_ec2 @@ -401,7 +401,7 @@ start_build() { --pids-limit=65536 \ --shm-size=2g \ --ulimit nofile=1048576:1048576 \ - aztecprotocol/devbox:3.0 bash -c $(printf '%q' "$container_script") + aztecprotocol/devbox:3.1 bash -c $(printf '%q' "$container_script") } # If stdout is a tty, run in foreground, otherwise run in background and handle spot termination notices. diff --git a/ci3/docker_isolate b/ci3/docker_isolate index 5e6724c7bb8..210937e0d9a 100755 --- a/ci3/docker_isolate +++ b/ci3/docker_isolate @@ -79,7 +79,7 @@ cid=$(docker run -d \ -e TEST_TIMING_FILE \ -e RUN_ID \ "${arg_env_vars[@]}" \ - aztecprotocol/build:3.0 \ + aztecprotocol/build:3.1 \ /bin/bash -c "$cmd") set +x # At this point we know it's started, and can handle SIGTERM to kill the container. diff --git a/ci3/tests/signal_test b/ci3/tests/signal_test index 1999eca1137..e76d9735091 100755 --- a/ci3/tests/signal_test +++ b/ci3/tests/signal_test @@ -73,8 +73,8 @@ if ! docker info &>/dev/null; then exit 0 fi -if ! docker image inspect aztecprotocol/build:3.0 &>/dev/null; then - echo "SKIP: aztecprotocol/build:3.0 image not available" +if ! docker image inspect aztecprotocol/build:3.1 &>/dev/null; then + echo "SKIP: aztecprotocol/build:3.1 image not available" exit 0 fi diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index 3d527c5886a..8193a733fe1 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -64,10 +64,9 @@ function build_src { # Clean rm -rf broadcast cache out serve - # Install - forge install - - # Ensure libraries are at the correct version + # Check out the libraries at the revisions this repository records. `forge install` is not used for + # this: it syncs every submodule of the repository to foundry.lock, and exits non-zero when the lock + # names a revision that a shallow clone does not have. git submodule update --init --recursive ./lib # Compile contracts @@ -108,6 +107,11 @@ function build_verifier { script/deploy/*.s.sol \ test/script/*.t.sol + # Leave the artifact set fully built so `forge test` has nothing left to compile. When forge + # incrementally compiles a handful of leftover files on top of an explicit-path build, the + # resulting artifacts are inconsistent and the Rollup-deploying suites revert in setUp. + forge build + cache_upload $artifact out cache generated fi } diff --git a/l1-contracts/foundry.toml b/l1-contracts/foundry.toml index 835637e0187..b2dc5d24443 100644 --- a/l1-contracts/foundry.toml +++ b/l1-contracts/foundry.toml @@ -11,6 +11,10 @@ evm_version = 'prague' optimizer = true optimizer_runs = 100 match_path = "test/**/*.t.sol" +# Run each test as one transaction rather than isolating every top-level call into its own. Per-call +# isolation charges cold-access and intrinsic gas on each call, which pushes the large validator-set +# tests past the gas limit. Gas reports isolate every call whatever this is set to. +isolate = false # Helper to get all the contract names in the src/governance and src/core directories # find ./src/governance ./src/core -type f -name "*.sol" -exec grep -h "^contract [A-Za-z]" {} \; | sed -E 's/contract ([A-Za-z0-9_]+).*/"\1"/' | tr "\n" ", " gas_reports = [ @@ -34,6 +38,12 @@ gas_reports = [ "MultiAdder", ] +# Remappings are listed explicitly instead of auto-detected from lib/. They are recorded in each +# contract's metadata, so its hash and therefore the deployed bytecode depend on them. Foundry 1.5+ +# auto-detects extra context-scoped remappings for nested libraries, some with absolute paths, which +# would make the bytecode differ from the deployed contracts and between checkouts. The last six +# entries of each list are what Foundry 1.4 used to auto-detect. +auto_detect_remappings = false remappings = [ "@oz/=lib/openzeppelin-contracts/contracts/", "@aztec/=src", @@ -42,6 +52,12 @@ remappings = [ "@zkpassport/=lib/circuits/src/solidity/src/", "@zkpassport-test/=lib/circuits/src/solidity/test", "@aztec-blob-lib/=src/mock/libraries", + "@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/", + "circuits/=lib/circuits/src/", + "erc4626-tests/=lib/openzeppelin-contracts/lib/erc4626-tests/", + "forge-std/=lib/forge-std/src/", + "halmos-cheatcodes/=lib/openzeppelin-contracts/lib/halmos-cheatcodes/src/", + "openzeppelin-contracts/=lib/openzeppelin-contracts/", ] # See more config options https://github.com/foundry-rs/foundry/tree/master/config @@ -71,11 +87,22 @@ no_match_contract = "(ScreamAndShoutTest|UniswapPortalTest|MerkleCheck)" [fmt] line_length = 120 tab_width = 2 -variable_override_spacing = false wrap_comments = true number_underscore = "thousands" override_spacing = false -ignore = ["script/deploy/HonkVerifier.sol"] +# HonkVerifier.sol is generated. The other files are ones Foundry 1.5+ would reformat, left as they were +# deployed: any change to a source file, whitespace included, changes the metadata hash in the bytecode +# of every contract built from it. +ignore = [ + "src/core/EscapeHatch.sol", + "src/core/libraries/rollup/FeeLib.sol", + "src/core/libraries/rollup/RewardLib.sol", + "src/core/libraries/rollup/STFLib.sol", + "src/governance/GSE.sol", + "src/governance/libraries/compressed-data/Proposal.sol", + "script/deploy/DeployRollupForUpgradeV6.s.sol", + "script/deploy/HonkVerifier.sol", +] [lint] ignore = ["./lib/**"] @@ -103,6 +130,12 @@ remappings = [ "@zkpassport/=lib/circuits/src/solidity/src/", "@zkpassport-test/=lib/circuits/src/solidity/test", "@aztec-blob-lib/=src/core/libraries/rollup/", + "@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/", + "circuits/=lib/circuits/src/", + "erc4626-tests/=lib/openzeppelin-contracts/lib/erc4626-tests/", + "forge-std/=lib/forge-std/src/", + "halmos-cheatcodes/=lib/openzeppelin-contracts/lib/halmos-cheatcodes/src/", + "openzeppelin-contracts/=lib/openzeppelin-contracts/", ] [profile.coverage] @@ -125,4 +158,10 @@ remappings = [ "@zkpassport/=src/mock/coverage/zkpassport/", "@zkpassport-test/=lib/circuits/src/solidity/test", "@aztec-blob-lib/=src/mock/libraries", + "@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/", + "circuits/=lib/circuits/src/", + "erc4626-tests/=lib/openzeppelin-contracts/lib/erc4626-tests/", + "forge-std/=lib/forge-std/src/", + "halmos-cheatcodes/=lib/openzeppelin-contracts/lib/halmos-cheatcodes/src/", + "openzeppelin-contracts/=lib/openzeppelin-contracts/", ] diff --git a/l1-contracts/scripts/forge_broadcast.js b/l1-contracts/scripts/forge_broadcast.js index 502f77eec4b..59ec3d024df 100755 --- a/l1-contracts/scripts/forge_broadcast.js +++ b/l1-contracts/scripts/forge_broadcast.js @@ -5,13 +5,13 @@ // speed. But a batched broadcast (forge's default sends many txs at once) can race the auto-miner: // it mines a block on the first ready tx and may leave txs that arrived just after the trigger // sitting in the pool, so forge waits forever for their receipts. We avoid the race without touching -// anvil's mining mode by broadcasting one tx at a time (`--batch-size 1`) only when anvil has -// automine ON: with a single tx in flight there is nothing for the auto-miner to strand. +// anvil's mining mode by broadcasting one tx at a time (`--slow`) only when anvil has automine ON: +// with a single tx in flight there is nothing for the auto-miner to strand. // // When anvil is in interval (or no) mining mode the race does not exist — the miner drains the whole // pool on each block — and serializing to one tx per block would stall the deploy for a full block -// interval per transaction, blowing past the broadcast timeout. There (and on real chains) we keep a -// larger batch size. A hard timeout guards against a broadcast hanging indefinitely. +// interval per transaction, blowing past the broadcast timeout. There (and on real chains) we keep +// forge's default batched sending. A hard timeout guards against a broadcast hanging indefinitely. // // A deploy that queues INITIAL_VALIDATORS does not activate them itself: once the deploy broadcast // has landed, this script runs FlushEntryQueue.s.sol against the deployed Rollup as a second, @@ -26,14 +26,125 @@ // simulated broadcast. On anvil, an empty block is mined after the flush so the flush is never in // the head block that callers estimate gas against (see the comment at the evm_mine call). // +// forge holds an exclusive lock on the script's recovery state (`cache/