From d91b65daba402ed5e9cabb1a26c26558d12a2eb5 Mon Sep 17 00:00:00 2001 From: daywalker90 Date: Fri, 9 Oct 2026 11:12:56 +0200 Subject: [PATCH 1/4] CI: stream memory usage during Valgrind/ASan `if: failure()` does not trigger when the VM gets killed, neither would `if: always()`. Instead we print used/available memory and the top five processes by total RSS every 15 seconds during the test step, so the readings before a kill survive in the job log. Per-process RSS alone was misleading since many lightningd processes each show small RSS. Changelog-None --- .github/workflows/ci.yaml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eec91fbd1373..c1c1ec942a8c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -707,14 +707,16 @@ jobs: TEST_DEBUG: 1 PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} --test-group-random-seed=42 run: | + # Memory is logged to the job log during the run, so readings before a VM kill survive. + (while sleep 15; do + printf '\n[mem %s] %s\n' "$(date +%T)" "$(free -m | awk '/Mem:/{print "used="$3"MB avail="$7"MB"}')" + ps -eo rss=,comm= | awk '{s[$2]+=$1; c[$2]++} END{for(k in s) printf "%d %d %s\n", s[k]/1024, c[k], k}' | sort -rn | head -n 5 | awk '{printf "[mem] %6d MB x%-4d %s\n", $1, $2, $3}' + done) & + mem_pid=$! + trap 'kill $mem_pid 2>/dev/null' EXIT # -n $(nproc) rather than $(nproc)+1: one fewer parallel node-cluster # keeps peak RSS under the 16GB hosted-runner ceiling. VALGRIND=1 sg wireshark "uv run eatmydata pytest tests/ -n $(nproc) ${PYTEST_OPTS} --test-group=${{ matrix.GROUP }} --test-group-count=${{ matrix.GROUP_COUNT }}" - - name: Report OOM kills - if: failure() - run: | - echo "=== dmesg (OOM) ==="; sudo dmesg | grep -i -E 'oom|out of memory|killed process' || echo "no OOM lines in dmesg" - echo "=== kernel log ==="; sudo journalctl -k --no-pager | tail -n 50 || true - name: Upload test results if: always() uses: actions/upload-artifact@v7 @@ -778,14 +780,16 @@ jobs: env: PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} --test-group-random-seed=42 run: | + # Memory is logged to the job log during the run, so readings before a VM kill survive. + (while sleep 15; do + printf '\n[mem %s] %s\n' "$(date +%T)" "$(free -m | awk '/Mem:/{print "used="$3"MB avail="$7"MB"}')" + ps -eo rss=,comm= | awk '{s[$2]+=$1; c[$2]++} END{for(k in s) printf "%d %d %s\n", s[k]/1024, c[k], k}' | sort -rn | head -n 5 | awk '{printf "[mem] %6d MB x%-4d %s\n", $1, $2, $3}' + done) & + mem_pid=$! + trap 'kill $mem_pid 2>/dev/null' EXIT # -n $(nproc) rather than $(nproc)+1: one fewer parallel node-cluster # keeps ASan peak RSS under the 16GB hosted-runner ceiling. sg wireshark "uv run eatmydata pytest tests/ -n $(nproc) ${PYTEST_OPTS} --test-group=${{ matrix.GROUP }} --test-group-count=${{ matrix.GROUP_COUNT }}" - - name: Report OOM kills - if: failure() - run: | - echo "=== dmesg (OOM) ==="; sudo dmesg | grep -i -E 'oom|out of memory|killed process' || echo "no OOM lines in dmesg" - echo "=== kernel log ==="; sudo journalctl -k --no-pager | tail -n 50 || true - name: Upload test results if: always() uses: actions/upload-artifact@v7 From efac2ccf450f8da5d27ca86ec4fb89f11ad7f809 Mon Sep 17 00:00:00 2001 From: daywalker90 Date: Mon, 12 Oct 2026 15:40:14 +0200 Subject: [PATCH 2/4] tests: shrink ASan-heavy tests to fit the CI runner test_parallel_opens starts about 600 channeld processes at full size. Under ASan that peaks near 33 GB, more than a 16 GB CI runner has, so the sanitizer shard is killed. Use 2 pairs x 10 opens when ASAN=1. The code path is unchanged and the full size still runs everywhere else. test_funding_cancel_race peaks near 15 GB PSS under ASan with 20 nodes, close to the 16 GB limit. Use 10 nodes when ASAN=1 (peak about 8 GB). The race is still exercised against each node. --- tests/test_connection.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index 5b1104a81280..18a6d113ef52 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -15,7 +15,7 @@ mine_funding_to_announce, first_scid, CHANNEL_SIZE ) -from pyln.testing.utils import VALGRIND, EXPERIMENTAL_DUAL_FUND, FUNDAMOUNT, RUST, SLOW_MACHINE +from pyln.testing.utils import VALGRIND, EXPERIMENTAL_DUAL_FUND, FUNDAMOUNT, RUST, SLOW_MACHINE, env import coincurve import hashlib @@ -1490,6 +1490,8 @@ def test_funding_cancel_race(node_factory, bitcoind, executor): if VALGRIND: num = 5 + elif env('ASAN') == '1': + num = 10 elif SLOW_MACHINE: num = 20 else: @@ -5525,7 +5527,10 @@ def test_parallel_opens(node_factory, bitcoind, executor): """On macOS a socket closed by its sender while in flight over SCM_RIGHTS could arrive unable to read, losing a subdaemon its hsmd or peer connection under load (#5808).""" - pairs, opens = 6, 50 + if env('ASAN') == '1': + pairs, opens = 2, 10 + else: + pairs, opens = 6, 50 nodes = node_factory.get_nodes(2 * pairs) openers, peers = nodes[:pairs], nodes[pairs:] From 17f7544634d91e2daa9d94efcdac9c794cc0348e Mon Sep 17 00:00:00 2001 From: daywalker90 Date: Mon, 12 Oct 2026 15:42:06 +0200 Subject: [PATCH 3/4] tests: run the heavy ASan tests one at a time Under ASan a handful of tests each need several GB and some reach 15-20 GB. When they land in the same shard and run together, the shard is killed by the 16 GB runner. Mark them xdist_group('heavy') and run the ASan/UBSan job with --dist loadgroup so they share one worker and never overlap. Other jobs do not use loadgroup, so the marker has no effect there. --- .github/workflows/ci.yaml | 5 ++--- tests/test_askrene.py | 1 + tests/test_closing.py | 1 + tests/test_connection.py | 3 +++ tests/test_xpay.py | 1 + 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c1c1ec942a8c..7323964b5433 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -787,9 +787,8 @@ jobs: done) & mem_pid=$! trap 'kill $mem_pid 2>/dev/null' EXIT - # -n $(nproc) rather than $(nproc)+1: one fewer parallel node-cluster - # keeps ASan peak RSS under the 16GB hosted-runner ceiling. - sg wireshark "uv run eatmydata pytest tests/ -n $(nproc) ${PYTEST_OPTS} --test-group=${{ matrix.GROUP }} --test-group-count=${{ matrix.GROUP_COUNT }}" + # loadgroup: tests marked heavy run one at a time on a single worker. + sg wireshark "uv run eatmydata pytest tests/ -n $(nproc) --dist loadgroup ${PYTEST_OPTS} --test-group=${{ matrix.GROUP }} --test-group-count=${{ matrix.GROUP_COUNT }}" - name: Upload test results if: always() uses: actions/upload-artifact@v7 diff --git a/tests/test_askrene.py b/tests/test_askrene.py index bbf07e63f7c9..87501bb3bff3 100644 --- a/tests/test_askrene.py +++ b/tests/test_askrene.py @@ -2107,6 +2107,7 @@ def test_unreserve_all(node_factory): assert l1.rpc.askrene_listreservations() == {"reservations": []} +@pytest.mark.xdist_group("heavy") def test_reservations_leak_under_load(node_factory, executor): """Stress-test reservation cleanup: concurrent payments over shared channels must leave zero stale reservations after all payments settle.""" diff --git a/tests/test_closing.py b/tests/test_closing.py index d4c678b1caf0..02cc27737324 100644 --- a/tests/test_closing.py +++ b/tests/test_closing.py @@ -3730,6 +3730,7 @@ def test_invalid_upfront_shutdown_script(node_factory, bitcoind, executor): @pytest.mark.slow_test +@pytest.mark.xdist_group("heavy") def test_segwit_shutdown_script(node_factory, bitcoind, executor): """ Try a range of future segwit versions as shutdown scripts. We create many nodes, so this is quite slow under valgrind diff --git a/tests/test_connection.py b/tests/test_connection.py index 18a6d113ef52..58b117862a2d 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1479,6 +1479,7 @@ def test_funding_v2_corners(node_factory, bitcoind): @pytest.mark.slow_test @pytest.mark.openchannel('v1') +@pytest.mark.xdist_group("heavy") def test_funding_cancel_race(node_factory, bitcoind, executor): l1 = node_factory.get_node() @@ -3648,6 +3649,7 @@ def test_feerate_stress(node_factory, executor): @pytest.mark.slow_test +@pytest.mark.xdist_group("heavy") def test_pay_disconnect_stress(node_factory, executor): """Expose race in htlc restoration in channeld: 50% chance of failure""" if VALGRIND: @@ -5523,6 +5525,7 @@ def reestablished(): @pytest.mark.slow_test +@pytest.mark.xdist_group("heavy") def test_parallel_opens(node_factory, bitcoind, executor): """On macOS a socket closed by its sender while in flight over SCM_RIGHTS could arrive unable to read, losing a subdaemon its hsmd or peer diff --git a/tests/test_xpay.py b/tests/test_xpay.py index 301f89fd4708..6cc0a598b775 100644 --- a/tests/test_xpay.py +++ b/tests/test_xpay.py @@ -1152,6 +1152,7 @@ def test_xpay_shadow_cltv(node_factory, bitcoind): assert l2.daemon.is_in_log(f"Adding HTLC .* amount=10000msat cltv={expected_cltv} gave CHANNEL_ERR_ADD_OK") +@pytest.mark.xdist_group("heavy") def test_xpay_limited_max_accepted_htlcs(node_factory): """xpay should try to reduce flows to 6 if there is an unannounced channel, and only try more if that fails""" CHANNEL_SIZE_SATS = 10**6 From d71bab089c1a89d2eb870aa13d019eec795713b5 Mon Sep 17 00:00:00 2001 From: daywalker90 Date: Fri, 16 Oct 2026 12:44:59 +0200 Subject: [PATCH 4/4] CI: move the memory monitor into a script, and add an ASAN constant The memory logging loop was duplicated in the valgrind and sanitizer jobs. Move it to .github/scripts/mem-monitor.sh so each job starts it with one line. Add ASAN to pyln.testing.utils next to VALGRIND, and use it in test_connection.py in place of reading the variable directly. Changelog-None Signed-off-by: daywalker90 --- .github/scripts/mem-monitor.sh | 7 +++++++ .github/workflows/ci.yaml | 12 ++---------- contrib/pyln-testing/pyln/testing/utils.py | 1 + tests/test_connection.py | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) create mode 100755 .github/scripts/mem-monitor.sh diff --git a/.github/scripts/mem-monitor.sh b/.github/scripts/mem-monitor.sh new file mode 100755 index 000000000000..819c0a04c40e --- /dev/null +++ b/.github/scripts/mem-monitor.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Log memory to the job log every 15s during a test run, so readings made +# before a VM kill survive. Run in the background and kill it when done. +while sleep 15; do + printf '\n[mem %s] %s\n' "$(date +%T)" "$(free -m | awk '/Mem:/{print "used="$3"MB avail="$7"MB"}')" + ps -eo rss=,comm= | awk '{s[$2]+=$1; c[$2]++} END{for(k in s) printf "%d %d %s\n", s[k]/1024, c[k], k}' | sort -rn | head -n 5 | awk '{printf "[mem] %6d MB x%-4d %s\n", $1, $2, $3}' +done diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7323964b5433..63cd78d85cff 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -707,11 +707,7 @@ jobs: TEST_DEBUG: 1 PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} --test-group-random-seed=42 run: | - # Memory is logged to the job log during the run, so readings before a VM kill survive. - (while sleep 15; do - printf '\n[mem %s] %s\n' "$(date +%T)" "$(free -m | awk '/Mem:/{print "used="$3"MB avail="$7"MB"}')" - ps -eo rss=,comm= | awk '{s[$2]+=$1; c[$2]++} END{for(k in s) printf "%d %d %s\n", s[k]/1024, c[k], k}' | sort -rn | head -n 5 | awk '{printf "[mem] %6d MB x%-4d %s\n", $1, $2, $3}' - done) & + .github/scripts/mem-monitor.sh & mem_pid=$! trap 'kill $mem_pid 2>/dev/null' EXIT # -n $(nproc) rather than $(nproc)+1: one fewer parallel node-cluster @@ -780,11 +776,7 @@ jobs: env: PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} --test-group-random-seed=42 run: | - # Memory is logged to the job log during the run, so readings before a VM kill survive. - (while sleep 15; do - printf '\n[mem %s] %s\n' "$(date +%T)" "$(free -m | awk '/Mem:/{print "used="$3"MB avail="$7"MB"}')" - ps -eo rss=,comm= | awk '{s[$2]+=$1; c[$2]++} END{for(k in s) printf "%d %d %s\n", s[k]/1024, c[k], k}' | sort -rn | head -n 5 | awk '{printf "[mem] %6d MB x%-4d %s\n", $1, $2, $3}' - done) & + .github/scripts/mem-monitor.sh & mem_pid=$! trap 'kill $mem_pid 2>/dev/null' EXIT # loadgroup: tests marked heavy run one at a time on a single worker. diff --git a/contrib/pyln-testing/pyln/testing/utils.py b/contrib/pyln-testing/pyln/testing/utils.py index 3c5973a8aaff..c8552424e88a 100644 --- a/contrib/pyln-testing/pyln/testing/utils.py +++ b/contrib/pyln-testing/pyln/testing/utils.py @@ -81,6 +81,7 @@ def env(name, default=None): VALGRIND = env("VALGRIND") == "1" +ASAN = env("ASAN", "0") == "1" TEST_NETWORK = env("TEST_NETWORK", 'regtest') TEST_DEBUG = env("TEST_DEBUG", "0") == "1" diff --git a/tests/test_connection.py b/tests/test_connection.py index 58b117862a2d..61c09a62edcb 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -15,7 +15,7 @@ mine_funding_to_announce, first_scid, CHANNEL_SIZE ) -from pyln.testing.utils import VALGRIND, EXPERIMENTAL_DUAL_FUND, FUNDAMOUNT, RUST, SLOW_MACHINE, env +from pyln.testing.utils import VALGRIND, ASAN, EXPERIMENTAL_DUAL_FUND, FUNDAMOUNT, RUST, SLOW_MACHINE import coincurve import hashlib @@ -1491,7 +1491,7 @@ def test_funding_cancel_race(node_factory, bitcoind, executor): if VALGRIND: num = 5 - elif env('ASAN') == '1': + elif ASAN: num = 10 elif SLOW_MACHINE: num = 20 @@ -5530,7 +5530,7 @@ def test_parallel_opens(node_factory, bitcoind, executor): """On macOS a socket closed by its sender while in flight over SCM_RIGHTS could arrive unable to read, losing a subdaemon its hsmd or peer connection under load (#5808).""" - if env('ASAN') == '1': + if ASAN: pairs, opens = 2, 10 else: pairs, opens = 6, 50