diff --git a/.github/scripts/mem-monitor.sh b/.github/scripts/mem-monitor.sh new file mode 100755 index 000000000000..819c0a04c40e --- /dev/null +++ b/.github/scripts/mem-monitor.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Log memory to the job log every 15s during a test run, so readings made +# before a VM kill survive. Run in the background and kill it when done. +while sleep 15; do + printf '\n[mem %s] %s\n' "$(date +%T)" "$(free -m | awk '/Mem:/{print "used="$3"MB avail="$7"MB"}')" + ps -eo rss=,comm= | awk '{s[$2]+=$1; c[$2]++} END{for(k in s) printf "%d %d %s\n", s[k]/1024, c[k], k}' | sort -rn | head -n 5 | awk '{printf "[mem] %6d MB x%-4d %s\n", $1, $2, $3}' +done diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eec91fbd1373..63cd78d85cff 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -707,14 +707,12 @@ jobs: TEST_DEBUG: 1 PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} --test-group-random-seed=42 run: | + .github/scripts/mem-monitor.sh & + mem_pid=$! + trap 'kill $mem_pid 2>/dev/null' EXIT # -n $(nproc) rather than $(nproc)+1: one fewer parallel node-cluster # keeps peak RSS under the 16GB hosted-runner ceiling. VALGRIND=1 sg wireshark "uv run eatmydata pytest tests/ -n $(nproc) ${PYTEST_OPTS} --test-group=${{ matrix.GROUP }} --test-group-count=${{ matrix.GROUP_COUNT }}" - - name: Report OOM kills - if: failure() - run: | - echo "=== dmesg (OOM) ==="; sudo dmesg | grep -i -E 'oom|out of memory|killed process' || echo "no OOM lines in dmesg" - echo "=== kernel log ==="; sudo journalctl -k --no-pager | tail -n 50 || true - name: Upload test results if: always() uses: actions/upload-artifact@v7 @@ -778,14 +776,11 @@ jobs: env: PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} --test-group-random-seed=42 run: | - # -n $(nproc) rather than $(nproc)+1: one fewer parallel node-cluster - # keeps ASan peak RSS under the 16GB hosted-runner ceiling. - sg wireshark "uv run eatmydata pytest tests/ -n $(nproc) ${PYTEST_OPTS} --test-group=${{ matrix.GROUP }} --test-group-count=${{ matrix.GROUP_COUNT }}" - - name: Report OOM kills - if: failure() - run: | - echo "=== dmesg (OOM) ==="; sudo dmesg | grep -i -E 'oom|out of memory|killed process' || echo "no OOM lines in dmesg" - echo "=== kernel log ==="; sudo journalctl -k --no-pager | tail -n 50 || true + .github/scripts/mem-monitor.sh & + mem_pid=$! + trap 'kill $mem_pid 2>/dev/null' EXIT + # loadgroup: tests marked heavy run one at a time on a single worker. + sg wireshark "uv run eatmydata pytest tests/ -n $(nproc) --dist loadgroup ${PYTEST_OPTS} --test-group=${{ matrix.GROUP }} --test-group-count=${{ matrix.GROUP_COUNT }}" - name: Upload test results if: always() uses: actions/upload-artifact@v7 diff --git a/contrib/pyln-testing/pyln/testing/utils.py b/contrib/pyln-testing/pyln/testing/utils.py index 3c5973a8aaff..c8552424e88a 100644 --- a/contrib/pyln-testing/pyln/testing/utils.py +++ b/contrib/pyln-testing/pyln/testing/utils.py @@ -81,6 +81,7 @@ def env(name, default=None): VALGRIND = env("VALGRIND") == "1" +ASAN = env("ASAN", "0") == "1" TEST_NETWORK = env("TEST_NETWORK", 'regtest') TEST_DEBUG = env("TEST_DEBUG", "0") == "1" diff --git a/tests/test_askrene.py b/tests/test_askrene.py index bbf07e63f7c9..87501bb3bff3 100644 --- a/tests/test_askrene.py +++ b/tests/test_askrene.py @@ -2107,6 +2107,7 @@ def test_unreserve_all(node_factory): assert l1.rpc.askrene_listreservations() == {"reservations": []} +@pytest.mark.xdist_group("heavy") def test_reservations_leak_under_load(node_factory, executor): """Stress-test reservation cleanup: concurrent payments over shared channels must leave zero stale reservations after all payments settle.""" diff --git a/tests/test_closing.py b/tests/test_closing.py index d4c678b1caf0..02cc27737324 100644 --- a/tests/test_closing.py +++ b/tests/test_closing.py @@ -3730,6 +3730,7 @@ def test_invalid_upfront_shutdown_script(node_factory, bitcoind, executor): @pytest.mark.slow_test +@pytest.mark.xdist_group("heavy") def test_segwit_shutdown_script(node_factory, bitcoind, executor): """ Try a range of future segwit versions as shutdown scripts. We create many nodes, so this is quite slow under valgrind diff --git a/tests/test_connection.py b/tests/test_connection.py index 5b1104a81280..61c09a62edcb 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -15,7 +15,7 @@ mine_funding_to_announce, first_scid, CHANNEL_SIZE ) -from pyln.testing.utils import VALGRIND, EXPERIMENTAL_DUAL_FUND, FUNDAMOUNT, RUST, SLOW_MACHINE +from pyln.testing.utils import VALGRIND, ASAN, EXPERIMENTAL_DUAL_FUND, FUNDAMOUNT, RUST, SLOW_MACHINE import coincurve import hashlib @@ -1479,6 +1479,7 @@ def test_funding_v2_corners(node_factory, bitcoind): @pytest.mark.slow_test @pytest.mark.openchannel('v1') +@pytest.mark.xdist_group("heavy") def test_funding_cancel_race(node_factory, bitcoind, executor): l1 = node_factory.get_node() @@ -1490,6 +1491,8 @@ def test_funding_cancel_race(node_factory, bitcoind, executor): if VALGRIND: num = 5 + elif ASAN: + num = 10 elif SLOW_MACHINE: num = 20 else: @@ -3646,6 +3649,7 @@ def test_feerate_stress(node_factory, executor): @pytest.mark.slow_test +@pytest.mark.xdist_group("heavy") def test_pay_disconnect_stress(node_factory, executor): """Expose race in htlc restoration in channeld: 50% chance of failure""" if VALGRIND: @@ -5521,11 +5525,15 @@ def reestablished(): @pytest.mark.slow_test +@pytest.mark.xdist_group("heavy") def test_parallel_opens(node_factory, bitcoind, executor): """On macOS a socket closed by its sender while in flight over SCM_RIGHTS could arrive unable to read, losing a subdaemon its hsmd or peer connection under load (#5808).""" - pairs, opens = 6, 50 + if ASAN: + pairs, opens = 2, 10 + else: + pairs, opens = 6, 50 nodes = node_factory.get_nodes(2 * pairs) openers, peers = nodes[:pairs], nodes[pairs:] diff --git a/tests/test_xpay.py b/tests/test_xpay.py index 301f89fd4708..6cc0a598b775 100644 --- a/tests/test_xpay.py +++ b/tests/test_xpay.py @@ -1152,6 +1152,7 @@ def test_xpay_shadow_cltv(node_factory, bitcoind): assert l2.daemon.is_in_log(f"Adding HTLC .* amount=10000msat cltv={expected_cltv} gave CHANNEL_ERR_ADD_OK") +@pytest.mark.xdist_group("heavy") def test_xpay_limited_max_accepted_htlcs(node_factory): """xpay should try to reduce flows to 6 if there is an unannounced channel, and only try more if that fails""" CHANNEL_SIZE_SATS = 10**6