CI: fix ASan shard getting killed

.github/scripts/mem-monitor.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Log memory to the job log every 15s during a test run, so readings made
+# before a VM kill survive. Run in the background and kill it when done.
+while sleep 15; do
+  printf '\n[mem %s] %s\n' "$(date +%T)" "$(free -m | awk '/Mem:/{print "used="$3"MB avail="$7"MB"}')"
+  ps -eo rss=,comm= | awk '{s[$2]+=$1; c[$2]++} END{for(k in s) printf "%d %d %s\n", s[k]/1024, c[k], k}' | sort -rn | head -n 5 | awk '{printf "[mem]   %6d MB  x%-4d %s\n", $1, $2, $3}'
+done

.github/workflows/ci.yaml

@@ -707,14 +707,12 @@ jobs:
           TEST_DEBUG: 1
           PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} --test-group-random-seed=42
         run: |
+          .github/scripts/mem-monitor.sh &
+          mem_pid=$!
+          trap 'kill $mem_pid 2>/dev/null' EXIT
           # -n $(nproc) rather than $(nproc)+1: one fewer parallel node-cluster
           # keeps peak RSS under the 16GB hosted-runner ceiling.
           VALGRIND=1 sg wireshark "uv run eatmydata pytest tests/ -n $(nproc) ${PYTEST_OPTS} --test-group=${{ matrix.GROUP }} --test-group-count=${{ matrix.GROUP_COUNT }}"
-      - name: Report OOM kills
-        if: failure()
-        run: |
-          echo "=== dmesg (OOM) ==="; sudo dmesg | grep -i -E 'oom|out of memory|killed process' || echo "no OOM lines in dmesg"
-          echo "=== kernel log ==="; sudo journalctl -k --no-pager | tail -n 50 || true
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v7
@@ -778,14 +776,11 @@ jobs:
         env:
           PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} --test-group-random-seed=42
         run: |
-          # -n $(nproc) rather than $(nproc)+1: one fewer parallel node-cluster
-          # keeps ASan peak RSS under the 16GB hosted-runner ceiling.
-          sg wireshark "uv run eatmydata pytest tests/ -n $(nproc) ${PYTEST_OPTS} --test-group=${{ matrix.GROUP }} --test-group-count=${{ matrix.GROUP_COUNT }}"
-      - name: Report OOM kills
-        if: failure()
-        run: |
-          echo "=== dmesg (OOM) ==="; sudo dmesg | grep -i -E 'oom|out of memory|killed process' || echo "no OOM lines in dmesg"
-          echo "=== kernel log ==="; sudo journalctl -k --no-pager | tail -n 50 || true
+          .github/scripts/mem-monitor.sh &
+          mem_pid=$!
+          trap 'kill $mem_pid 2>/dev/null' EXIT
+          # loadgroup: tests marked heavy run one at a time on a single worker.
+          sg wireshark "uv run eatmydata pytest tests/ -n $(nproc) --dist loadgroup ${PYTEST_OPTS} --test-group=${{ matrix.GROUP }} --test-group-count=${{ matrix.GROUP_COUNT }}"
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v7

contrib/pyln-testing/pyln/testing/utils.py

@@ -81,6 +81,7 @@ def env(name, default=None):
 
 
 VALGRIND = env("VALGRIND") == "1"
+ASAN = env("ASAN", "0") == "1"
 TEST_NETWORK = env("TEST_NETWORK", 'regtest')
 TEST_DEBUG = env("TEST_DEBUG", "0") == "1"
 

tests/test_askrene.py

@@ -2107,6 +2107,7 @@ def test_unreserve_all(node_factory):
     assert l1.rpc.askrene_listreservations() == {"reservations": []}
 
 
+@pytest.mark.xdist_group("heavy")
 def test_reservations_leak_under_load(node_factory, executor):
     """Stress-test reservation cleanup: concurrent payments over shared channels
     must leave zero stale reservations after all payments settle."""

tests/test_closing.py

@@ -3730,6 +3730,7 @@ def test_invalid_upfront_shutdown_script(node_factory, bitcoind, executor):
 
 
 @pytest.mark.slow_test
+@pytest.mark.xdist_group("heavy")
 def test_segwit_shutdown_script(node_factory, bitcoind, executor):
     """
 Try a range of future segwit versions as shutdown scripts.  We create many nodes, so this is quite slow under valgrind

tests/test_connection.py

@@ -15,7 +15,7 @@
     mine_funding_to_announce, first_scid,
     CHANNEL_SIZE
 )
-from pyln.testing.utils import VALGRIND, EXPERIMENTAL_DUAL_FUND, FUNDAMOUNT, RUST, SLOW_MACHINE
+from pyln.testing.utils import VALGRIND, ASAN, EXPERIMENTAL_DUAL_FUND, FUNDAMOUNT, RUST, SLOW_MACHINE
 
 import coincurve
 import hashlib
@@ -1479,6 +1479,7 @@ def test_funding_v2_corners(node_factory, bitcoind):
 
 @pytest.mark.slow_test
 @pytest.mark.openchannel('v1')
+@pytest.mark.xdist_group("heavy")
 def test_funding_cancel_race(node_factory, bitcoind, executor):
     l1 = node_factory.get_node()
 
@@ -1490,6 +1491,8 @@ def test_funding_cancel_race(node_factory, bitcoind, executor):
 
     if VALGRIND:
         num = 5
+    elif ASAN:
+        num = 10
     elif SLOW_MACHINE:
         num = 20
     else:
@@ -3646,6 +3649,7 @@ def test_feerate_stress(node_factory, executor):
 
 
 @pytest.mark.slow_test
+@pytest.mark.xdist_group("heavy")
 def test_pay_disconnect_stress(node_factory, executor):
     """Expose race in htlc restoration in channeld: 50% chance of failure"""
     if VALGRIND:
@@ -5521,11 +5525,15 @@ def reestablished():
 
 
 @pytest.mark.slow_test
+@pytest.mark.xdist_group("heavy")
 def test_parallel_opens(node_factory, bitcoind, executor):
     """On macOS a socket closed by its sender while in flight over SCM_RIGHTS
     could arrive unable to read, losing a subdaemon its hsmd or peer
     connection under load (#5808)."""
-    pairs, opens = 6, 50
+    if ASAN:
+        pairs, opens = 2, 10
+    else:
+        pairs, opens = 6, 50
     nodes = node_factory.get_nodes(2 * pairs)
     openers, peers = nodes[:pairs], nodes[pairs:]
 

tests/test_xpay.py

@@ -1152,6 +1152,7 @@ def test_xpay_shadow_cltv(node_factory, bitcoind):
     assert l2.daemon.is_in_log(f"Adding HTLC .* amount=10000msat cltv={expected_cltv} gave CHANNEL_ERR_ADD_OK")
 
 
+@pytest.mark.xdist_group("heavy")
 def test_xpay_limited_max_accepted_htlcs(node_factory):
     """xpay should try to reduce flows to 6 if there is an unannounced channel, and only try more if that fails"""
     CHANNEL_SIZE_SATS = 10**6