fix(ci): verify CN exit before collecting BVT coverage

.github/workflows/check-action-file.yaml

@@ -12,6 +12,7 @@ on:
       - 'actions/resolve-sca-license-scope/**'
       - 'actions/restore-sca-analysis-cache/**'
       - 'actions/seed-go-caches/**'
+      - 'actions/collect-bvt-coverage/**'
       - 'actions/upload-artifact-with-retry/action.yaml'
       - 'actions/upload-artifact-with-retry/state-machine.schema.json'
       - 'scripts/select_coverage_artifacts.py'
@@ -81,6 +82,8 @@ jobs:
           go vet \
             scripts/compact_coverage_profile.go \
             scripts/compact_coverage_profile_test.go
+      - name: Test BVT coverage collection
+        run: python3 -m unittest discover -s actions/collect-bvt-coverage -p 'test_*.py' -v
       - name: Test coverage artifact generation selection
         run: python3 scripts/test_select_coverage_artifacts.py -v
       - name: Test TKE merge subject identity contract
@@ -164,6 +167,7 @@ jobs:
             actions/resolve-sca-license-scope/action.yaml
             actions/restore-sca-analysis-cache/action.yaml
             actions/seed-go-caches/action.yaml
+            actions/collect-bvt-coverage/action.yaml
             actions/upload-artifact-with-retry/action.yaml
       - name: Validate artifact retry state machine
         uses: dsanders11/json-schema-validate-action@eddf079f55830cc9a916a3c512ba9086240d2fea # v2.0.0

.github/workflows/e2e-compose-parallel.yaml

@@ -258,6 +258,8 @@ jobs:
           # Handle various docker-compose volume formats: /test, /test:options, /test followed by space
           sed -i 's|:/test\b|:'"$GITHUB_WORKSPACE"'/test|g' ./etc/launch-tae-compose/compose.yaml
 
+          # A rerun must never inherit counters from an earlier container generation.
+          sudo rm -rf -- "$GITHUB_WORKSPACE/coverage"
           mkdir -p "$GITHUB_WORKSPACE/coverage"
 
           # Assemble the runtime image from the shared -cover build when the
@@ -479,33 +481,13 @@ jobs:
         run: |
           ./optools/check_log_count.sh 1000 60 # {count threshold} {metric collected interval}
 
-      - name: shutdown containers
+      - name: Collect BVT coverage after verified CN shutdown
         if: ${{ always() }}
-        run: |
-          docker ps
-          docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn down --remove-orphans
-          docker volume rm launch-tae-compose_minio_storage || true
-
-      - name: Generate Compose BVT coverage profile
-        if: ${{ always() && !cancelled() }}
-        run: |
-          set -uo pipefail
-          coverage_dir="$GITHUB_WORKSPACE/coverage"
-          coverage_profile="$RUNNER_TEMP/bvt-compose.out"
-          compose_file="$GITHUB_WORKSPACE/etc/launch-tae-compose/compose.yaml"
-          if ! grep -Eq 'GOCOVERDIR=/coverage' "${compose_file}" \
-            || ! grep -Eq 'coverage:/coverage' "${compose_file}"; then
-            echo '::warning::Compose BVT coverage is unsupported by this legacy branch; keeping the successful BVT result'
-            exit 0
-          fi
-          if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then
-            test -s "${coverage_profile}"
-          elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then
-            echo '::error::failed to generate Compose BVT coverage after a successful BVT'
-            exit 1
-          else
-            echo '::warning::Compose BVT coverage unavailable because BVT did not finish successfully'
-          fi
+        timeout-minutes: 6
+        uses: matrixorigin/CI/actions/collect-bvt-coverage@main
+        with:
+          coverage-dir: ${{ github.workspace }}/coverage
+          output: ${{ runner.temp }}/bvt-compose.out
 
       - name: Upload Compose BVT coverage
         if: ${{ always() && !cancelled() }}

.github/workflows/e2e-standalone-parallel.yaml

@@ -312,6 +312,8 @@ jobs:
           cat ./etc/launch-tae-compose/config/cn-1.toml
           cat ./etc/launch-tae-compose/config/tn.toml
           sed -i 's|:/test\b|:'"$GITHUB_WORKSPACE"'/test|g' ./etc/launch-tae-compose/compose.yaml
+          # A rerun must never inherit counters from an earlier container generation.
+          sudo rm -rf -- "$GITHUB_WORKSPACE/coverage"
           mkdir -p "$GITHUB_WORKSPACE/coverage" "$GITHUB_WORKSPACE/docker-compose-log"
 
           # Assemble the runtime image from the shared -cover build when the
@@ -519,33 +521,13 @@ jobs:
         run: |
           ./optools/check_log_count.sh 1000 60 # {count threshold} {metric collected interval}
 
-      - name: shutdown containers
+      - name: Collect BVT coverage after verified CN shutdown
         if: ${{ always() }}
-        run: |
-          docker ps
-          docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn down --remove-orphans
-          docker volume rm launch-tae-compose_minio_storage || true
-
-      - name: Generate Compose BVT coverage profile
-        if: ${{ always() && !cancelled() }}
-        run: |
-          set -uo pipefail
-          coverage_dir="$GITHUB_WORKSPACE/coverage"
-          coverage_profile="${RUNNER_TEMP}/bvt-pessimistic.out"
-          compose_file="$GITHUB_WORKSPACE/etc/launch-tae-compose/compose.yaml"
-          if ! grep -Eq 'GOCOVERDIR=/coverage' "${compose_file}" \
-            || ! grep -Eq 'coverage:/coverage' "${compose_file}"; then
-            echo '::warning::Compose BVT coverage is unsupported by this legacy branch; keeping the successful BVT result'
-            exit 0
-          fi
-          if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then
-            test -s "${coverage_profile}"
-          elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then
-            echo '::error::failed to generate Compose + Pessimistic BVT coverage after a successful BVT'
-            exit 1
-          else
-            echo '::warning::Compose + Pessimistic BVT coverage unavailable because BVT did not finish successfully'
-          fi
+        timeout-minutes: 6
+        uses: matrixorigin/CI/actions/collect-bvt-coverage@main
+        with:
+          coverage-dir: ${{ github.workspace }}/coverage
+          output: ${{ runner.temp }}/bvt-pessimistic.out
 
       - name: Upload Compose + Pessimistic BVT coverage
         if: ${{ always() && !cancelled() }}

actions/collect-bvt-coverage/action.yaml

@@ -0,0 +1,48 @@
+name: Collect BVT coverage
+description: Verify CN shutdown and SQL execution counters before publishing BVT coverage
+inputs:
+  compose-file:
+    description: MatrixOne Compose configuration
+    default: etc/launch-tae-compose/compose.yaml
+  coverage-dir:
+    description: Run-owned raw Go coverage directory
+    required: true
+  output:
+    description: Destination coverage profile
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Stop CNs and verify normal exit
+      id: stop
+      if: ${{ !cancelled() }}
+      shell: bash
+      env:
+        ACTION_PATH: ${{ github.action_path }}
+        COMPOSE_FILE: ${{ inputs.compose-file }}
+        COVERAGE_OUTPUT: ${{ inputs.output }}
+      run: |
+        set -euo pipefail
+        rm -f -- "${COVERAGE_OUTPUT}"
+        python3 "${ACTION_PATH}/collect.py" stop --compose-file "${COMPOSE_FILE}"
+    - name: Remove containers
+      id: cleanup
+      if: ${{ always() }}
+      shell: bash
+      env:
+        COMPOSE_FILE: ${{ inputs.compose-file }}
+      run: |
+        docker compose -f "${COMPOSE_FILE}" --profile launch-multi-cn down --remove-orphans
+        docker volume rm launch-tae-compose_minio_storage || true
+    - name: Generate and validate coverage profile
+      if: ${{ always() && !cancelled() && steps.stop.outcome == 'success' && steps.cleanup.outcome == 'success' }}
+      shell: bash
+      env:
+        ACTION_PATH: ${{ github.action_path }}
+        COMPOSE_FILE: ${{ inputs.compose-file }}
+        COVERAGE_DIR: ${{ inputs.coverage-dir }}
+        COVERAGE_OUTPUT: ${{ inputs.output }}
+      run: |
+        python3 "${ACTION_PATH}/collect.py" profile \
+          --compose-file "${COMPOSE_FILE}" --coverage-dir "${COVERAGE_DIR}" \
+          --output "${COVERAGE_OUTPUT}"

actions/collect-bvt-coverage/collect.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Preserve CN shutdown evidence before Compose removes the containers."""
+
+import argparse
+import json
+from pathlib import Path
+import re
+import subprocess
+import tempfile
+
+
+CN_SERVICES = ("cn-0", "cn-1")
+SQL_EXECUTOR = "/pkg/frontend/mysql_cmd_executor.go:"
+BLOCK = re.compile(r"\S+:\d+\.\d+,\d+\.\d+ \d+ (\d+)$")
+
+
+def run(args, timeout=30):
+    return subprocess.run(
+        args, check=True, text=True, stdout=subprocess.PIPE, timeout=timeout
+    ).stdout
+
+
+def supports_coverage(compose_file):
+    config = compose_file.read_text()
+    return "GOCOVERDIR=/coverage" in config and "coverage:/coverage" in config
+
+
+def stop_cns(compose_file):
+    if not supports_coverage(compose_file):
+        print("::warning::Skipping CN coverage checks for this legacy Compose configuration")
+        return
+    compose = ["docker", "compose", "-f", str(compose_file),
+               "--profile", "launch-multi-cn"]
+    containers = {}
+    for service in CN_SERVICES:
+        ids = run(compose + ["ps", "--all", "--quiet", service]).split()
+        if len(ids) != 1:
+            raise ValueError(f"expected one {service} container, found {len(ids)}")
+        containers[service] = ids[0]
+
+    # CN shutdown allows one minute for draining, plus shutdown profiles.
+    # Keep TN/log dependencies alive and allow more than Docker's default 10s.
+    # A hung shutdown still fails the exit-state check below.
+    print(run(compose + ["stop", "--timeout", "120", *CN_SERVICES], timeout=180), end="")
+    failures = []
+    for service, container in containers.items():
+        state = json.loads(run(["docker", "inspect", "--format", "{{json .State}}", container]))
+        print(f"{service} shutdown: {json.dumps(state, sort_keys=True)}", flush=True)
+        if (state.get("Status") != "exited" or state.get("ExitCode") != 0
+                or state.get("OOMKilled") is not False or state.get("Error")):
+            failures.append(service)
+    if failures:
+        raise ValueError(f"CN shutdown did not complete cleanly: {', '.join(failures)}; "
+                         "coverage may be incomplete")
+
+
+def validate_profile(profile):
+    sql_executed = False
+    with profile.open() as stream:
+        if stream.readline().strip() not in {"mode: set", "mode: count", "mode: atomic"}:
+            raise ValueError("invalid coverage profile mode")
+        for line in stream:
+            block = BLOCK.fullmatch(line.strip())
+            if block is None:
+                raise ValueError("invalid coverage profile block")
+            if SQL_EXECUTOR in line and int(block.group(1)) > 0:
+                sql_executed = True
+    if not sql_executed:
+        raise ValueError("BVT coverage has no SQL executor hits; CN counters are missing")
+
+
+def generate_profile(compose_file, coverage_dir, output):
+    output.unlink(missing_ok=True)
+    if not supports_coverage(compose_file):
+        print("::warning::BVT coverage is unsupported by this legacy Compose configuration")
+        return
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(dir=output.parent, prefix="bvt-coverage-") as directory:
+        temporary = Path(directory) / "profile.out"
+        run(["go", "tool", "covdata", "textfmt", f"-i={coverage_dir}", f"-o={temporary}"], timeout=180)
+        validate_profile(temporary)
+        temporary.replace(output)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("operation", choices=("stop", "profile"))
+    parser.add_argument("--compose-file", type=Path, required=True)
+    parser.add_argument("--coverage-dir", type=Path)
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+    if args.operation == "stop":
+        stop_cns(args.compose_file)
+    else:
+        if args.coverage_dir is None or args.output is None:
+            parser.error("profile requires --coverage-dir and --output")
+        generate_profile(args.compose_file, args.coverage_dir, args.output)
+
+
+if __name__ == "__main__":
+    main()