Run the conformance tests on labeled PRs, weekly, and for releases

.github/workflows/ci.yml

@@ -4,6 +4,9 @@ on:
     tags:
       - "*"
   pull_request:
+# Jobs that write with GITHUB_TOKEN ask for it themselves
+permissions:
+  contents: read
 jobs:
   # Check formatting and run clippy lints
   linting:
@@ -359,75 +362,20 @@ jobs:
         with:
           name: java-bindings-pom
           path: ffi/bindings/java/dnp3/pom.xml
-  # Run the conformance tests, for each revision of the test procedures at the subset level it tests
-  conformance-run:
-    name: conformance (${{ matrix.revision }}, level ${{ matrix.level }})
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - revision: "2.7"
-            level: 2
-          - revision: "3.1r1"
-            level: 3
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - name: Build JNI
-        run: cargo build --release -p dnp3-ffi-java
-      - name: Build Java bindings
-        run: cargo run --release --bin dnp3-bindings -- --java
-      - name: Install Java bindings
-        shell: bash
-        run: (cd ffi/bindings/java/dnp3 && sudo mvn install)
-      - name: Capture packets during the conformance tests
-        working-directory: conformance
-        # capturing needs root, which the tests run as; local runs leave it off
-        run: echo "tcp.capture.adapter = /127.0.0.1" >> device.ini
-      - name: Select the subset level
-        working-directory: conformance
-        # device.ini is for 3.1r1 at level 3; a later value overrides it
-        run: echo "outstation.subset.level = ${{ matrix.level }}" >> device.ini
-      - name: Run the conformance tests
-        working-directory: conformance
-        run: sudo mvn --batch-mode scala:run -Dconformance.revision=${{ matrix.revision }}
-      - name: Check the packet captures
-        # dnp3c keeps each revision's results, for each subset level, in a folder of its own
-        working-directory: conformance/results/${{ matrix.revision }}/L${{ matrix.level }}
-        # every procedure whose log shows a transmitted frame ("--P->") must have captured packets;
-        # a pcap of only its 24-octet header captured none
-        run: |
-          sent=0
-          empty=0
-          for dir in */; do
-            grep -qE -- '--P-(>|>)' "$dir/log.html" 2>/dev/null || continue
-            sent=$((sent + 1))
-            if [ ! -f "$dir/dump.pcap" ] || [ "$(stat -c %s "$dir/dump.pcap")" -le 24 ]; then
-              echo "No packets captured for ${dir%/}"
-              empty=$((empty + 1))
-            fi
-          done
-          echo "$sent procedures sent frames; $empty captured no packets"
-          [ "$sent" -gt 0 ] && [ "$empty" -eq 0 ]
-      - name: Upload conformance test results
-        if: always()
-        uses: actions/upload-artifact@v7
-        with:
-          name: conformance-results-${{ matrix.revision }}
-          path: conformance/results
-  # The check branch protection requires: passes only if every revision's run passed
+  # Run the conformance tests for releases; PRs labeled `conformance` run them through conformance.yml itself
   conformance:
-    needs: [conformance-run]
+    if: startsWith(github.ref, 'refs/tags/')
+    uses: ./.github/workflows/conformance.yml
+  # The check branch protection requires: passes only if every job a PR runs passed
+  ci-ok:
+    needs: [linting, msrv, test, features, packaging]
     if: always()
     runs-on: ubuntu-latest
     steps:
-      - name: Check the conformance runs
+      - name: Check the CI jobs
         run: |
-          echo "conformance runs: ${{ needs.conformance-run.result }}"
-          [ "${{ needs.conformance-run.result }}" = "success" ]
+          echo '${{ toJSON(needs) }}'
+          [ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }}" = "false" ]
   # Quality gate to ensure all tests pass before releases
   quality-gate:
     needs: [linting, msrv, test, features, conformance]
@@ -629,6 +577,9 @@ jobs:
     needs: [release-docs, release-maven, release-nuget, release-crates-io]
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/')
+    # creates the draft release and uploads its assets
+    permissions:
+      contents: write
     steps:
       - name: Download all release artifacts
         uses: actions/download-artifact@v8

.github/workflows/conformance.yml

@@ -0,0 +1,78 @@
+name: Conformance
+on:
+  # Runs on PRs only when they carry the `conformance` label (see CLAUDE.md)
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+  # Best effort: GitHub may delay or drop scheduled runs
+  schedule:
+    - cron: "17 6 * * 1"
+  workflow_dispatch:
+  # ci.yml calls this for release tags, so quality-gate waits on it and the release jobs publish its results
+  workflow_call:
+permissions:
+  contents: read
+jobs:
+  # Run the conformance tests, for each revision of the test procedures at the subset level it tests
+  conformance-run:
+    name: conformance (${{ matrix.revision }}, level ${{ matrix.level }})
+    # on PRs, run when labeled `conformance`, and not again when some other label is added
+    if: >-
+      github.event_name != 'pull_request' ||
+      (contains(github.event.pull_request.labels.*.name, 'conformance') &&
+       (github.event.action != 'labeled' || github.event.label.name == 'conformance'))
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - revision: "2.7"
+            level: 2
+          - revision: "3.1r1"
+            level: 3
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+      - name: Build JNI
+        run: cargo build --release -p dnp3-ffi-java
+      - name: Build Java bindings
+        run: cargo run --release --bin dnp3-bindings -- --java
+      - name: Install Java bindings
+        shell: bash
+        run: (cd ffi/bindings/java/dnp3 && sudo mvn install)
+      - name: Capture packets during the conformance tests
+        working-directory: conformance
+        # capturing needs root, which the tests run as; local runs leave it off
+        run: echo "tcp.capture.adapter = /127.0.0.1" >> device.ini
+      - name: Select the subset level
+        working-directory: conformance
+        # device.ini is for 3.1r1 at level 3; a later value overrides it
+        run: echo "outstation.subset.level = ${{ matrix.level }}" >> device.ini
+      - name: Run the conformance tests
+        working-directory: conformance
+        run: sudo mvn --batch-mode scala:run -Dconformance.revision=${{ matrix.revision }}
+      - name: Check the packet captures
+        # dnp3c keeps each revision's results, for each subset level, in a folder of its own
+        working-directory: conformance/results/${{ matrix.revision }}/L${{ matrix.level }}
+        # every procedure whose log shows a transmitted frame ("--P->") must have captured packets;
+        # a pcap of only its 24-octet header captured none
+        run: |
+          sent=0
+          empty=0
+          for dir in */; do
+            grep -qE -- '--P-(>|>)' "$dir/log.html" 2>/dev/null || continue
+            sent=$((sent + 1))
+            if [ ! -f "$dir/dump.pcap" ] || [ "$(stat -c %s "$dir/dump.pcap")" -le 24 ]; then
+              echo "No packets captured for ${dir%/}"
+              empty=$((empty + 1))
+            fi
+          done
+          echo "$sent procedures sent frames; $empty captured no packets"
+          [ "$sent" -gt 0 ] && [ "$empty" -eq 0 ]
+      - name: Upload conformance test results
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: conformance-results-${{ matrix.revision }}
+          path: conformance/results

CLAUDE.md

@@ -14,6 +14,11 @@
 - Check lints: `cargo clippy`
 - Run with logging: `RUST_LOG=debug cargo run --example outstation`
 
+## Conformance Tests
+- `.github/workflows/conformance.yml` runs the DNP3 IED certification procedures against the outstation with dnp3c (about 12 minutes). It runs on release tags (gating publishing), weekly on `main`, on demand (`gh workflow run conformance.yml --ref <branch>`), and on PRs labeled `conformance`
+- Label a PR `conformance` (`gh pr create --label conformance` or `gh pr edit <N> --add-label conformance`) when it changes what the outstation sends or accepts: the outstation, the application, transport, or link layers, or the Java FFI the tests drive. Don't label master-only, docs, CI, or other-binding changes
+- On PRs the result is advisory, not a required check, so look at it before merging
+
 ## Code Style Guidelines
 - **No unsafe code**: Unsafe code is forbidden (`unsafe_code = "forbid"`)
 - **Documentation**: All public items must be documented (`missing_docs = "deny"`)
@@ -22,4 +27,11 @@
 - **Naming**: Follow Rust naming conventions (snake_case for functions/variables, CamelCase for types)
 - **Imports**: Organize imports by standard lib, external crates, then internal modules
 - **Features**: Support compile-time features (tls, serial, serialization)
-- **Minimum Rust Version**: 1.85+ (the workspace `rust-version`, checked by the `msrv` CI job)
+- **Minimum Rust Version**: 1.85+ (the workspace `rust-version`, checked by the `msrv` CI job)
+
+## Reference Documents
+- The private repository `stepfunc/dnp3-reference`, checked out next to this one at `../dnp3-reference`, holds the DNP3 source documents: IEEE 1815-2012 and related standards, the DNP Users Group technical bulletins, and the IED certification test procedures. Some documents (1815-2012 and the 2.7 and 3.1r1 test procedures) have searchable Markdown extractions; the bulletins are PDFs with an index of what each one changes. It also records our interpretation decisions
+- Use it to answer questions about what the spec or test procedures require. Read its `CLAUDE.md` first and follow its guidance on reading, citing, and interpreting the documents. Before relying on a 1815-2012 requirement, check its bulletin index for amendments
+- Its rules for maintaining that repository (such as committing straight to `main`) apply only there, not here
+- If it isn't checked out, ask the user for it rather than working from memory of the spec
+- **Keep it private**: the documents are copyrighted. This repository is public, so never copy or upload their PDFs, extractions, screenshots, text, tables, or figures into code, comments, docs, commit messages, issues, PRs, or attachments. Cite by document and clause or test number instead (e.g. "IEEE 1815-2012 4.5.10", "TB2016-001")

RELEASE.md

@@ -23,17 +23,28 @@ changed, refresh `allowed.json` so the packaging license check passes, and
 verify any dependency/security claims in the changelog against the actual
 `Cargo.lock` versions.
 
-Open the PR and merge it to `main`.
+Open the PR with the `conformance` label (`gh pr create --label conformance`).
+The conformance tests don't run on unlabeled PRs, and a failure on the tag
+blocks every publisher, so merge the PR to `main` only once both conformance
+jobs are green on its latest revision, after its final update with `main`.
 
 ## 2. Tag on `main` and push
 
-CI only publishes from tags, and by convention tags live on `main`:
+CI only publishes from tags, and by convention tags live on `main`. Tag the
+commit the `prepare` PR's merge produced, by its SHA, since `main` may have
+moved since:
 
 ```bash
 git checkout main && git pull
-git tag X.Y.Z && git push origin X.Y.Z
+SHA=$(gh pr view <prepare PR number> --json mergeCommit --jq .mergeCommit.oid)
+git tag X.Y.Z "$SHA" && git push origin X.Y.Z
 ```
 
+To tag a later commit instead, run the conformance tests on it first
+(`gh workflow run conformance.yml --ref main`), wait for the run to pass, and
+tag the SHA it tested (its `headSha` in `gh run list --workflow conformance.yml
+--json headSha,conclusion`).
+
 crates.io publishing is **irreversible** — a version can never be re-published.
 Confirm the version before pushing. (`-RCn` publishes as a pre-release, which
 cargo won't select by default.)