wip: refactor/update STAN

.github/workflows/stan-validation.yml

@@ -0,0 +1,115 @@
+name: "Validate the Stan model"
+
+on:
+  # Not on pull_request: the grid is ~1400 fits and takes about ten minutes,
+  # which is too slow to sit in front of every change. The fast checks that do
+  # run per pull request are the "stan" marked tests in the Run Tests workflow,
+  # plus the two tests there that hold the *recorded* results in
+  # pymare/tests/data/stan_validation.json to these same thresholds. This
+  # workflow is what re-measures them.
+  push:
+    branches:
+      - "master"
+    paths:
+      - "pymare/estimators/stan/**"
+      - "pymare/estimators/estimators.py"
+      - "validation/stan/**"
+      - ".github/workflows/stan-validation.yml"
+  schedule:
+    # Monthly, to catch the model rotting against a new CmdStan or ArviZ rather
+    # than against a change to PyMARE.
+    - cron: "0 0 1 * *"
+  workflow_dispatch:
+    inputs:
+      replications:
+        description: "Replications per design cell (at least 100)"
+        required: false
+        default: "100"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: stan-validation-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CMDSTAN_VERSION: "2.36.0"
+  REPLICATIONS: ${{ github.event.inputs.replications || '100' }}
+
+jobs:
+  validate:
+    name: Bias and credible-interval coverage
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+      - name: "Set up python"
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: "Install PyMARE with the stan extra"
+        run: |
+          python -m pip install --progress-bar off --upgrade pip setuptools wheel
+          python -m pip install -e .[tests,stan]
+      - name: "Cache CmdStan"
+        id: cache_cmdstan
+        uses: actions/cache@v4
+        with:
+          path: ~/.cmdstan
+          key: cmdstan-${{ runner.os }}-${{ runner.arch }}-${{ env.CMDSTAN_VERSION }}
+      - name: "Install CmdStan"
+        if: steps.cache_cmdstan.outputs.cache-hit != 'true'
+        run: python -m cmdstanpy.install_cmdstan --version "${CMDSTAN_VERSION}" --cores 2
+
+      # Regenerates pymare/tests/data/stan_validation.json and fails if any cell
+      # misses STAN_VALIDATION_THRESHOLDS. The numbers are stochastic, so unlike
+      # the robumeta reference this cannot require the file to be unchanged --
+      # a correct model produces slightly different numbers every run. The
+      # thresholds are the claim; the file is the record of it.
+      - name: "Re-measure the model and enforce the thresholds"
+        id: measure
+        run: |
+          python validation/stan/simulate.py \
+              --replications "${REPLICATIONS}" \
+              --jobs 2 \
+              --check \
+              | tee measured.txt
+
+      - name: "Report what was measured"
+        if: always()
+        run: |
+          {
+              echo "## Stan model validation"
+              echo
+              echo "\`${REPLICATIONS}\` replications per design cell, CmdStan \`${CMDSTAN_VERSION}\`."
+              echo
+              echo '```'
+              tail -n 20 measured.txt 2>/dev/null || echo "the run produced no output"
+              echo '```'
+              echo
+              if git diff --quiet -- pymare/tests/data/stan_validation.json; then
+                  echo "The recorded results did not move."
+              else
+                  echo "<details><summary>Change against the recorded results</summary>"
+                  echo
+                  echo '```diff'
+                  git diff -- pymare/tests/data/stan_validation.json
+                  echo '```'
+                  echo
+                  echo "</details>"
+                  echo
+                  echo "These numbers are stochastic, so movement is expected. What matters"
+                  echo "is that every cell still clears the thresholds, which the step above"
+                  echo "enforces. Commit the regenerated file to refresh the record."
+              fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: "Upload the regenerated results"
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: stan-validation
+          path: pymare/tests/data/stan_validation.json

.github/workflows/testing.yml

@@ -30,10 +30,17 @@ env:
   # One place to change how the suite is invoked. --cov-append lets each job
   # write a coverage file that upload_to_codecov merges.
   PYTEST_COMMON_ARGS: --cov-append --cov-report=xml --cov=pymare
-  # Everything except the Stan estimator, whose model has to be compiled and so
-  # costs minutes rather than seconds. It gets the job below to itself.
+  # Everything except the tests that sample, whose Stan model has to be compiled
+  # and so costs minutes rather than seconds. They get the job below to
+  # themselves. The tests that only check how PyMARE's inputs are translated
+  # into Stan's data block are deliberately unmarked, so they run here on every
+  # platform without needing CmdStan.
   PYTEST_UNIT_MARKERS: not stan
   PYTEST_STAN_MARKERS: stan
+  # Pinned rather than "latest": an unpinned version makes the cache key move on
+  # its own and makes a red run ambiguous between a PyMARE change and a Stan
+  # release. Same reasoning as the pinned R image in validation/robumeta.
+  CMDSTAN_VERSION: "2.36.0"
 
 jobs:
   # Determine if tests should be run based on commit message.
@@ -86,10 +93,18 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: "Display Python version"
         run: python -c "import sys; print(sys.version)"
+      # The stan extra, not just tests: it brings in ArviZ, and the results
+      # container is ArviZ-only code that nothing here could otherwise execute.
+      # Without it the whole of BayesianMetaRegressionResults is reachable only
+      # from the single Stan job below, so its coverage -- and the ArviZ 0.x
+      # versus 1.x handling in particular -- would rest on one job on one
+      # Python. cmdstanpy comes along too but stays idle: it is pure Python,
+      # CmdStan is not installed here, and the tests that sample are excluded by
+      # the marker filter regardless.
       - name: "Install PyMARE"
         run: |
           python -m pip install --progress-bar off --upgrade pip setuptools wheel
-          python -m pip install -e .[tests]
+          python -m pip install -e .[tests,stan]
       - name: "Run tests"
         run: python -m pytest -m "${PYTEST_UNIT_MARKERS}" ${PYTEST_COMMON_ARGS}
       - name: "Upload coverage"
@@ -104,6 +119,13 @@ jobs:
     needs: check_skip
     if: ${{ needs.check_skip.outputs.skip == 'false' }}
     runs-on: ubuntu-latest
+    env:
+      # Read by the pytest_collection_modifyitems hook in
+      # pymare/tests/conftest.py, which turns a missing or broken CmdStan into a
+      # failure rather than a skip. The previous gate probed for a module name
+      # that never existed, so this job passed for years while running none of
+      # the tests it exists to run, and nothing in a green log said so.
+      PYMARE_REQUIRE_CMDSTAN: "1"
     defaults:
       run:
         shell: bash
@@ -117,6 +139,41 @@ jobs:
         run: |
           python -m pip install --progress-bar off --upgrade pip setuptools wheel
           python -m pip install -e .[tests,stan]
+      - name: "Cache CmdStan"
+        id: cache_cmdstan
+        uses: actions/cache@v4
+        with:
+          path: ~/.cmdstan
+          # runner.arch is part of the key because CmdStan is a native build:
+          # restoring an x86_64 tree onto an arm64 runner fails confusingly.
+          key: cmdstan-${{ runner.os }}-${{ runner.arch }}-${{ env.CMDSTAN_VERSION }}
+      - name: "Install CmdStan"
+        if: steps.cache_cmdstan.outputs.cache-hit != 'true'
+        run: python -m cmdstanpy.install_cmdstan --version "${CMDSTAN_VERSION}" --cores 2
+      - name: "Cache the compiled model"
+        uses: actions/cache@v4
+        with:
+          # CmdStanPy compiles beside the .stan source, not under ~/.cmdstan, so
+          # this needs a cache entry of its own. The key covers all four inputs
+          # the executable depends on.
+          path: pymare/estimators/stan
+          key: >-
+            stanexe-${{ runner.os }}-${{ runner.arch }}-${{ env.CMDSTAN_VERSION }}-${{
+            hashFiles('pymare/estimators/stan/*.stan') }}
+      - name: "Check the Stan program with pedantic mode"
+        # Informational. Pedantic mode flags a parameter with no prior, which is
+        # what the old model's tau2 was, but it also flags beta's deliberately
+        # flat prior, so this reports rather than gates.
+        #
+        # Invoke stanc directly rather than cmdstanpy.compile_stan_file: that
+        # skips its work when an up-to-date executable exists, so with the model
+        # cache restored above it would silently check nothing on most runs.
+        # stanc only translates Stan to C++, which takes a fifth of a second and
+        # leaves the cached executable alone, so it always runs.
+        continue-on-error: true
+        run: |
+          STANC="$(python -c "import cmdstanpy, os; print(os.path.join(cmdstanpy.cmdstan_path(), 'bin', 'stanc'))")"
+          "${STANC}" --warn-pedantic --o=/dev/null pymare/estimators/stan/meta_regression.stan
       - name: "Run tests"
         run: python -m pytest -m "${PYTEST_STAN_MARKERS}" ${PYTEST_COMMON_ARGS}
       - name: "Upload coverage"

.gitignore

@@ -119,3 +119,10 @@ fmriprep.sqlite
 .vscode
 # asv benchmark environments and results
 .asv/
+
+# CmdStan build artifacts. CmdStanModel compiles the model in place, dropping
+# the executable and its intermediates next to the .stan source.
+pymare/estimators/stan/meta_regression
+pymare/estimators/stan/meta_regression.exe
+pymare/estimators/stan/*.hpp
+pymare/estimators/stan/*.o

CONTRIBUTING.md

@@ -98,15 +98,54 @@ environment does not have:
 
 | Target | What it runs | Needs |
 | --- | --- | --- |
-| `make unittest` | everything except the Stan tests | nothing extra |
-| `make test_stan` | the Stan estimator tests | `pip install -e .[stan]` |
+| `make unittest` | everything except the Stan sampling tests | nothing extra |
+| `make test_stan` | the Stan sampling tests | `pip install -e .[stan]`, then `make install_cmdstan` |
 | `make test_robumeta` | the robumeta alignment tests | nothing extra |
 | `make check_robumeta_alignment` | regenerates the robumeta reference values | Docker |
+| `make validate_stan` | re-measures the Stan model's bias and coverage (~10 min) | the same as `test_stan` |
 | `make lint` | flake8 over `pymare` and `benchmarks` | nothing extra |
 
 Each of these has a GitHub Actions job behind it, so a target that passes
 locally is the same check that runs on your pull request.
 
+`make test_stan` needs two installation steps rather than one: the `stan` extra
+brings in cmdstanpy, but CmdStan itself is a C++ build rather than a Python
+package, so `make install_cmdstan` fetches and builds it. That takes several
+minutes the first time and nothing thereafter.
+
+**Those tests skip locally when CmdStan is missing, but fail in CI.** The
+asymmetry is deliberate. A contributor without CmdStan should not see red, but a
+skip is indistinguishable from a pass in a CI log, and that is exactly how the
+Stan job passed for years while running none of the tests it existed to run --
+its gate probed for a module name that PyStan 3 never provided. The Stan job now
+sets `PYMARE_REQUIRE_CMDSTAN=1`, and the `pytest_collection_modifyitems` hook in
+`pymare/tests/conftest.py` fails the run outright, at collection, wherever that
+is set and CmdStan is missing.
+
+Only the tests that actually sample are marked `stan`. The ones that check how
+PyMARE's inputs are translated into Stan's data block need neither cmdstanpy nor
+CmdStan, so they are unmarked and run in the ordinary unit job on every
+platform.
+
+The model's accuracy is measured separately, by `validation/stan/simulate.py`,
+which reports bias and credible-interval coverage across a grid of designs and
+records them in `pymare/tests/data/stan_validation.json`. It follows the same
+three-layer arrangement as the robumeta alignment:
+
+1. `make validate_stan` regenerates that file and fails if any design cell
+   misses `pymare.tests.utils.STAN_VALIDATION_THRESHOLDS`.
+2. Two tests in `test_stan_estimators.py` hold the *recorded* file to those same
+   thresholds and to the expected list of design cells. They read the file
+   rather than re-measuring, so they cost nothing and run everywhere — which is
+   what stops the pin from quietly going stale.
+3. The `Validate the Stan model` workflow re-measures on a schedule, on pushes
+   to master that touch the model, and on demand.
+
+The grid takes about ten minutes, which is why it is not part of `test_stan` and
+not run per pull request. Unlike the robumeta reference, these numbers are
+stochastic, so the pin cannot be enforced by requiring the file not to move; the
+thresholds are the claim, and the file is the record of it.
+
 ### Alignment with robumeta
 
 `pymare/tests/test_robumeta_alignment.py` pins PyMARE's correlated-effects model

MANIFEST.in

@@ -1,2 +1,3 @@
 include versioneer.py
 include pymare/_version.py
+recursive-include pymare *.stan

Makefile

@@ -1,4 +1,5 @@
-.PHONY: all_tests benchmark check_robumeta_alignment help lint test_robumeta test_stan unittest
+.PHONY: all_tests benchmark check_robumeta_alignment help install_cmdstan lint test_robumeta
+.PHONY: test_stan unittest validate_stan
 
 # --cov-append matches what CI does, so a local run of two targets in a row
 # reports their combined coverage rather than only the last one's.
@@ -9,10 +10,12 @@ all_tests: lint unittest test_stan test_robumeta
 help:
 	@echo "Please use 'make <target>' where <target> is one of:"
 	@echo "  lint                       to run flake8 over pymare and the benchmarks"
-	@echo "  unittest                   to run every test except the Stan ones"
-	@echo "  test_stan                  to run the Stan estimator tests (needs the stan extra)"
+	@echo "  unittest                   to run every test except the Stan sampling ones"
+	@echo "  install_cmdstan            to install the CmdStan that test_stan needs"
+	@echo "  test_stan                  to run the Stan sampling tests (needs the stan extra and CmdStan)"
 	@echo "  test_robumeta              to run the robumeta alignment tests"
 	@echo "  check_robumeta_alignment   to regenerate the robumeta reference values (needs Docker)"
+	@echo "  validate_stan              to re-measure the Stan model's bias and coverage (~10 min)"
 	@echo "  benchmark                  to run the asv suite once in the current environment"
 	@echo "  all_tests                  to run lint and every test target"
 
@@ -22,12 +25,26 @@ lint:
 unittest:
 	@python -m pytest -m "not stan" $(PYTEST_COV)
 
+# CmdStan is a C++ build rather than a Python package, so `pip install -e .[stan]`
+# gets cmdstanpy but not the CmdStan it drives. This is the missing second step.
+install_cmdstan:
+	@python -m cmdstanpy.install_cmdstan
+
+# Skips rather than fails when CmdStan is absent. CI sets PYMARE_REQUIRE_CMDSTAN=1
+# so that a job which is supposed to have it goes red instead of quietly empty.
 test_stan:
 	@python -m pytest -m "stan" $(PYTEST_COV)
 
 test_robumeta:
 	@python -m pytest -m "robumeta" $(PYTEST_COV)
 
+# What the "Validate the Stan model" workflow runs. Regenerates
+# pymare/tests/data/stan_validation.json and fails if any design cell misses the
+# thresholds in pymare.tests.utils.STAN_VALIDATION_THRESHOLDS. Slow -- about ten
+# minutes -- which is why it is not part of unittest or test_stan.
+validate_stan:
+	@python validation/stan/simulate.py --check
+
 # What the "Check robumeta alignment" workflow runs. Needs Docker, because the
 # reference values come from R.
 check_robumeta_alignment:

docs/installation.rst

@@ -15,5 +15,29 @@ If you want to use the most up-to-date version, you can install from the ``maste
 
     pip install git+https://github.com/neurostuff/PyMARE.git
 
-PyMARE requires Python >=3.8 and a number of packages.
+PyMARE requires Python >=3.9 and a number of packages.
 For a complete list, please see ``setup.cfg``.
+
+Bayesian estimation with Stan
+-----------------------------
+
+:class:`~pymare.estimators.StanMetaRegression` is optional, and needs two
+installation steps rather than one:
+
+.. code-block:: bash
+
+    pip install pymare[stan]
+    python -m cmdstanpy.install_cmdstan
+
+The first installs CmdStanPy. The second fetches and builds CmdStan itself,
+which is a C++ program rather than a Python package and so needs a C++ toolchain
+(``g++`` and ``make`` on Linux, the Command Line Tools on macOS, RTools on
+Windows). It takes several minutes, once per machine.
+
+The Stan model is compiled the first time the estimator is fitted, which takes
+roughly another minute. The compiled model is cached alongside the installed
+package, so later fits and later processes reuse it. If PyMARE is installed
+somewhere unwritable, the model is compiled into ``~/.pymare/stan`` instead and a
+warning says so.
+
+Every other estimator in PyMARE is pure Python and needs none of this.

examples/02_meta-analysis/plot_meta-analysis_walkthrough.py

@@ -232,8 +232,12 @@
 ###############################################################################
 # What about the Stan estimator?
 # `````````````````````````````````````````````````````````````````````````````
-# We're going to skip this one here because of how computationally intensive it
-# is.
+# We're going to skip this one here. It needs two things the documentation build
+# does not have: CmdStan, which is a C++ build rather than a Python package, and
+# a C++ toolchain to compile the model with. The compilation is a one-time cost
+# per installation rather than a per-fit one, so it is much less of an obstacle
+# in your own environment than it is here. Install it with
+# ``pip install pymare[stan]`` followed by ``python -m cmdstanpy.install_cmdstan``.
 
 ###############################################################################
 # Let's check out our results!