From 8ff80c53a52ae529cb69a651b70e8ab57a135c12 Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Fri, 14 Aug 2026 09:18:20 -0400 Subject: [PATCH 01/12] ENH: wire the JupyterLite build into the docs Enables jupyterlite_sphinx, stages the data the browser notebooks read, prepends the setup cell to each notebook copy, and leaves the launch badge off the pages that cannot run in the browser. --- .circleci/config.yml | 11 + .gitignore | 6 + doc/Makefile | 2 +- doc/conf.py | 478 +++++++++++++++++++++++- doc/documentation/datasets.rst | 11 + doc/jupyterlite_contents/.gitkeep | 0 doc/sphinxext/jupyterlite_cell_notes.py | 109 ++++++ 7 files changed, 615 insertions(+), 2 deletions(-) create mode 100644 doc/jupyterlite_contents/.gitkeep create mode 100644 doc/sphinxext/jupyterlite_cell_notes.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e9f05a4803..f9b18d3ee01 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -249,6 +249,17 @@ jobs: cp junit-results.xml doc/_build/test-results/test-doc/junit.xml; cp coverage.xml doc/_build/test-results/test-doc/coverage.xml; fi; + # Ensure the data the JupyterLite notebooks need is on disk so conf.py can + # copy the required subset for the build. lite_data fetches only the + # curated files the browser notebooks use (from the MNE-lite-data OSF + # project) instead of the full sample/kiloword/erp_core/mtrf/eegbci + # datasets, which slims the build. The curated files share hashes with the + # full datasets, so on a full build that already downloaded them nothing + # extra is fetched. + - run: + name: Ensure MNE data for JupyterLite + command: | + python -c "import mne; mne.datasets.lite_data.data_path(update_path=True)" # Build docs - run: name: make html diff --git a/.gitignore b/.gitignore index 21275b21c0b..8c3de85c2c6 100644 --- a/.gitignore +++ b/.gitignore @@ -103,4 +103,10 @@ venv/ .hypothesis/ .ruff_cache/ .ipynb_checkpoints/ + +# generated by the JupyterLite docs build (doc/conf.py) +doc/jupyterlite_contents/* +!doc/jupyterlite_contents/.gitkeep +doc/lite_extra/ + /.claude/ \ No newline at end of file diff --git a/doc/Makefile b/doc/Makefile index be167d67c33..3926d206a71 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -65,7 +65,7 @@ html_dev-noplot: html-noplot html_dev-front: html-front linkcheck: - @$(SPHINXBUILD) -b linkcheck -D nitpicky=0 -q -D plot_gallery=0 -D exclude_patterns="cited.rst,whats_new.rst,configure_git.rst,_includes,changes/dev" -d _build/doctrees . _build/linkcheck + @$(SPHINXBUILD) -b linkcheck -D nitpicky=0 -q -D plot_gallery=0 -D exclude_patterns="cited.rst,whats_new.rst,configure_git.rst,_includes,changes/dev,jupyterlite_contents,lite_extra,pypi" -d _build/doctrees . _build/linkcheck doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) _build/doctest diff --git a/doc/conf.py b/doc/conf.py index 875b88c7935..9c16ee5e494 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -11,6 +11,7 @@ import faulthandler import os +import shutil import subprocess import sys import tomllib @@ -53,6 +54,7 @@ curpath = Path(__file__).parent.resolve(strict=True) sys.path.append(str(curpath / "sphinxext")) +from build_lite_wheel import build_wheel, find_wheels # noqa: E402 from credit_tools import generate_credit_rst # noqa: E402 from mne_doc_utils import report_scraper, reset_warnings, sphinx_logger # noqa: E402 @@ -117,6 +119,7 @@ "sphinx_copybutton", "sphinx_design", "sphinx_gallery.gen_gallery", + "jupyterlite_sphinx", "sphinxcontrib.bibtex", "sphinxcontrib.youtube", "sphinxcontrib.towncrier.ext", @@ -140,7 +143,13 @@ # This pattern also affects html_static_path and html_extra_path. # NB: changes here should also be made to the linkcheck target in the Makefile -exclude_patterns = ["_includes", "changes/dev"] +exclude_patterns = [ + "_includes", + "changes/dev", + "jupyterlite_contents", + "lite_extra", + "pypi", +] # The suffix of source filenames. source_suffix = ".rst" @@ -476,7 +485,339 @@ compress_images = () sphinx_gallery_parallel = int(os.getenv("MNE_DOC_BUILD_N_JOBS", "1")) +jupyterlite_contents = ["jupyterlite_contents"] +jupyterlite_bind_ipynb_suffix = False + +# Inject the required subset of MNE-sample-data for JupyterLite. The data is +# placed under doc/lite_extra/mne_data and served at the docs root via +# html_extra_path (added below). The JupyterLite setup cell fetches these +# files over HTTP into the Pyodide kernel: the /drive virtual-filesystem +# bridge needs cross-origin-isolation (COOP/COEP) headers that static +# artifact servers (e.g. CircleCI) do not send, so it is unusable there. +# lite_data (mne.datasets.lite_data) extracts the curated subset here, with the +# files under their original dataset folders (MNE-sample-data/, ...). +mne_data_base = Path(os.path.expanduser("~/mne_data")) +lite_root = mne_data_base / "MNE-lite-data" +src_sample_data = lite_root / "MNE-sample-data" +lite_extra_base = ( + Path(os.path.abspath(os.path.dirname(__file__))) / "lite_extra" / "mne_data" +) +dst_sample_data = lite_extra_base / "MNE-sample-data" +dst_sample_data.mkdir(parents=True, exist_ok=True) + + +def _lite_src(folder, rel): + """Return where a dataset file can be read from, or None if nowhere. + + The curated lite_data archive only carries the files it was published with, + so look in whatever CI restored of the real dataset first and fall back to + the archive. Sourcing from the archive alone means anything added since it + was last uploaded goes missing without the build failing. + """ + for root in (mne_data_base / folder, lite_root / folder): + candidate = root / rel + if candidate.exists(): + return candidate + return None + + +sphinx_logger.info( + f"[JupyterLite] Sample data: real dataset=" + f"{(mne_data_base / 'MNE-sample-data').exists()}, " + f"curated archive={src_sample_data.exists()}" +) +if (mne_data_base / "MNE-sample-data").exists() or src_sample_data.exists(): + required_files = [ + "version.txt", + "MEG/sample/sample_audvis_raw.fif", + "MEG/sample/sample_audvis_filt-0-40_raw.fif", + "MEG/sample/sample_audvis_raw-eve.fif", + "MEG/sample/sample_audvis_filt-0-40_raw-eve.fif", + "MEG/sample/sample_audvis_ecg-proj.fif", + "MEG/sample/sample_audvis-ave.fif", + "MEG/sample/sample_audvis-cov.fif", + "MEG/sample/sample_audvis-meg-eeg-oct-6-fwd.fif", + "MEG/sample/sample_audvis-meg-oct-6-meg-inv.fif", + "MEG/sample/sample_audvis-meg-oct-6-fwd.fif", + "MEG/sample/sample_audvis-meg-oct-6-meg-fixed-inv.fif", + "MEG/sample/ernoise_raw.fif", + "MEG/sample/sample_audvis-no-filter-ave.fif", + "MEG/sample/sample_audvis_raw-trans.fif", + "MEG/sample/sample_audvis-shrunk-cov.fif", + "MEG/sample/sample_audvis-meg-lh.stc", + "MEG/sample/sample_audvis-meg-rh.stc", + "MEG/sample/sample_audvis-meg-eeg-lh.stc", + "MEG/sample/sample_audvis-meg-eeg-rh.stc", + "MEG/sample/sample_audvis_ecg-eve.fif", + # Maxwell-filter calibration pair, read from inside maxwell_filter + # rather than through a shimmable reader (86 KB, so fetched eagerly) + "SSS/sss_cal_mgh.dat", + "SSS/ct_sparse_mgh.fif", + "subjects/sample/mri/T1.mgz", + "subjects/sample/mri/aseg.mgz", + # read_talxfm builds this path itself, so nothing in the tutorials + # names it; plot_alignment needs it to estimate MRI fiducials + "subjects/sample/mri/transforms/talairach.xfm", + "subjects/sample/bem/sample-oct-6-src.fif", + # Head and skull surfaces for plot_alignment. outer_skin.surf is what + # MNE picks first, so serving it makes the browser figure match the + # rendered docs; sample-head.fif is the later fallback. There is no + # sample-head-dense.fif in the dataset; lh.seghead is the documented + # second candidate for the dense surface. (These three .surf paths are + # symlinks into bem/flash/, and copy2 follows them.) + "subjects/sample/bem/outer_skin.surf", + "subjects/sample/bem/outer_skull.surf", + "subjects/sample/bem/inner_skull.surf", + "subjects/sample/bem/sample-head.fif", + "subjects/sample/surf/lh.seghead", + # single-layer BEM solution (the 3-layer one is 237 MB, so notebooks + # needing that are excluded instead) + "subjects/sample/bem/sample-5120-bem-sol.fif", + # fsaverage source space, used by the morphing and cluster-stats + # notebooks; it ships inside MNE-sample-data + "subjects/fsaverage/bem/fsaverage-ico-5-src.fif", + "subjects/sample/surf/rh.pial", + "subjects/sample/surf/lh.pial", + "subjects/sample/surf/rh.white", + "subjects/sample/surf/lh.white", + "subjects/sample/surf/rh.inflated", + "subjects/sample/surf/lh.inflated", + "subjects/sample/surf/rh.curv", + "subjects/sample/surf/lh.curv", + # setup_source_space maps each hemisphere onto its sphere for any + # ico/oct spacing, and _create_surf_spacing reads surf/{hemi}.sphere + # by a path it builds itself (5.6 MB each) + "subjects/sample/surf/lh.sphere", + "subjects/sample/surf/rh.sphere", + "subjects/sample/label/lh.aparc.annot", + "subjects/sample/label/rh.aparc.annot", + # the auditory/visual ROIs; about nine notebooks build these names with + # an f-string, so a scan of the tutorial text never sees them + "MEG/sample/labels/Aud-lh.label", + "MEG/sample/labels/Aud-rh.label", + "MEG/sample/labels/Vis-lh.label", + "MEG/sample/labels/Vis-rh.label", + ] + for req in required_files: + s = _lite_src("MNE-sample-data", req) + d = dst_sample_data / req + if s is not None: + d.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(s, d) + sphinx_logger.info(f"[JupyterLite] Copied: {req}") + else: + sphinx_logger.info(f"[JupyterLite] MISSING: {req}") + + +# Also inject SSVEP and EEGLAB testing datasets for JupyterLite +lite_data_base = lite_extra_base +lite_data_base.mkdir(parents=True, exist_ok=True) + +src_ssvep = mne_data_base / "ssvep-example-data" +dst_ssvep = lite_data_base / "ssvep-example-data" +sphinx_logger.info(f"[JupyterLite] SSVEP data source exists: {src_ssvep.exists()}") +if src_ssvep.exists() and not dst_ssvep.exists(): + shutil.copytree(src_ssvep, dst_ssvep, dirs_exist_ok=True) + sphinx_logger.info("[JupyterLite] Copied ssvep-example-data") + +src_eeglab = mne_data_base / "MNE-testing-data" / "EEGLAB" +dst_eeglab = lite_data_base / "MNE-testing-data" / "EEGLAB" +sphinx_logger.info(f"[JupyterLite] EEGLAB data source exists: {src_eeglab.exists()}") +if src_eeglab.exists() and not dst_eeglab.exists(): + shutil.copytree(src_eeglab, dst_eeglab, dirs_exist_ok=True) + sphinx_logger.info("[JupyterLite] Copied MNE-testing-data/EEGLAB") + +# The head-position and Maxwell-filtering tutorials read one continuous +# movement recording out of the testing dataset. CI already restores it from +# data-cache-testing, so only these two files are copied, not the 1.6 GB set. +testing_files = [ + "SSS/test_move_anon_raw.fif", + "SSS/test_move_anon_raw.pos", +] +for testing_file in testing_files: + s = _lite_src("MNE-testing-data", testing_file) + d = lite_data_base / "MNE-testing-data" / testing_file + if s is not None and not d.exists(): + d.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(s, d) + _mb = s.stat().st_size / 1e6 + sphinx_logger.info(f"[JupyterLite] Copied {testing_file} ({_mb:.1f} MB)") + elif s is None: + sphinx_logger.info(f"[JupyterLite] MISSING {testing_file}") + +# The remaining datasets are each used by one or two notebooks that read only a +# couple of files out of them. CI already downloads all of these in +# tools/circleci_download.sh, so copying is free, but their sizes vary a lot, +# so refuse anything past this limit rather than bloat the artifact. For scale, +# the largest file already served (sample_audvis_raw.fif) is 128 MB. +LITE_MAX_FILE_MB = 150 + + +def _lite_copy(folder, rel_paths): + """Copy selected files of a dataset into the served tree.""" + for rel in rel_paths: + s = _lite_src(folder, rel) + if s is None: + sphinx_logger.info(f"[JupyterLite] MISSING {folder}/{rel}") + continue + size_mb = s.stat().st_size / 1e6 + if size_mb > LITE_MAX_FILE_MB: + sphinx_logger.info( + f"[JupyterLite] SKIPPED {folder}/{rel} ({size_mb:.1f} MB)" + ) + continue + d = lite_data_base / folder / rel + if not d.exists(): + d.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(s, d) + sphinx_logger.info( + f"[JupyterLite] Copied {folder}/{rel} ({size_mb:.1f} MB)" + ) + + +def _lite_copy_tree(folder, rel_dir): + """Copy a directory-shaped recording, leaving a manifest for the browser. + + read_raw_nirx and read_raw_egi are handed a folder rather than a file, so + the setup cell has no way to know what to fetch without a listing. + """ + src = mne_data_base / folder / rel_dir + if not src.is_dir(): + sphinx_logger.info(f"[JupyterLite] MISSING {folder}/{rel_dir}") + return + names, total_mb = [], 0.0 + for f in sorted(src.rglob("*")): + if not f.is_file(): + continue + # zero-byte members (an .mff carries a couple of lock files) do not + # survive the artifact upload, so listing them only yields a 404 + if f.stat().st_size == 0: + continue + size_mb = f.stat().st_size / 1e6 + if size_mb > LITE_MAX_FILE_MB: + sphinx_logger.info( + f"[JupyterLite] SKIPPED {folder}/{rel_dir} ({size_mb:.1f} MB)" + ) + return + names.append(str(f.relative_to(src))) + total_mb += size_mb + dst = lite_data_base / folder / rel_dir + dst.mkdir(parents=True, exist_ok=True) + for name in names: + d = dst / name + d.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src / name, d) + (dst / "_lite_manifest.txt").write_text("\n".join(names)) + sphinx_logger.info( + f"[JupyterLite] Copied {folder}/{rel_dir} " + f"({len(names)} files, {total_mb:.1f} MB)" + ) + + +_lite_copy( + "MNE-misc-data", + [ + "xdf/sub-P001_ses-S004_task-Default_run-001_eeg_a2.xdf", + "movement/simulated_quats.pos", + "movement/simulated_movement_raw.fif", + "movement/simulated_stationary_raw.fif", + "eyetracking/eyelink/px_textpage_ws.asc", + "eyetracking/eyelink/HREF_textpage_ws.asc", + ], +) +_lite_copy( + "MNE-eyelink-data", + [ + "freeviewing/sub-01_task-freeview_eyetrack.asc", + "freeviewing/stim/naturalistic.png", + "eeg-et/sub-01_task-plr_eyetrack.asc", + ], +) +_lite_copy_tree("MNE-eyelink-data", "eeg-et/sub-01_task-plr_eeg.mff") +_lite_copy_tree("MNE-fNIRS-motor-data", "Participant-1") + +# The logging tutorial reads a KIT file that lives inside the package itself, +# under mne/io/kit/tests/. pyproject excludes "/mne/**/tests" from the wheel, so +# it is absent from the browser kernel, so serve it and let the setup cell stage +# it back into the path the tutorial builds. +_kit_src = Path(mne.__file__).parent / "io" / "kit" / "tests" / "data" / "test.sqd" +_kit_dst = lite_data_base / "MNE-kit-testdata" / "test.sqd" +if _kit_src.exists(): + if not _kit_dst.exists(): + _kit_dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(_kit_src, _kit_dst) + sphinx_logger.info( + f"[JupyterLite] Copied MNE-kit-testdata/test.sqd " + f"({_kit_src.stat().st_size / 1e6:.1f} MB)" + ) +else: + sphinx_logger.info("[JupyterLite] MISSING MNE-kit-testdata/test.sqd") + +_lite_copy("MNE-phantom-kernel-data", ["phantom_32_100nam_raw.fif"]) +_lite_copy("MNE-multimodal-data", ["multimodal_raw.fif"]) +_lite_copy("MNE-refmeg-noise-data", ["sample_reference_MEG_noise-raw.fif"]) + +# somato is deliberately not served: its raw alone is 344 MB and the six +# notebooks that read it are on the exclude list instead. + +# Inject the single needed file(s) from extra datasets used by the Epochs and +# decoding examples. Sizes are all within what we already serve +# (sample_audvis_raw.fif is 128.5 MB): kiloword 28.7 MB, erp_core 123.6 MB, +# mtrf speech_data.mat 17.2 MB, eegbci 3x2.6 MB. The CI "Ensure ... data" step +# downloads them so the sources exist here. +for _folder, _ds_files in ( + ("MNE-kiloword-data", ["kword_metadata-epo.fif"]), + ("MNE-ERP-CORE-data", ["ERP-CORE_Subject-001_Task-Flankers_eeg.fif"]), + ("mTRF_1.5", ["speech_data.mat"]), + ( + "MNE-eegbci-data", + # exactly the runs tools/circleci_download.sh fetches: subject 1 runs + # 3/6/10/14 and run 3 for subjects 2-4. Notebooks wanting run 1 or 2 are + # excluded instead, since that data never reaches the CI box. + [ + "files/eegmmidb/1.0.0/S001/S001R03.edf", + "files/eegmmidb/1.0.0/S001/S001R06.edf", + "files/eegmmidb/1.0.0/S001/S001R10.edf", + "files/eegmmidb/1.0.0/S001/S001R14.edf", + "files/eegmmidb/1.0.0/S002/S002R03.edf", + "files/eegmmidb/1.0.0/S003/S003R03.edf", + "files/eegmmidb/1.0.0/S004/S004R03.edf", + ], + ), +): + _dst_ds = lite_data_base / _folder + for _ds_file in _ds_files: + s = _lite_src(_folder, _ds_file) + d = _dst_ds / _ds_file + if s is not None: + d.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(s, d) + sphinx_logger.info(f"[JupyterLite] Copied: {_folder}/{_ds_file}") + else: + sphinx_logger.info(f"[JupyterLite] MISSING: {_folder}/{_ds_file}") + + +# Provide the development MNE wheel so JupyterLite installs the current version +# rather than the older release from PyPI. ``doc/sphinxext/build_lite_wheel.py`` +# builds it into ``doc/pypi``, where the jupyterlite-pyodide-kernel PipliteAddon +# discovers and indexes it. Running that script before the docs build (in CI or +# locally) means Sphinx reuses the wheel instead of rebuilding it on every +# invocation; if none is present we build it here, so the docs build never +# depends on the pre-step having run. +_lite_wheels = find_wheels() or build_wheel() +sphinx_logger.info(f"[JupyterLite] MNE wheel for the browser kernel: {_lite_wheels}") + sphinx_gallery_conf = { + "jupyterlite": { + "use_jupyter_lab": True, + "jupyterlite_contents": "jupyterlite_contents", + # named rather than passed: sphinx_gallery_conf has to stay + # JSON-serializable (see the is_serializable assert below), so + # sphinx-gallery imports this dotted path itself + "notebook_modification_function": ( + "jupyterlite_cell_notes.note_unrunnable_cells" + ), + }, "doc_module": ("mne",), "reference_url": dict(mne=None), "examples_dirs": examples_dirs, @@ -578,6 +919,138 @@ "parallel": sphinx_gallery_parallel, } assert is_serializable(sphinx_gallery_conf) + +# --------------------------------------------------------------------------- +# Drop the "Open in JupyterLite" launch badge from gallery pages whose +# notebooks cannot run in the browser kernel at all: they need the R runtime +# (rpy2), a compiled package Pyodide does not ship (antio), or multi-GB +# datasets that cannot be bundled/slimmed. sphinx-gallery adds the badge to +# every example unconditionally, so we wrap its badge generator and return an +# empty string for these files. This only removes the badge/link; the +# notebook source is untouched (no in-code guard). Files that merely need data +# bundled, a pure-Python package installed, or pyvista 3D are NOT listed here +# (they are fixable, not impossible). +JUPYTERLITE_EXCLUDE = ( + # Tier 1, impossible: R runtime / compiled package / huge single dataset + "examples/stats/r_interop.py", # rpy2 -> needs the R runtime + "examples/io/read_impedances.py", # antio (compiled, not in Pyodide) + "examples/decoding/decoding_rsa_sgskip.py", # visual_92_categories ~6 GB + "examples/decoding/decoding_spoc_CMC.py", # fieldtrip_cmc ~700 MB + "examples/decoding/ssd_spatial_filters.py", # fieldtrip_cmc ~700 MB + # Tier 2: multi-GB datasets (brainstorm / spm_face / opm / hf_sef) + "examples/datasets/brainstorm_data.py", + "examples/datasets/hf_sef_data.py", + "examples/datasets/opm_data.py", + "examples/datasets/spm_faces_dataset.py", + "examples/preprocessing/movement_detection.py", + "examples/preprocessing/muscle_detection.py", + "examples/preprocessing/otp.py", + "examples/time_frequency/source_power_spectrum_opm.py", + "examples/visualization/evoked_arrowmap.py", + "examples/visualization/meg_sensors.py", + "tutorials/inverse/80_brainstorm_phantom_elekta.py", + "tutorials/inverse/85_brainstorm_phantom_ctf.py", + "tutorials/io/60_ctf_bst_auditory.py", + "tutorials/preprocessing/80_opm_processing.py", + # Tier 3: several blockers each, none of them worth clearing on its own + # the volume inverse is ~178 MB and volume source estimates are not + # rendered in the browser + "examples/inverse/compute_mne_inverse_volume.py", + # needs aseg.mgz and the mixed source space, and calls src.plot(), which + # is the 3D SourceSpaces view + "examples/inverse/mixed_source_space_inverse.py", + # nilearn.datasets.load_mni152_template() downloads a template at runtime, + # which the browser blocks (CORS); the surrounding try only catches + # TypeError, so the failure is not survivable + "tutorials/inverse/20_dipole_fit.py", + # make_field_map(upsampling=2) subdivides the helmet mesh through VTK, and + # plot_field needs the interactive viewer that the browser renderer skips + "examples/visualization/mne_helmet.py", + # Tier 4: mne.viz.Brain. The browser renderer draws static meshes; Brain + # additionally wants dock widgets, a toolbar and a time slider, so these + # are blocked on the interactive layer rather than on data. + "examples/visualization/brain.py", + "examples/visualization/parcellation.py", + "tutorials/clinical/20_seeg.py", + "tutorials/forward/10_background_freesurfer.py", + "tutorials/forward/50_background_freesurfer_mne.py", + "tutorials/inverse/60_visualize_stc.py", + "tutorials/io/30_reading_fnirs_data.py", + "tutorials/preprocessing/70_fnirs_processing.py", + # Tier 5: one-off blockers with no browser path + # plot_field needs the interactive viewer + "tutorials/evoked/20_visualize_evoked.py", + # the three-layer BEM solution alone is 237 MB + "examples/inverse/multi_dipole_model.py", + # openneuro fetches the recording at runtime, which the browser blocks + "examples/preprocessing/esg_rm_heart_artefact_pcaobs.py", + # physionet.org is not CORS-enabled and the dataset is not on the CI box + "tutorials/clinical/60_sleep.py", + # the 4D/BTi phantom dataset is not among the ones CI downloads + "tutorials/inverse/90_phantom_4DBTi.py", + # needs mne_bids as well as the epilepsy_ecog dataset and 3D sensor views + "tutorials/clinical/30_ecog.py", + # Tier 6: fetch_fsaverage. _manifest_check_download only skips the + # download when every one of its ~190 manifest entries is already present, + # so fsaverage cannot be part-bundled, and MNE-sample-data ships no + # fsaverage/bem at all. The volume forward and inverse these two want are + # 187 MB and 360 MB on top of that. + "examples/inverse/morph_volume_stc.py", + "tutorials/inverse/50_beamformer_lcmv.py", + "examples/visualization/montage_sgskip.py", + # same, plus fetch_infant_template downloads a second template + "tutorials/forward/35_eeg_no_mri.py", + # snapshot_brain_montage needs a real 3D window to read pixels back from + "examples/visualization/3d_to_2d.py", + # the three-layer BEM solution is 237 MB, and T1_electrodes.mgz would pull + # in the misc dataset's MRI as well + "tutorials/inverse/70_eeg_mri_coords.py", + # mne_bids is not installable in the browser kernel + "tutorials/inverse/95_phantom_KIT.py", + # Tier 7: somato. Serving it costs 404 MB (the raw alone is 344 MB) on + # every docs deploy, which is more than these six pages are worth; the + # dataset is not copied at all. Restoring them means putting the somato + # block back in the copy step above. + "examples/inverse/dics_epochs.py", + "examples/inverse/dics_source_power.py", + "examples/inverse/evoked_ers_source_power.py", + "examples/inverse/multidict_reweighted_tfmxne.py", + "examples/time_frequency/time_frequency_global_field_power.py", + "tutorials/time-freq/20_sensors_time_frequency.py", + # Single recordings well past LITE_MAX_FILE_MB, confirmed against the full + # build: 379 MB and 251 MB for one example each, so they are skipped by the + # copy step and the badge would have nothing to load. + "examples/datasets/kernel_phantom.py", + "examples/io/elekta_epochs.py", + # These want EEGBCI runs 1 and 2, which tools/circleci_download.sh never + # fetches (it takes subject 1 runs 3/6/10/14 and run 3 for subjects 2-4), + # so the data is not on the machine that builds the docs. eeg_bridging + # alone would need run 1 for ten subjects. + "examples/visualization/onionskin.py", + "examples/preprocessing/muscle_ica.py", + "examples/preprocessing/eeg_bridging.py", + # Both Report tutorials build their figures by screenshotting a 3D scene + # (Report._itv calls backend._take_3d_screenshot), and vtk.js cannot hand a + # framebuffer back to Python, so those sections would embed blank images. + # 70_report additionally round-trips a report through HDF5. + "tutorials/intro/70_report.py", + "tutorials/preprocessing/14_quality_control_report.py", +) + +import sphinx_gallery.gen_rst as _sg_gen_rst # noqa: E402 + +_orig_gen_jupyterlite_rst = _sg_gen_rst.gen_jupyterlite_rst + + +def _lite_badge_filtered(fpath, gallery_conf): + """Return the JupyterLite badge reST, or "" for excluded notebooks.""" + _p = str(fpath).replace(os.sep, "/") + if any(_p.endswith(_ex) for _ex in JUPYTERLITE_EXCLUDE): + return "" + return _orig_gen_jupyterlite_rst(fpath, gallery_conf) + + +_sg_gen_rst.gen_jupyterlite_rst = _lite_badge_filtered # Files were renamed from plot_* with: # find . -type f -name 'plot_*.py' -exec sh -c 'x="{}"; xn=`basename "${x}"`; git mv "$x" `dirname "${x}"`/${xn:5}' \; # noqa @@ -882,6 +1355,9 @@ def fix_sklearn_inherited_docstrings(app, what, name, obj, options, lines): "documentation.html", "getting_started.html", "install_mne_python.html", + # Serve the pre-bundled JupyterLite sample data at the docs root + # (e.g. /mne_data/...). The lite setup cell fetches it over HTTP. + "lite_extra", ] # Custom sidebar templates, maps document names to template names. diff --git a/doc/documentation/datasets.rst b/doc/documentation/datasets.rst index 2ec98664e74..66b93b992d9 100644 --- a/doc/documentation/datasets.rst +++ b/doc/documentation/datasets.rst @@ -541,6 +541,17 @@ the people in the scene were unrecognizable. * :ref:`tut-eyetrack-heatmap` +.. note:: Not every tutorial and example can run in the browser, so the + "Open in JupyterLite" badge is only shown on the pages that work there. A page + is left without a badge when it needs a non-Python runtime (for example the R + interoperability example, via ``rpy2``), a compiled reader with no WebAssembly + build (such as ``antio``), or a dataset too large to serve to a browser + (brainstorm, spm_face, opm, hf_sef, and similar). + + 3D is also limited: source estimates are rendered with vtk.js, but the + coregistration and sensor-plotting views that rely on the full VTK stack are + not available. + References ========== diff --git a/doc/jupyterlite_contents/.gitkeep b/doc/jupyterlite_contents/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/doc/sphinxext/jupyterlite_cell_notes.py b/doc/sphinxext/jupyterlite_cell_notes.py new file mode 100644 index 00000000000..8897185d0ae --- /dev/null +++ b/doc/sphinxext/jupyterlite_cell_notes.py @@ -0,0 +1,109 @@ +"""Per-notebook fixups applied to the JupyterLite copies only. + +:func:`note_unrunnable_cells` does two things to every notebook sphinx-gallery +copies into the JupyterLite contents: + +1. Prepends the setup cell that installs MNE and patches the browser + environment (see :mod:`jupyterlite_setup_cell`). +2. Swaps any cell that cannot run in the browser for a note that keeps the code + in view and says what it needs instead; a cell here and there is blocked + even though the rest of its notebook runs, and dropping a whole page from + the launcher over one cell costs more than it saves. + +Both are deliberately done here rather than through ``first_notebook_cell``, +which sphinx-gallery applies while *generating* the notebook and therefore also +writes into the ``.ipynb`` offered for download, where ``piplite`` does not +exist and the notebook would fail on its first cell. Doing it at copy time +keeps the download and the rendered page exactly as the docs built them. + +It lives here rather than in ``conf.py`` because ``sphinx_gallery_conf`` has to +stay JSON-serializable (``sphinx.config.is_serializable`` rejects functions), so +the config names the dotted path and sphinx-gallery imports it. +""" + +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + +import os + +import sphinx.util.logging +from jupyterlite_setup_cell import LITE_SETUP_CELL + +# not mne_doc_utils.sphinx_logger: that module pulls in mne and pyvista, which +# this one has no use for +logger = sphinx.util.logging.getLogger("mne") + +# first line of LITE_SETUP_CELL, used to spot a cell that is already there +_SETUP_MARKER = LITE_SETUP_CELL.strip().split("\n", 1)[0] + +# (notebook path suffix, substring identifying the cell, replacement markdown). +# Keep this short: a page that is mostly unavailable belongs in +# JUPYTERLITE_EXCLUDE instead of here. +CELL_NOTES = ( + ( + "forward/20_source_alignment.ipynb", + "mne.gui.coregistration", + "**This cell does not run in the browser.**\n" + "\n" + "`mne.gui.coregistration` sets the fiducials by clicking on the scalp\n" + "surface, and the vtk.js renderer used here draws scenes without a\n" + "picker, so there is nothing for those clicks to hit. Run it from a\n" + "local MNE install instead:\n" + "\n" + "```python\n" + 'mne.gui.coregistration(subject="sample", subjects_dir=subjects_dir)\n' + "```\n" + "\n" + "The video above walks through the same steps, and the rest of this\n" + "notebook runs normally.\n", + ), +) + + +def note_unrunnable_cells(notebook_content, notebook_filename): + """Add the setup cell and note the cells that cannot run in the browser. + + Parameters + ---------- + notebook_content : dict + The parsed notebook, modified in place. + notebook_filename : path-like + Where the notebook will be written inside the JupyterLite contents. + """ + # setdefault, not get: a missing key would otherwise hand back a throwaway + # list and the insert would silently not stick + cells = notebook_content.setdefault("cells", []) + # stale .ipynb from an earlier build can already carry the cell; adding a + # second one would install everything twice + already = cells and _SETUP_MARKER in "".join(cells[0].get("source", [])) + if not already: + cells.insert( + 0, + { + "cell_type": "code", + "execution_count": None, + "metadata": {"collapsed": False}, + "outputs": [], + # .strip() to match what sphinx-gallery's add_code_cell wrote + # while this went through first_notebook_cell + "source": [LITE_SETUP_CELL.strip()], + }, + ) + path = str(notebook_filename).replace(os.sep, "/") + for suffix, needle, note in CELL_NOTES: + if not path.endswith(suffix): + continue + for cell in cells: + # prose mentions the same function, so only rewrite real code + if cell.get("cell_type") != "code": + continue + if needle not in "".join(cell.get("source", [])): + continue + cell["cell_type"] = "markdown" + cell["source"] = [note] + cell["metadata"] = {} + # markdown cells carry neither of these, and nbformat rejects them + cell.pop("outputs", None) + cell.pop("execution_count", None) + logger.info(f"[JupyterLite] {suffix}: noted {needle} cell") From 74e2101b9ad4951e8231b8ee37c848cf46d5715f Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Fri, 14 Aug 2026 09:19:59 -0400 Subject: [PATCH 02/12] DOC: add the changelog entry for the JupyterLite docs wiring --- doc/changes/dev/14157.newfeature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/dev/14157.newfeature.rst diff --git a/doc/changes/dev/14157.newfeature.rst b/doc/changes/dev/14157.newfeature.rst new file mode 100644 index 00000000000..879bbec54a8 --- /dev/null +++ b/doc/changes/dev/14157.newfeature.rst @@ -0,0 +1 @@ +Add an "Open in JupyterLite" badge to the tutorials and examples that can run in the browser, by `Natneal B`_. From aeb6018dc7f66ac783812ef6d8d476c4e513bf7f Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Fri, 14 Aug 2026 09:52:50 -0400 Subject: [PATCH 03/12] MAINT: run the JupyterLite data step with the other data work It sits with "Get data and triage examples to run" rather than after test-doc. Same effect, since all it has to do is precede make html, and it keeps the step away from the wheel build's insertion point. --- .circleci/config.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f9b18d3ee01..a09aaf20b5e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -232,6 +232,17 @@ jobs: no_output_timeout: 10m command: | ./tools/circleci_download.sh + # Ensure the data the JupyterLite notebooks need is on disk so conf.py can + # copy the required subset for the build. lite_data fetches only the + # curated files the browser notebooks use (from the MNE-lite-data OSF + # project) instead of the full sample/kiloword/erp_core/mtrf/eegbci + # datasets, which slims the build. The curated files share hashes with the + # full datasets, so on a full build that already downloaded them nothing + # extra is fetched. + - run: + name: Ensure MNE data for JupyterLite + command: | + python -c "import mne; mne.datasets.lite_data.data_path(update_path=True)" - run: name: Verify build type command: | @@ -249,17 +260,6 @@ jobs: cp junit-results.xml doc/_build/test-results/test-doc/junit.xml; cp coverage.xml doc/_build/test-results/test-doc/coverage.xml; fi; - # Ensure the data the JupyterLite notebooks need is on disk so conf.py can - # copy the required subset for the build. lite_data fetches only the - # curated files the browser notebooks use (from the MNE-lite-data OSF - # project) instead of the full sample/kiloword/erp_core/mtrf/eegbci - # datasets, which slims the build. The curated files share hashes with the - # full datasets, so on a full build that already downloaded them nothing - # extra is fetched. - - run: - name: Ensure MNE data for JupyterLite - command: | - python -c "import mne; mne.datasets.lite_data.data_path(update_path=True)" # Build docs - run: name: make html From 4613c2950655e737cef76c9aea291739db378d67 Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Tue, 18 Aug 2026 23:03:25 -0400 Subject: [PATCH 04/12] MAINT: log the JupyterLite wheel path as a plain string find_wheels() returns Path objects since the wheel build moved to pathlib, so the log line was rendering as [PosixPath('...')]. --- doc/conf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 96d8558ca85..02f55ee20ba 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -805,7 +805,10 @@ def _lite_copy_tree(folder, rel_dir): # invocation; if none is present we build it here, so the docs build never # depends on the pre-step having run. _lite_wheels = find_wheels() or build_wheel() -sphinx_logger.info(f"[JupyterLite] MNE wheel for the browser kernel: {_lite_wheels}") +_lite_wheel_names = ", ".join(str(_wheel) for _wheel in _lite_wheels) +sphinx_logger.info( + f"[JupyterLite] MNE wheel for the browser kernel: {_lite_wheel_names}" +) sphinx_gallery_conf = { "jupyterlite": { From cc3a9ef0048112af89b6e5295966de04e910b1e6 Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Wed, 26 Aug 2026 13:33:22 -0400 Subject: [PATCH 05/12] MAINT: exclude 10_publication_figure from the JupyterLite build The tutorial crops the white margins off brain.screenshot(), and vtk.js cannot hand a framebuffer back to Python, so there is no screenshot to crop. --- doc/conf.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 876e4212309..07eebb83777 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1037,12 +1037,16 @@ def _lite_copy_tree(folder, rel_dir): "examples/visualization/onionskin.py", "examples/preprocessing/muscle_ica.py", "examples/preprocessing/eeg_bridging.py", - # Both Report tutorials build their figures by screenshotting a 3D scene - # (Report._itv calls backend._take_3d_screenshot), and vtk.js cannot hand a - # framebuffer back to Python, so those sections would embed blank images. + # These read a 3D scene back as pixels, and vtk.js cannot hand a + # framebuffer back to Python. Both Report tutorials build their figures by + # screenshotting (Report._itv calls backend._take_3d_screenshot), and # 70_report additionally round-trips a report through HDF5. "tutorials/intro/70_report.py", "tutorials/preprocessing/14_quality_control_report.py", + # 10_publication_figure is about cropping the white margins off + # brain.screenshot(), so without a real screenshot there is no tutorial + # left; browser brain.screenshot() raises rather than return a blank image. + "tutorials/visualization/10_publication_figure.py", ) import sphinx_gallery.gen_rst as _sg_gen_rst # noqa: E402 From 4534977a37e605faed269bcad440cac707b879f1 Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Wed, 26 Aug 2026 14:58:21 -0400 Subject: [PATCH 06/12] MAINT: refresh the JupyterLite exclude list Two entries had gone stale: upstream renamed decoding_rsa_sgskip.py and montage_sgskip.py, so both examples were badged again despite still needing the datasets they were excluded for. Also excludes the new 21_interactive_dipole_fit, which is a tour of mne.gui.dipolefit and needs a picker vtk.js does not provide. --- doc/conf.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index ba66393c37e..f59ab1eff92 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -943,7 +943,7 @@ def _lite_copy_tree(folder, rel_dir): # Tier 1, impossible: R runtime / compiled package / huge single dataset "examples/stats/r_interop.py", # rpy2 -> needs the R runtime "examples/io/read_impedances.py", # antio (compiled, not in Pyodide) - "examples/decoding/decoding_rsa_sgskip.py", # visual_92_categories ~6 GB + "examples/decoding/decoding_rsa.py", # visual_92_categories ~6 GB "examples/decoding/decoding_spoc_CMC.py", # fieldtrip_cmc ~700 MB "examples/decoding/ssd_spatial_filters.py", # fieldtrip_cmc ~700 MB # Tier 2: multi-GB datasets (brainstorm / spm_face / opm / hf_sef) @@ -1006,7 +1006,7 @@ def _lite_copy_tree(folder, rel_dir): # 187 MB and 360 MB on top of that. "examples/inverse/morph_volume_stc.py", "tutorials/inverse/50_beamformer_lcmv.py", - "examples/visualization/montage_sgskip.py", + "examples/visualization/montage.py", # same, plus fetch_infant_template downloads a second template "tutorials/forward/35_eeg_no_mri.py", # snapshot_brain_montage needs a real 3D window to read pixels back from @@ -1048,6 +1048,12 @@ def _lite_copy_tree(folder, rel_dir): # brain.screenshot(), so without a real screenshot there is no tutorial # left; browser brain.screenshot() raises rather than return a blank image. "tutorials/visualization/10_publication_figure.py", + # The whole page drives mne.gui.dipolefit and narrates one GUI window as + # its state evolves. The vtk.js renderer draws without a picker, so there + # is nothing for those clicks to hit; that is also why 20_source_alignment + # carries a cell note for mne.gui.coregistration. Here it is the entire + # tutorial rather than one cell, so it is excluded instead. + "tutorials/inverse/21_interactive_dipole_fit.py", ) import sphinx_gallery.gen_rst as _sg_gen_rst # noqa: E402 From 45e32742ac4471f4f44ab26733e745100ab68855 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 11 Sep 2026 12:51:33 -0400 Subject: [PATCH 07/12] Gate the JupyterLite build behind full builds and refresh the exclusions Stage the served data from a sphinxext module only when MNE_DOC_BUILD_JUPYTERLITE=1, which make html sets, so pattern and noplot builds skip the copy and the lite build; skip files already staged; drop the exclusions for the pages that now run through the backend; fix stale text. --- doc/Makefile | 10 +- doc/conf.py | 374 +++--------------------------- doc/documentation/datasets.rst | 6 +- doc/sphinxext/jupyterlite_data.py | 211 +++++++++++++++++ 4 files changed, 247 insertions(+), 354 deletions(-) create mode 100644 doc/sphinxext/jupyterlite_data.py diff --git a/doc/Makefile b/doc/Makefile index db1e632798f..6fa28f2f441 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -34,26 +34,26 @@ jupyterlite_wheel: @python sphinxext/build_lite_wheel.py html: jupyterlite_wheel - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) _build/html + MNE_DOC_BUILD_JUPYTERLITE=1 $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) _build/html @echo @echo "Build finished. The HTML pages are in _build/html." html-memory: jupyterlite_wheel - $(MPROF) -b html $(ALLSPHINXOPTS) _build/html + MNE_DOC_BUILD_JUPYTERLITE=1 $(MPROF) -b html $(ALLSPHINXOPTS) _build/html @echo @echo "Build finished. The HTML pages are in _build/html." -html-pattern: jupyterlite_wheel +html-pattern: $(SPHINXBUILD) -D sphinx_gallery_conf.filename_pattern=$(PATTERN) -D sphinx_gallery_conf.run_stale_examples=True -b html $(ALLSPHINXOPTS) _build/html @echo @echo "Build finished. The HTML pages are in _build/html" -html-pattern-memory: jupyterlite_wheel +html-pattern-memory: $(MPROF) -D sphinx_gallery_conf.filename_pattern=$(PATTERN) -D sphinx_gallery_conf.run_stale_examples=True -b html $(ALLSPHINXOPTS) _build/html @echo @echo "Build finished. The HTML pages are in _build/html" -html-noplot: jupyterlite_wheel +html-noplot: $(SPHINXBUILD) -D plot_gallery=0 -b html $(ALLSPHINXOPTS) _build/html @echo @echo "Build finished. The HTML pages are in _build/html." diff --git a/doc/conf.py b/doc/conf.py index ce6b0d70c24..8c8727eb2b8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -11,7 +11,6 @@ import faulthandler import os -import shutil import subprocess import sys import tomllib @@ -54,7 +53,6 @@ curpath = Path(__file__).parent.resolve(strict=True) sys.path.append(str(curpath / "sphinxext")) -from build_lite_wheel import build_wheel, find_wheels # noqa: E402 from credit_tools import generate_credit_rst # noqa: E402 from mne_doc_utils import ( # noqa: E402 check_links, @@ -124,7 +122,6 @@ "sphinx_copybutton", "sphinx_design", "sphinx_gallery.gen_gallery", - "jupyterlite_sphinx", "sphinxcontrib.bibtex", "sphinxcontrib.youtube", "sphinxcontrib.towncrier.ext", @@ -491,342 +488,19 @@ compress_images = () sphinx_gallery_parallel = int(os.getenv("MNE_DOC_BUILD_N_JOBS", "1")) -jupyterlite_contents = ["jupyterlite_contents"] -jupyterlite_bind_ipynb_suffix = False - -# Inject the required subset of MNE-sample-data for JupyterLite. The data is -# placed under doc/lite_extra/mne_data and served at the docs root via -# html_extra_path (added below). The JupyterLite setup cell fetches these -# files over HTTP into the Pyodide kernel: the /drive virtual-filesystem -# bridge needs cross-origin-isolation (COOP/COEP) headers that static -# artifact servers (e.g. CircleCI) do not send, so it is unusable there. -# lite_data (mne.datasets.lite_data) extracts the curated subset here, with the -# files under their original dataset folders (MNE-sample-data/, ...). -mne_data_base = Path(os.path.expanduser("~/mne_data")) -lite_root = mne_data_base / "MNE-lite-data" -src_sample_data = lite_root / "MNE-sample-data" -lite_extra_base = ( - Path(os.path.abspath(os.path.dirname(__file__))) / "lite_extra" / "mne_data" -) -dst_sample_data = lite_extra_base / "MNE-sample-data" -dst_sample_data.mkdir(parents=True, exist_ok=True) - - -def _lite_src(folder, rel): - """Return where a dataset file can be read from, or None if nowhere. - - The curated lite_data archive only carries the files it was published with, - so look in whatever CI restored of the real dataset first and fall back to - the archive. Sourcing from the archive alone means anything added since it - was last uploaded goes missing without the build failing. - """ - for root in (mne_data_base / folder, lite_root / folder): - candidate = root / rel - if candidate.exists(): - return candidate - return None - - -sphinx_logger.info( - f"[JupyterLite] Sample data: real dataset=" - f"{(mne_data_base / 'MNE-sample-data').exists()}, " - f"curated archive={src_sample_data.exists()}" -) -if (mne_data_base / "MNE-sample-data").exists() or src_sample_data.exists(): - required_files = [ - "version.txt", - "MEG/sample/sample_audvis_raw.fif", - "MEG/sample/sample_audvis_filt-0-40_raw.fif", - "MEG/sample/sample_audvis_raw-eve.fif", - "MEG/sample/sample_audvis_filt-0-40_raw-eve.fif", - "MEG/sample/sample_audvis_ecg-proj.fif", - "MEG/sample/sample_audvis-ave.fif", - "MEG/sample/sample_audvis-cov.fif", - "MEG/sample/sample_audvis-meg-eeg-oct-6-fwd.fif", - "MEG/sample/sample_audvis-meg-oct-6-meg-inv.fif", - "MEG/sample/sample_audvis-meg-oct-6-fwd.fif", - "MEG/sample/sample_audvis-meg-oct-6-meg-fixed-inv.fif", - "MEG/sample/ernoise_raw.fif", - "MEG/sample/sample_audvis-no-filter-ave.fif", - "MEG/sample/sample_audvis_raw-trans.fif", - "MEG/sample/sample_audvis-shrunk-cov.fif", - "MEG/sample/sample_audvis-meg-lh.stc", - "MEG/sample/sample_audvis-meg-rh.stc", - "MEG/sample/sample_audvis-meg-eeg-lh.stc", - "MEG/sample/sample_audvis-meg-eeg-rh.stc", - "MEG/sample/sample_audvis_ecg-eve.fif", - # Maxwell-filter calibration pair, read from inside maxwell_filter - # rather than through a shimmable reader (86 KB, so fetched eagerly) - "SSS/sss_cal_mgh.dat", - "SSS/ct_sparse_mgh.fif", - "subjects/sample/mri/T1.mgz", - "subjects/sample/mri/aseg.mgz", - # read_talxfm builds this path itself, so nothing in the tutorials - # names it; plot_alignment needs it to estimate MRI fiducials - "subjects/sample/mri/transforms/talairach.xfm", - "subjects/sample/bem/sample-oct-6-src.fif", - # Head and skull surfaces for plot_alignment. outer_skin.surf is what - # MNE picks first, so serving it makes the browser figure match the - # rendered docs; sample-head.fif is the later fallback. There is no - # sample-head-dense.fif in the dataset; lh.seghead is the documented - # second candidate for the dense surface. (These three .surf paths are - # symlinks into bem/flash/, and copy2 follows them.) - "subjects/sample/bem/outer_skin.surf", - "subjects/sample/bem/outer_skull.surf", - "subjects/sample/bem/inner_skull.surf", - "subjects/sample/bem/sample-head.fif", - "subjects/sample/surf/lh.seghead", - # single-layer BEM solution (the 3-layer one is 237 MB, so notebooks - # needing that are excluded instead) - "subjects/sample/bem/sample-5120-bem-sol.fif", - # fsaverage source space, used by the morphing and cluster-stats - # notebooks; it ships inside MNE-sample-data - "subjects/fsaverage/bem/fsaverage-ico-5-src.fif", - "subjects/sample/surf/rh.pial", - "subjects/sample/surf/lh.pial", - "subjects/sample/surf/rh.white", - "subjects/sample/surf/lh.white", - "subjects/sample/surf/rh.inflated", - "subjects/sample/surf/lh.inflated", - "subjects/sample/surf/rh.curv", - "subjects/sample/surf/lh.curv", - # setup_source_space maps each hemisphere onto its sphere for any - # ico/oct spacing, and _create_surf_spacing reads surf/{hemi}.sphere - # by a path it builds itself (5.6 MB each) - "subjects/sample/surf/lh.sphere", - "subjects/sample/surf/rh.sphere", - "subjects/sample/label/lh.aparc.annot", - "subjects/sample/label/rh.aparc.annot", - # the auditory/visual ROIs; about nine notebooks build these names with - # an f-string, so a scan of the tutorial text never sees them - "MEG/sample/labels/Aud-lh.label", - "MEG/sample/labels/Aud-rh.label", - "MEG/sample/labels/Vis-lh.label", - "MEG/sample/labels/Vis-rh.label", - ] - for req in required_files: - s = _lite_src("MNE-sample-data", req) - d = dst_sample_data / req - if s is not None: - d.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(s, d) - sphinx_logger.info(f"[JupyterLite] Copied: {req}") - else: - sphinx_logger.info(f"[JupyterLite] MISSING: {req}") - - -# Also inject SSVEP and EEGLAB testing datasets for JupyterLite -lite_data_base = lite_extra_base -lite_data_base.mkdir(parents=True, exist_ok=True) - -src_ssvep = mne_data_base / "ssvep-example-data" -dst_ssvep = lite_data_base / "ssvep-example-data" -sphinx_logger.info(f"[JupyterLite] SSVEP data source exists: {src_ssvep.exists()}") -if src_ssvep.exists() and not dst_ssvep.exists(): - shutil.copytree(src_ssvep, dst_ssvep, dirs_exist_ok=True) - sphinx_logger.info("[JupyterLite] Copied ssvep-example-data") - -src_eeglab = mne_data_base / "MNE-testing-data" / "EEGLAB" -dst_eeglab = lite_data_base / "MNE-testing-data" / "EEGLAB" -sphinx_logger.info(f"[JupyterLite] EEGLAB data source exists: {src_eeglab.exists()}") -if src_eeglab.exists() and not dst_eeglab.exists(): - shutil.copytree(src_eeglab, dst_eeglab, dirs_exist_ok=True) - sphinx_logger.info("[JupyterLite] Copied MNE-testing-data/EEGLAB") - -# The head-position and Maxwell-filtering tutorials read one continuous -# movement recording out of the testing dataset. CI already restores it from -# data-cache-testing, so only these two files are copied, not the 1.6 GB set. -testing_files = [ - "SSS/test_move_anon_raw.fif", - "SSS/test_move_anon_raw.pos", -] -for testing_file in testing_files: - s = _lite_src("MNE-testing-data", testing_file) - d = lite_data_base / "MNE-testing-data" / testing_file - if s is not None and not d.exists(): - d.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(s, d) - _mb = s.stat().st_size / 1e6 - sphinx_logger.info(f"[JupyterLite] Copied {testing_file} ({_mb:.1f} MB)") - elif s is None: - sphinx_logger.info(f"[JupyterLite] MISSING {testing_file}") - -# The remaining datasets are each used by one or two notebooks that read only a -# couple of files out of them. CI already downloads all of these in -# tools/circleci_download.sh, so copying is free, but their sizes vary a lot, -# so refuse anything past this limit rather than bloat the artifact. For scale, -# the largest file already served (sample_audvis_raw.fif) is 128 MB. -LITE_MAX_FILE_MB = 150 - - -def _lite_copy(folder, rel_paths): - """Copy selected files of a dataset into the served tree.""" - for rel in rel_paths: - s = _lite_src(folder, rel) - if s is None: - sphinx_logger.info(f"[JupyterLite] MISSING {folder}/{rel}") - continue - size_mb = s.stat().st_size / 1e6 - if size_mb > LITE_MAX_FILE_MB: - sphinx_logger.info( - f"[JupyterLite] SKIPPED {folder}/{rel} ({size_mb:.1f} MB)" - ) - continue - d = lite_data_base / folder / rel - if not d.exists(): - d.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(s, d) - sphinx_logger.info( - f"[JupyterLite] Copied {folder}/{rel} ({size_mb:.1f} MB)" - ) - - -def _lite_copy_tree(folder, rel_dir): - """Copy a directory-shaped recording, leaving a manifest for the browser. - - read_raw_nirx and read_raw_egi are handed a folder rather than a file, so - the setup cell has no way to know what to fetch without a listing. - """ - src = mne_data_base / folder / rel_dir - if not src.is_dir(): - sphinx_logger.info(f"[JupyterLite] MISSING {folder}/{rel_dir}") - return - names, total_mb = [], 0.0 - for f in sorted(src.rglob("*")): - if not f.is_file(): - continue - # zero-byte members (an .mff carries a couple of lock files) do not - # survive the artifact upload, so listing them only yields a 404 - if f.stat().st_size == 0: - continue - size_mb = f.stat().st_size / 1e6 - if size_mb > LITE_MAX_FILE_MB: - sphinx_logger.info( - f"[JupyterLite] SKIPPED {folder}/{rel_dir} ({size_mb:.1f} MB)" - ) - return - names.append(str(f.relative_to(src))) - total_mb += size_mb - dst = lite_data_base / folder / rel_dir - dst.mkdir(parents=True, exist_ok=True) - for name in names: - d = dst / name - d.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src / name, d) - (dst / "_lite_manifest.txt").write_text("\n".join(names)) - sphinx_logger.info( - f"[JupyterLite] Copied {folder}/{rel_dir} " - f"({len(names)} files, {total_mb:.1f} MB)" - ) - - -_lite_copy( - "MNE-misc-data", - [ - "xdf/sub-P001_ses-S004_task-Default_run-001_eeg_a2.xdf", - "movement/simulated_quats.pos", - "movement/simulated_movement_raw.fif", - "movement/simulated_stationary_raw.fif", - "eyetracking/eyelink/px_textpage_ws.asc", - "eyetracking/eyelink/HREF_textpage_ws.asc", - ], -) -_lite_copy( - "MNE-eyelink-data", - [ - "freeviewing/sub-01_task-freeview_eyetrack.asc", - "freeviewing/stim/naturalistic.png", - "eeg-et/sub-01_task-plr_eyetrack.asc", - ], -) -_lite_copy_tree("MNE-eyelink-data", "eeg-et/sub-01_task-plr_eeg.mff") -_lite_copy_tree("MNE-fNIRS-motor-data", "Participant-1") - -# The logging tutorial reads a KIT file that lives inside the package itself, -# under mne/io/kit/tests/. pyproject excludes "/mne/**/tests" from the wheel, so -# it is absent from the browser kernel, so serve it and let the setup cell stage -# it back into the path the tutorial builds. -_kit_src = Path(mne.__file__).parent / "io" / "kit" / "tests" / "data" / "test.sqd" -_kit_dst = lite_data_base / "MNE-kit-testdata" / "test.sqd" -if _kit_src.exists(): - if not _kit_dst.exists(): - _kit_dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(_kit_src, _kit_dst) - sphinx_logger.info( - f"[JupyterLite] Copied MNE-kit-testdata/test.sqd " - f"({_kit_src.stat().st_size / 1e6:.1f} MB)" - ) -else: - sphinx_logger.info("[JupyterLite] MISSING MNE-kit-testdata/test.sqd") - -_lite_copy("MNE-phantom-kernel-data", ["phantom_32_100nam_raw.fif"]) -_lite_copy("MNE-multimodal-data", ["multimodal_raw.fif"]) -_lite_copy("MNE-refmeg-noise-data", ["sample_reference_MEG_noise-raw.fif"]) - -# somato is deliberately not served: its raw alone is 344 MB and the six -# notebooks that read it are on the exclude list instead. - -# Inject the single needed file(s) from extra datasets used by the Epochs and -# decoding examples. Sizes are all within what we already serve -# (sample_audvis_raw.fif is 128.5 MB): kiloword 28.7 MB, erp_core 123.6 MB, -# mtrf speech_data.mat 17.2 MB, eegbci 3x2.6 MB. The CI "Ensure ... data" step -# downloads them so the sources exist here. -for _folder, _ds_files in ( - ("MNE-kiloword-data", ["kword_metadata-epo.fif"]), - ("MNE-ERP-CORE-data", ["ERP-CORE_Subject-001_Task-Flankers_eeg.fif"]), - ("mTRF_1.5", ["speech_data.mat"]), - ( - "MNE-eegbci-data", - # exactly the runs tools/circleci_download.sh fetches: subject 1 runs - # 3/6/10/14 and run 3 for subjects 2-4. Notebooks wanting run 1 or 2 are - # excluded instead, since that data never reaches the CI box. - [ - "files/eegmmidb/1.0.0/S001/S001R03.edf", - "files/eegmmidb/1.0.0/S001/S001R06.edf", - "files/eegmmidb/1.0.0/S001/S001R10.edf", - "files/eegmmidb/1.0.0/S001/S001R14.edf", - "files/eegmmidb/1.0.0/S002/S002R03.edf", - "files/eegmmidb/1.0.0/S003/S003R03.edf", - "files/eegmmidb/1.0.0/S004/S004R03.edf", - ], - ), -): - _dst_ds = lite_data_base / _folder - for _ds_file in _ds_files: - s = _lite_src(_folder, _ds_file) - d = _dst_ds / _ds_file - if s is not None: - d.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(s, d) - sphinx_logger.info(f"[JupyterLite] Copied: {_folder}/{_ds_file}") - else: - sphinx_logger.info(f"[JupyterLite] MISSING: {_folder}/{_ds_file}") - - -# Provide the development MNE wheel so JupyterLite installs the current version -# rather than the older release from PyPI. ``doc/sphinxext/build_lite_wheel.py`` -# builds it into ``doc/pypi``, where the jupyterlite-pyodide-kernel PipliteAddon -# discovers and indexes it. Running that script before the docs build (in CI or -# locally) means Sphinx reuses the wheel instead of rebuilding it on every -# invocation; if none is present we build it here, so the docs build never -# depends on the pre-step having run. -_lite_wheels = find_wheels() or build_wheel() -_lite_wheel_names = ", ".join(str(_wheel) for _wheel in _lite_wheels) -sphinx_logger.info( - f"[JupyterLite] MNE wheel for the browser kernel: {_lite_wheel_names}" -) +# The JupyterLite site, and the data it serves (about 1 GB), only belong in a +# full build: `make html` turns this on, pattern and noplot builds leave it off. +build_jupyterlite = os.getenv("MNE_DOC_BUILD_JUPYTERLITE", "0") == "1" +if build_jupyterlite: + from jupyterlite_data import stage_lite_data # noqa: E402 + + extensions.append("jupyterlite_sphinx") + jupyterlite_contents = ["jupyterlite_contents"] + jupyterlite_bind_ipynb_suffix = False + # served at the docs root (/mne_data/...) through html_extra_path below + stage_lite_data(curpath / "lite_extra" / "mne_data") sphinx_gallery_conf = { - "jupyterlite": { - "use_jupyter_lab": True, - "jupyterlite_contents": "jupyterlite_contents", - # named rather than passed: sphinx_gallery_conf has to stay - # JSON-serializable (see the is_serializable assert below), so - # sphinx-gallery imports this dotted path itself - "notebook_modification_function": ( - "jupyterlite_cell_notes.note_unrunnable_cells" - ), - }, "doc_module": ("mne",), "reference_url": dict(mne=None), "examples_dirs": examples_dirs, @@ -927,6 +601,16 @@ def _lite_copy_tree(folder, rel_dir): "copyfile_regex": r".*index\.rst", # allow custom index.rst files "parallel": sphinx_gallery_parallel, } +if build_jupyterlite: + sphinx_gallery_conf["jupyterlite"] = { + "use_jupyter_lab": True, + "jupyterlite_contents": "jupyterlite_contents", + # a dotted path rather than the function: sphinx_gallery_conf has to + # stay JSON-serializable, so sphinx-gallery imports it itself + "notebook_modification_function": ( + "jupyterlite_cell_notes.note_unrunnable_cells" + ), + } assert is_serializable(sphinx_gallery_conf) # --------------------------------------------------------------------------- @@ -975,17 +659,16 @@ def _lite_copy_tree(folder, rel_dir): # make_field_map(upsampling=2) subdivides the helmet mesh through VTK, and # plot_field needs the interactive viewer that the browser renderer skips "examples/visualization/mne_helmet.py", - # Tier 4: mne.viz.Brain. The browser renderer draws static meshes; Brain - # additionally wants dock widgets, a toolbar and a time slider, so these - # are blocked on the interactive layer rather than on data. + # Tier 4: mne.viz.Brain features the browser renderer lacks. Brain itself + # draws (static, single time point), and the fNIRS tutorials and + # 50_background_freesurfer_mne run in full, but these lean on + # add_annotation's hover callback, brain.screenshot, legends, silhouettes + # or the flatmap, so most of their cells fail. "examples/visualization/brain.py", "examples/visualization/parcellation.py", "tutorials/clinical/20_seeg.py", "tutorials/forward/10_background_freesurfer.py", - "tutorials/forward/50_background_freesurfer_mne.py", "tutorials/inverse/60_visualize_stc.py", - "tutorials/io/30_reading_fnirs_data.py", - "tutorials/preprocessing/70_fnirs_processing.py", # Tier 5: one-off blockers with no browser path # plot_field needs the interactive viewer "tutorials/evoked/20_visualize_evoked.py", @@ -1375,10 +1058,9 @@ def fix_sklearn_inherited_docstrings(app, what, name, obj, options, lines): "documentation.html", "getting_started.html", "install_mne_python.html", - # Serve the pre-bundled JupyterLite sample data at the docs root - # (e.g. /mne_data/...). The lite setup cell fetches it over HTTP. - "lite_extra", ] +if build_jupyterlite: # the served data, at /mne_data/... + html_extra_path.append("lite_extra") # Custom sidebar templates, maps document names to template names. html_sidebars = { diff --git a/doc/documentation/datasets.rst b/doc/documentation/datasets.rst index 5a5e49e1252..926f680a1ae 100644 --- a/doc/documentation/datasets.rst +++ b/doc/documentation/datasets.rst @@ -566,9 +566,9 @@ dataset fetchers above. build (such as ``antio``), or a dataset too large to serve to a browser (brainstorm, spm_face, opm, hf_sef, and similar). - 3D is also limited: source estimates are rendered with vtk.js, but the - coregistration and sensor-plotting views that rely on the full VTK stack are - not available. + 3D is also limited: sensor alignment and source estimates are drawn with + vtk.js as static scenes, so the coregistration GUI and the interactive + viewers (time slider, hover, screenshots) are not available. References ========== diff --git a/doc/sphinxext/jupyterlite_data.py b/doc/sphinxext/jupyterlite_data.py new file mode 100644 index 00000000000..4aedbc4e1f6 --- /dev/null +++ b/doc/sphinxext/jupyterlite_data.py @@ -0,0 +1,211 @@ +"""Stage the data the JupyterLite notebooks read, and the MNE wheel they install. + +The setup cell fetches files over HTTP into the Pyodide kernel, since the +``/drive`` filesystem bridge needs cross-origin-isolation headers that static +hosts do not send. So ``conf.py`` serves a subset of the datasets at the docs +root (``/mne_data/...``, via ``html_extra_path``), copied here from +``~/mne_data`` and the curated ``lite_data`` archive, which extracts the same +files under their original dataset folders. + +Every notebook that gets an "Open in JupyterLite" badge has to find what it +reads below; ``JUPYTERLITE_EXCLUDE`` in ``conf.py`` lists the pages that do not. +""" + +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + +import os +import shutil +from pathlib import Path + +from build_lite_wheel import build_wheel, find_wheels +from mne_doc_utils import sphinx_logger + +import mne + +MNE_DATA = Path(os.path.expanduser("~/mne_data")) +LITE_DATA = MNE_DATA / "MNE-lite-data" +# Refuse anything past this rather than bloat the deploy; the largest file that +# has to be served (sample_audvis_raw.fif) is 128 MB. +MAX_FILE_MB = 150 + +# MNE-sample-data files the badged notebooks read. Anything not here is a 404 +# in the browser, so this list follows the tutorials, not the dataset. +SAMPLE_FILES = [ + "version.txt", + "MEG/sample/sample_audvis_raw.fif", + "MEG/sample/sample_audvis_filt-0-40_raw.fif", + "MEG/sample/sample_audvis_raw-eve.fif", + "MEG/sample/sample_audvis_filt-0-40_raw-eve.fif", + "MEG/sample/sample_audvis_ecg-proj.fif", + "MEG/sample/sample_audvis-ave.fif", + "MEG/sample/sample_audvis-cov.fif", + "MEG/sample/sample_audvis-meg-eeg-oct-6-fwd.fif", + "MEG/sample/sample_audvis-meg-oct-6-meg-inv.fif", + "MEG/sample/sample_audvis-meg-oct-6-fwd.fif", + "MEG/sample/sample_audvis-meg-oct-6-meg-fixed-inv.fif", + "MEG/sample/ernoise_raw.fif", + "MEG/sample/sample_audvis-no-filter-ave.fif", + "MEG/sample/sample_audvis_raw-trans.fif", + "MEG/sample/sample_audvis-shrunk-cov.fif", + "MEG/sample/sample_audvis-meg-lh.stc", + "MEG/sample/sample_audvis-meg-rh.stc", + "MEG/sample/sample_audvis-meg-eeg-lh.stc", + "MEG/sample/sample_audvis-meg-eeg-rh.stc", + "MEG/sample/sample_audvis_ecg-eve.fif", + "SSS/sss_cal_mgh.dat", # the Maxwell-filter calibration pair + "SSS/ct_sparse_mgh.fif", + "subjects/sample/mri/T1.mgz", + "subjects/sample/mri/aseg.mgz", + # read_talxfm builds this path itself; plot_alignment estimates the MRI + # fiducials from it + "subjects/sample/mri/transforms/talairach.xfm", + "subjects/sample/bem/sample-oct-6-src.fif", + # head and skull surfaces for plot_alignment, in the order MNE tries them + # so the browser draws the same one the rendered docs do; there is no + # sample-head-dense.fif, and lh.seghead is the dense fallback. The .surf + # files are symlinks into bem/flash/, which copy2 follows. + "subjects/sample/bem/outer_skin.surf", + "subjects/sample/bem/outer_skull.surf", + "subjects/sample/bem/inner_skull.surf", + "subjects/sample/bem/sample-head.fif", + "subjects/sample/surf/lh.seghead", + # the single-layer BEM solution; the three-layer one is 237 MB, so the + # notebooks needing it are excluded instead + "subjects/sample/bem/sample-5120-bem-sol.fif", + # the fsaverage source space ships inside MNE-sample-data + "subjects/fsaverage/bem/fsaverage-ico-5-src.fif", + "subjects/sample/surf/rh.pial", + "subjects/sample/surf/lh.pial", + "subjects/sample/surf/rh.white", + "subjects/sample/surf/lh.white", + "subjects/sample/surf/rh.inflated", + "subjects/sample/surf/lh.inflated", + "subjects/sample/surf/rh.curv", + "subjects/sample/surf/lh.curv", + # setup_source_space reads surf/{hemi}.sphere by a path it builds itself + "subjects/sample/surf/lh.sphere", + "subjects/sample/surf/rh.sphere", + "subjects/sample/label/lh.aparc.annot", + "subjects/sample/label/rh.aparc.annot", + # the auditory/visual ROIs, whose names about nine notebooks build with an + # f-string, so a scan of the tutorial text never sees them + "MEG/sample/labels/Aud-lh.label", + "MEG/sample/labels/Aud-rh.label", + "MEG/sample/labels/Vis-lh.label", + "MEG/sample/labels/Vis-rh.label", +] + +# (dataset folder, files): each used by one or two notebooks that read only a +# couple of files out of it. tools/circleci_download.sh fetches all of these, +# and the CI "Ensure MNE data for JupyterLite" step adds the last four. +DATASET_FILES = { + "MNE-sample-data": SAMPLE_FILES, + # the head-position and Maxwell-filtering tutorials read one movement + # recording out of the testing dataset (CI restores it from its cache) + "MNE-testing-data": ["SSS/test_move_anon_raw.fif", "SSS/test_move_anon_raw.pos"], + "MNE-misc-data": [ + "xdf/sub-P001_ses-S004_task-Default_run-001_eeg_a2.xdf", + "movement/simulated_quats.pos", + "movement/simulated_movement_raw.fif", + "movement/simulated_stationary_raw.fif", + "eyetracking/eyelink/px_textpage_ws.asc", + "eyetracking/eyelink/HREF_textpage_ws.asc", + ], + "MNE-eyelink-data": [ + "freeviewing/sub-01_task-freeview_eyetrack.asc", + "freeviewing/stim/naturalistic.png", + "eeg-et/sub-01_task-plr_eyetrack.asc", + ], + "MNE-phantom-kernel-data": ["phantom_32_100nam_raw.fif"], + "MNE-multimodal-data": ["multimodal_raw.fif"], + "MNE-refmeg-noise-data": ["sample_reference_MEG_noise-raw.fif"], + "MNE-kiloword-data": ["kword_metadata-epo.fif"], + "MNE-ERP-CORE-data": ["ERP-CORE_Subject-001_Task-Flankers_eeg.fif"], + "mTRF_1.5": ["speech_data.mat"], + # exactly the runs tools/circleci_download.sh fetches (subject 1 runs + # 3/6/10/14, run 3 for subjects 2-4); notebooks wanting runs 1 or 2 are + # excluded instead + "MNE-eegbci-data": [ + f"files/eegmmidb/1.0.0/S{s:03d}/S{s:03d}R{r:02d}.edf" + for s, r in ((1, 3), (1, 6), (1, 10), (1, 14), (2, 3), (3, 3), (4, 3)) + ], +} +# whole folders, for the readers that are handed a directory rather than a +# file (read_raw_egi, read_raw_nirx); a manifest is left for the setup cell +DATASET_TREES = [ + ("MNE-eyelink-data", "eeg-et/sub-01_task-plr_eeg.mff"), + ("MNE-fNIRS-motor-data", "Participant-1"), +] +# somato is deliberately not served: its raw alone is 344 MB, and the six +# notebooks that read it are excluded instead + + +def _source(folder, rel): + """Return where a dataset file can be read from, or None if nowhere. + + The real dataset CI restored comes first, then the curated archive, which + only carries the files it was published with. + """ + for root in (MNE_DATA / folder, LITE_DATA / folder): + if (root / rel).exists(): + return root / rel + return None + + +def _copy(src, dst): + """Copy ``src`` to ``dst`` unless a same-sized copy is already there.""" + if dst.exists() and dst.stat().st_size == src.stat().st_size: + return False + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + return True + + +def stage_lite_data(dst_base): + """Copy the served data subset under ``dst_base`` and build the MNE wheel.""" + dst_base = Path(dst_base) + n_copied = n_missing = 0 + for folder, rels in DATASET_FILES.items(): + for rel in rels: + src = _source(folder, rel) + if src is None: + sphinx_logger.info(f"[JupyterLite] MISSING {folder}/{rel}") + n_missing += 1 + elif src.stat().st_size / 1e6 > MAX_FILE_MB: + sphinx_logger.info(f"[JupyterLite] SKIPPED {folder}/{rel} (too big)") + else: + n_copied += _copy(src, dst_base / folder / rel) + for folder, rel_dir in DATASET_TREES: + src_dir = MNE_DATA / folder / rel_dir + if not src_dir.is_dir(): + sphinx_logger.info(f"[JupyterLite] MISSING {folder}/{rel_dir}") + n_missing += 1 + continue + # zero-byte members (an .mff carries a couple of lock files) do not + # survive the artifact upload, so listing them would only yield 404s + names = [ + str(f.relative_to(src_dir)) + for f in sorted(src_dir.rglob("*")) + if f.is_file() and 0 < f.stat().st_size / 1e6 <= MAX_FILE_MB + ] + for name in names: + n_copied += _copy(src_dir / name, dst_base / folder / rel_dir / name) + (dst_base / folder / rel_dir / "_lite_manifest.txt").write_text( + "\n".join(names) + ) + # The logging tutorial reads a KIT file that lives inside the package under + # mne/io/kit/tests/, which the wheel leaves out, so serve it and let the + # setup cell stage it back into the path the tutorial builds. + kit = Path(mne.__file__).parent / "io" / "kit" / "tests" / "data" / "test.sqd" + n_copied += _copy(kit, dst_base / "MNE-kit-testdata" / "test.sqd") + sphinx_logger.info( + f"[JupyterLite] Served data: {n_copied} files copied, {n_missing} missing" + ) + # the development wheel, so the browser installs this MNE rather than the + # PyPI release: doc/sphinxext/build_lite_wheel.py puts it in doc/pypi, where + # the piplite addon indexes it; `make html` runs that first, so this is + # only a fallback for a bare sphinx-build + wheels = find_wheels() or build_wheel() + sphinx_logger.info(f"[JupyterLite] MNE wheel for the browser kernel: {wheels}") From 94eb69093ac2a2ab7fcc659c378baaefc30dd9a6 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 11 Sep 2026 12:52:53 -0400 Subject: [PATCH 08/12] Do not serve the .mff video or datasets whose only pages are excluded --- doc/sphinxext/jupyterlite_data.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/sphinxext/jupyterlite_data.py b/doc/sphinxext/jupyterlite_data.py index 4aedbc4e1f6..dc48b693e14 100644 --- a/doc/sphinxext/jupyterlite_data.py +++ b/doc/sphinxext/jupyterlite_data.py @@ -118,8 +118,6 @@ "freeviewing/stim/naturalistic.png", "eeg-et/sub-01_task-plr_eyetrack.asc", ], - "MNE-phantom-kernel-data": ["phantom_32_100nam_raw.fif"], - "MNE-multimodal-data": ["multimodal_raw.fif"], "MNE-refmeg-noise-data": ["sample_reference_MEG_noise-raw.fif"], "MNE-kiloword-data": ["kword_metadata-epo.fif"], "MNE-ERP-CORE-data": ["ERP-CORE_Subject-001_Task-Flankers_eeg.fif"], @@ -184,11 +182,14 @@ def stage_lite_data(dst_base): n_missing += 1 continue # zero-byte members (an .mff carries a couple of lock files) do not - # survive the artifact upload, so listing them would only yield 404s + # survive the artifact upload, so listing them would only yield 404s; + # the .mov an .mff can carry is a video no reader opens (35 MB) names = [ str(f.relative_to(src_dir)) for f in sorted(src_dir.rglob("*")) - if f.is_file() and 0 < f.stat().st_size / 1e6 <= MAX_FILE_MB + if f.is_file() + and f.suffix != ".mov" + and 0 < f.stat().st_size / 1e6 <= MAX_FILE_MB ] for name in names: n_copied += _copy(src_dir / name, dst_base / folder / rel_dir / name) From 66507763f0f745c65d625fe9361243a3506bb7b9 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 11 Sep 2026 14:18:01 -0400 Subject: [PATCH 09/12] Stop serving the four largest single-page datasets; serve ssvep again --- .circleci/config.yml | 11 ---- doc/changes/dev/14157.apichange.rst | 1 + doc/conf.py | 27 +++++++--- doc/documentation/datasets.rst | 14 ++---- doc/sphinxext/_lite_setup_cell.py | 11 ++-- doc/sphinxext/jupyterlite_cell_notes.py | 26 +++++----- doc/sphinxext/jupyterlite_data.py | 67 +++++++------------------ mne/datasets/config.py | 3 +- mne/datasets/lite_data/lite_data.py | 22 ++++---- mne/datasets/tests/test_datasets.py | 6 ++- 10 files changed, 76 insertions(+), 112 deletions(-) create mode 100644 doc/changes/dev/14157.apichange.rst diff --git a/.circleci/config.yml b/.circleci/config.yml index 1122d4ee81a..1ac8e04562e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -276,17 +276,6 @@ jobs: no_output_timeout: 10m command: | ./tools/circleci_download.sh - # Ensure the data the JupyterLite notebooks need is on disk so conf.py can - # copy the required subset for the build. lite_data fetches only the - # curated files the browser notebooks use (from the MNE-lite-data OSF - # project) instead of the full sample/kiloword/erp_core/mtrf/eegbci - # datasets, which slims the build. The curated files share hashes with the - # full datasets, so on a full build that already downloaded them nothing - # extra is fetched. - - run: - name: Ensure MNE data for JupyterLite - command: | - python -c "import mne; mne.datasets.lite_data.data_path(update_path=True)" - run: name: Verify build type command: | diff --git a/doc/changes/dev/14157.apichange.rst b/doc/changes/dev/14157.apichange.rst new file mode 100644 index 00000000000..5acdb31f86a --- /dev/null +++ b/doc/changes/dev/14157.apichange.rst @@ -0,0 +1 @@ +:func:`mne.datasets.lite_data.data_path` is deprecated and will be removed in MNE 1.15; the documentation build now serves the browser notebooks from the regular datasets, by `Eric Larson`_. diff --git a/doc/conf.py b/doc/conf.py index 5ade91cf9c4..9cb23dc0be2 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -699,19 +699,32 @@ "tutorials/inverse/70_eeg_mri_coords.py", # mne_bids is not installable in the browser kernel "tutorials/inverse/95_phantom_KIT.py", - # Tier 7: somato. Serving it costs 404 MB (the raw alone is 344 MB) on - # every docs deploy, which is more than these six pages are worth; the - # dataset is not copied at all. Restoring them means putting the somato - # block back in the copy step above. + # Tier 7: served size. Every file below is copied into every docs deploy, + # so a dataset that only one or two pages read has to earn its place; + # these did not (sizes are what the staging step copied). Restoring a page + # means adding what it reads to DATASET_FILES in jupyterlite_data.py. + # somato: 404 MB (the raw alone is 344 MB) for six pages "examples/inverse/dics_epochs.py", "examples/inverse/dics_source_power.py", "examples/inverse/evoked_ers_source_power.py", "examples/inverse/multidict_reweighted_tfmxne.py", "examples/time_frequency/time_frequency_global_field_power.py", "tutorials/time-freq/20_sensors_time_frequency.py", - # Single recordings well past LITE_MAX_FILE_MB, confirmed against the full - # build: 379 MB and 251 MB for one example each, so they are skipped by the - # copy step and the badge would have nothing to load. + # the .mff EEG recording is a 133 MB folder, for one page + "tutorials/preprocessing/90_eyetracking_data.py", + # ERP-CORE: 118 MB for two pages + "examples/preprocessing/epochs_metadata.py", + "tutorials/epochs/40_autogenerate_metadata.py", + # refmeg_noise: 93 MB for one page + "examples/preprocessing/find_ref_artifacts.py", + # testing: the SSS movement recording (38 MB) and EEGLAB folder (34 MB), + # two pages each + "tutorials/preprocessing/59_head_positions.py", + "tutorials/preprocessing/60_maxwell_filtering_sss.py", + "tutorials/intro/20_events_from_raw.py", + "examples/visualization/roi_erpimage_by_rt.py", + # single recordings well past MAX_FILE_MB, 379 MB and 251 MB, so the + # staging step skips them and the badge would have nothing to load "examples/datasets/kernel_phantom.py", "examples/io/elekta_epochs.py", # These want EEGBCI runs 1 and 2, which tools/circleci_download.sh never diff --git a/doc/documentation/datasets.rst b/doc/documentation/datasets.rst index 926f680a1ae..1b503b0a4c3 100644 --- a/doc/documentation/datasets.rst +++ b/doc/documentation/datasets.rst @@ -547,17 +547,9 @@ JupyterLite data ================ :func:`mne.datasets.lite_data.data_path` -A small curated archive holding the data files needed to run the tutorials and -examples in the browser, taken from the ``sample``, ``kiloword``, ``erp_core``, -``mtrf`` and ``eegbci`` datasets. The files are unchanged and keep the same -checksums as the full datasets. It extracts to ``MNE-lite-data/``, keeping each -file under its original dataset folder (``MNE-sample-data/``, -``MNE-kiloword-data/``, ...). -The ``somato`` dataset is not included, so the somatosensory tutorials and -examples are not available in the browser. - -This exists for the documentation build; for analysis, use the individual -dataset fetchers above. +Deprecated, and will be removed in MNE 1.15. This curated archive existed for +the documentation build, which now serves the browser notebooks from the +regular datasets above; use those fetchers instead. .. note:: Not every tutorial and example can run in the browser, so the "Open in JupyterLite" badge is only shown on the pages that work there. A page diff --git a/doc/sphinxext/_lite_setup_cell.py b/doc/sphinxext/_lite_setup_cell.py index 32d5daa2219..ec9be131c4d 100644 --- a/doc/sphinxext/_lite_setup_cell.py +++ b/doc/sphinxext/_lite_setup_cell.py @@ -226,16 +226,13 @@ def _data_path(*args, **kwargs): for _ds, _folder, _probe in ( ("sample", "MNE-sample-data", None), - ("testing", "MNE-testing-data", None), ("ssvep", "ssvep-example-data", None), ("misc", "MNE-misc-data", None), ("eyelink", "MNE-eyelink-data", None), ("fnirs_motor", "MNE-fNIRS-motor-data", None), - ("refmeg_noise", "MNE-refmeg-noise-data", None), ("phantom_kernel", "MNE-phantom-kernel-data", None), ("multimodal", "MNE-multimodal-data", None), ("kiloword", "MNE-kiloword-data", "kword_metadata-epo.fif"), - ("erp_core", "MNE-ERP-CORE-data", "ERP-CORE_Subject-001_Task-Flankers_eeg.fif"), ("mtrf", "mTRF_1.5", "speech_data.mat"), ): getattr(mne.datasets, _ds).data_path = _lite_dataset_path(_folder, _probe) @@ -284,10 +281,12 @@ def _lite_check_fname( mne_check._check_fname = _lite_check_fname _lite_rebind("_check_fname", _orig_check_fname, _lite_check_fname) -import matplotlib.pyplot as plt # imread: the eyetracking heatmap's stimulus +import matplotlib.pyplot as plt # the eyetracking heatmap reads its stimulus +import nibabel # a few tutorials load an MRI themselves for _module, _name, _siblings in ( (plt, "imread", None), + (nibabel, "load", None), (mne.io, "read_raw_eeglab", lambda rel: [rel.removesuffix(".set") + ".fdt"]), ( mne.io, @@ -308,9 +307,7 @@ def _lite_check_fname( _lite_wrap_reader(pyxdf, "load_xdf") except Exception: pass -# folders rather than files -mne.io.read_raw_nirx = _lite_dir_reader(mne.io.read_raw_nirx) -mne.io.read_raw_egi = _lite_dir_reader(mne.io.read_raw_egi) +mne.io.read_raw_nirx = _lite_dir_reader(mne.io.read_raw_nirx) # a folder # Filesystem probes: fetch the candidates first, in the order MNE tries them, # then let it choose as it normally would. The viz modules bind these names at diff --git a/doc/sphinxext/jupyterlite_cell_notes.py b/doc/sphinxext/jupyterlite_cell_notes.py index 8897185d0ae..17d97f2ba1e 100644 --- a/doc/sphinxext/jupyterlite_cell_notes.py +++ b/doc/sphinxext/jupyterlite_cell_notes.py @@ -44,19 +44,19 @@ ( "forward/20_source_alignment.ipynb", "mne.gui.coregistration", - "**This cell does not run in the browser.**\n" - "\n" - "`mne.gui.coregistration` sets the fiducials by clicking on the scalp\n" - "surface, and the vtk.js renderer used here draws scenes without a\n" - "picker, so there is nothing for those clicks to hit. Run it from a\n" - "local MNE install instead:\n" - "\n" - "```python\n" - 'mne.gui.coregistration(subject="sample", subjects_dir=subjects_dir)\n' - "```\n" - "\n" - "The video above walks through the same steps, and the rest of this\n" - "notebook runs normally.\n", + """**This cell does not run in the browser.** + +`mne.gui.coregistration` sets the fiducials by clicking on the scalp surface, +and the vtk.js renderer used here draws scenes without a picker, so there is +nothing for those clicks to hit. Run it from a local MNE install instead: + +```python +mne.gui.coregistration(subject="sample", subjects_dir=subjects_dir) +``` + +The video above walks through the same steps, and the rest of this notebook +runs normally. +""", ), ) diff --git a/doc/sphinxext/jupyterlite_data.py b/doc/sphinxext/jupyterlite_data.py index dc48b693e14..829380b4d3d 100644 --- a/doc/sphinxext/jupyterlite_data.py +++ b/doc/sphinxext/jupyterlite_data.py @@ -1,14 +1,9 @@ """Stage the data the JupyterLite notebooks read, and the MNE wheel they install. -The setup cell fetches files over HTTP into the Pyodide kernel, since the -``/drive`` filesystem bridge needs cross-origin-isolation headers that static -hosts do not send. So ``conf.py`` serves a subset of the datasets at the docs -root (``/mne_data/...``, via ``html_extra_path``), copied here from -``~/mne_data`` and the curated ``lite_data`` archive, which extracts the same -files under their original dataset folders. - -Every notebook that gets an "Open in JupyterLite" badge has to find what it -reads below; ``JUPYTERLITE_EXCLUDE`` in ``conf.py`` lists the pages that do not. +The setup cell fetches files over HTTP, so ``conf.py`` serves this subset of +the datasets at the docs root (``/mne_data/...``), copied from ``~/mne_data``. +A badged notebook has to find everything it reads here; ``JUPYTERLITE_EXCLUDE`` +in ``conf.py`` lists the ones that do not. """ # Authors: The MNE-Python contributors. @@ -25,7 +20,6 @@ import mne MNE_DATA = Path(os.path.expanduser("~/mne_data")) -LITE_DATA = MNE_DATA / "MNE-lite-data" # Refuse anything past this rather than bloat the deploy; the largest file that # has to be served (sample_audvis_raw.fif) is 128 MB. MAX_FILE_MB = 150 @@ -54,8 +48,6 @@ "MEG/sample/sample_audvis-meg-eeg-lh.stc", "MEG/sample/sample_audvis-meg-eeg-rh.stc", "MEG/sample/sample_audvis_ecg-eve.fif", - "SSS/sss_cal_mgh.dat", # the Maxwell-filter calibration pair - "SSS/ct_sparse_mgh.fif", "subjects/sample/mri/T1.mgz", "subjects/sample/mri/aseg.mgz", # read_talxfm builds this path itself; plot_alignment estimates the MRI @@ -98,13 +90,13 @@ ] # (dataset folder, files): each used by one or two notebooks that read only a -# couple of files out of it. tools/circleci_download.sh fetches all of these, -# and the CI "Ensure MNE data for JupyterLite" step adds the last four. +# couple of files out of it. A full docs build downloads all of these datasets. DATASET_FILES = { "MNE-sample-data": SAMPLE_FILES, - # the head-position and Maxwell-filtering tutorials read one movement - # recording out of the testing dataset (CI restores it from its cache) - "MNE-testing-data": ["SSS/test_move_anon_raw.fif", "SSS/test_move_anon_raw.pos"], + "ssvep-example-data": [ + f"sub-02/ses-01/eeg/sub-02_ses-01_task-ssvep_eeg{s}" + for s in (".vhdr", ".eeg", ".vmrk") + ], "MNE-misc-data": [ "xdf/sub-P001_ses-S004_task-Default_run-001_eeg_a2.xdf", "movement/simulated_quats.pos", @@ -116,11 +108,8 @@ "MNE-eyelink-data": [ "freeviewing/sub-01_task-freeview_eyetrack.asc", "freeviewing/stim/naturalistic.png", - "eeg-et/sub-01_task-plr_eyetrack.asc", ], - "MNE-refmeg-noise-data": ["sample_reference_MEG_noise-raw.fif"], "MNE-kiloword-data": ["kword_metadata-epo.fif"], - "MNE-ERP-CORE-data": ["ERP-CORE_Subject-001_Task-Flankers_eeg.fif"], "mTRF_1.5": ["speech_data.mat"], # exactly the runs tools/circleci_download.sh fetches (subject 1 runs # 3/6/10/14, run 3 for subjects 2-4); notebooks wanting runs 1 or 2 are @@ -130,26 +119,11 @@ for s, r in ((1, 3), (1, 6), (1, 10), (1, 14), (2, 3), (3, 3), (4, 3)) ], } -# whole folders, for the readers that are handed a directory rather than a -# file (read_raw_egi, read_raw_nirx); a manifest is left for the setup cell -DATASET_TREES = [ - ("MNE-eyelink-data", "eeg-et/sub-01_task-plr_eeg.mff"), - ("MNE-fNIRS-motor-data", "Participant-1"), -] -# somato is deliberately not served: its raw alone is 344 MB, and the six -# notebooks that read it are excluded instead - - -def _source(folder, rel): - """Return where a dataset file can be read from, or None if nowhere. - - The real dataset CI restored comes first, then the curated archive, which - only carries the files it was published with. - """ - for root in (MNE_DATA / folder, LITE_DATA / folder): - if (root / rel).exists(): - return root / rel - return None +# whole folders, for readers handed a directory rather than a file +# (read_raw_nirx); a manifest is left for the setup cell +DATASET_TREES = [("MNE-fNIRS-motor-data", "Participant-1")] +# somato, ERP-CORE, refmeg_noise and the rest of the testing and eyelink +# datasets are deliberately not served; see the size tiers in conf.py def _copy(src, dst): @@ -167,8 +141,8 @@ def stage_lite_data(dst_base): n_copied = n_missing = 0 for folder, rels in DATASET_FILES.items(): for rel in rels: - src = _source(folder, rel) - if src is None: + src = MNE_DATA / folder / rel + if not src.exists(): sphinx_logger.info(f"[JupyterLite] MISSING {folder}/{rel}") n_missing += 1 elif src.stat().st_size / 1e6 > MAX_FILE_MB: @@ -181,15 +155,12 @@ def stage_lite_data(dst_base): sphinx_logger.info(f"[JupyterLite] MISSING {folder}/{rel_dir}") n_missing += 1 continue - # zero-byte members (an .mff carries a couple of lock files) do not - # survive the artifact upload, so listing them would only yield 404s; - # the .mov an .mff can carry is a video no reader opens (35 MB) + # zero-byte members do not survive the artifact upload, so listing + # them would only yield 404s names = [ str(f.relative_to(src_dir)) for f in sorted(src_dir.rglob("*")) - if f.is_file() - and f.suffix != ".mov" - and 0 < f.stat().st_size / 1e6 <= MAX_FILE_MB + if f.is_file() and 0 < f.stat().st_size / 1e6 <= MAX_FILE_MB ] for name in names: n_copied += _copy(src_dir / name, dst_base / folder / rel_dir / name) diff --git a/mne/datasets/config.py b/mne/datasets/config.py index 293c29bcbb3..95cd2738987 100644 --- a/mne/datasets/config.py +++ b/mne/datasets/config.py @@ -209,8 +209,7 @@ config_key="MNE_DATASETS_SAMPLE_PATH", ) -# Curated subset of sample (plus a few files from kiloword/erp_core/mtrf/eegbci) -# used by the JupyterLite browser docs; see mne/datasets/lite_data/. +# TODO VERSION: remove in 1.15 with the deprecated mne.datasets.lite_data MNE_DATASETS["lite_data"] = dict( archive_name="MNE-lite-data.tar.gz", hash="md5:5f9c4fffed32e79bc2bc2061bf22ce99", diff --git a/mne/datasets/lite_data/lite_data.py b/mne/datasets/lite_data/lite_data.py index 7e52909bafc..53e8e3fa611 100644 --- a/mne/datasets/lite_data/lite_data.py +++ b/mne/datasets/lite_data/lite_data.py @@ -2,19 +2,13 @@ # License: BSD-3-Clause # Copyright the MNE-Python contributors. -"""Curated data subset used by the JupyterLite browser documentation. - -``lite_data`` holds the data files needed to run the tutorials and examples in -the browser, taken from ``sample``, ``kiloword``, ``erp_core``, ``mtrf`` and -``eegbci``. The files are unchanged and keep the same checksums as the full -datasets. It extracts to ``MNE-lite-data/`` with each file under its original -dataset folder (``MNE-sample-data/``, ``MNE-kiloword-data/``, ...), so paths -match. -The ``somato`` dataset is not included, so the somatosensory tutorials and -examples do not run in the browser. +"""Curated data subset once used by the JupyterLite browser documentation. + +The documentation build now serves the browser notebooks from the regular +datasets, so this archive is no longer needed. """ -from ...utils import verbose +from ...utils import deprecated, verbose from ..utils import _data_path_doc, _download_mne_dataset, _get_version, _version_doc @@ -35,6 +29,11 @@ def data_path( data_path.__doc__ = _data_path_doc.format( name="lite_data", conf="MNE_DATASETS_LITE_DATA_PATH" ) +_DEPRECATED = ( + "The documentation build no longer uses the lite_data archive, so it will be " + "removed in MNE 1.15; use the individual dataset fetchers instead" +) +data_path = deprecated(_DEPRECATED)(data_path) def get_version(): # noqa: D103 @@ -42,3 +41,4 @@ def get_version(): # noqa: D103 get_version.__doc__ = _version_doc.format(name="lite_data") +get_version = deprecated(_DEPRECATED)(get_version) diff --git a/mne/datasets/tests/test_datasets.py b/mne/datasets/tests/test_datasets.py index db447b13006..3ad21a5bffb 100644 --- a/mne/datasets/tests/test_datasets.py +++ b/mne/datasets/tests/test_datasets.py @@ -358,5 +358,7 @@ def test_lite_data(): assert cfg["hash"].startswith("md5:") assert cfg["url"].startswith("https://osf.io/") assert cfg["config_key"] == "MNE_DATASETS_LITE_DATA_PATH" - assert callable(lite_data.data_path) - assert callable(lite_data.get_version) + with pytest.warns(FutureWarning, match="removed in MNE 1.15"): + lite_data.data_path(download=False) + with pytest.warns(FutureWarning, match="removed in MNE 1.15"): + lite_data.get_version() From 1d36011413d5d3f827362e9d4808741f378933d8 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 11 Sep 2026 14:57:17 -0400 Subject: [PATCH 10/12] TST: Try [circle full] From d3ed77de6dc11fa27ecac419c392915eda61b7db Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 11 Sep 2026 16:15:57 -0400 Subject: [PATCH 11/12] TST: Again [circle full] From 98b3406fbbea5584eb54f8cfc41eb56d559402a2 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 11 Sep 2026 17:42:58 -0400 Subject: [PATCH 12/12] TST: Again [circle full]