diff --git a/.gitignore b/.gitignore index 7d451f18385..b5deeffcb22 100644 --- a/.gitignore +++ b/.gitignore @@ -104,6 +104,12 @@ venv/ .hypothesis/ .ruff_cache/ .ipynb_checkpoints/ + +# generated by the JupyterLite docs build (doc/conf.py) +doc/jupyterlite_contents/* +!doc/jupyterlite_contents/.gitkeep +doc/lite_extra/ + /.claude/ uv.lock diff --git a/doc/Makefile b/doc/Makefile index ee8dfefa96c..6fa28f2f441 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -34,26 +34,26 @@ jupyterlite_wheel: @python sphinxext/build_lite_wheel.py html: jupyterlite_wheel - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) _build/html + MNE_DOC_BUILD_JUPYTERLITE=1 $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) _build/html @echo @echo "Build finished. The HTML pages are in _build/html." html-memory: jupyterlite_wheel - $(MPROF) -b html $(ALLSPHINXOPTS) _build/html + MNE_DOC_BUILD_JUPYTERLITE=1 $(MPROF) -b html $(ALLSPHINXOPTS) _build/html @echo @echo "Build finished. The HTML pages are in _build/html." -html-pattern: jupyterlite_wheel +html-pattern: $(SPHINXBUILD) -D sphinx_gallery_conf.filename_pattern=$(PATTERN) -D sphinx_gallery_conf.run_stale_examples=True -b html $(ALLSPHINXOPTS) _build/html @echo @echo "Build finished. The HTML pages are in _build/html" -html-pattern-memory: jupyterlite_wheel +html-pattern-memory: $(MPROF) -D sphinx_gallery_conf.filename_pattern=$(PATTERN) -D sphinx_gallery_conf.run_stale_examples=True -b html $(ALLSPHINXOPTS) _build/html @echo @echo "Build finished. The HTML pages are in _build/html" -html-noplot: jupyterlite_wheel +html-noplot: $(SPHINXBUILD) -D plot_gallery=0 -b html $(ALLSPHINXOPTS) _build/html @echo @echo "Build finished. The HTML pages are in _build/html." @@ -68,7 +68,7 @@ html_dev-noplot: html-noplot html_dev-front: html-front linkcheck: - @$(SPHINXBUILD) -b linkcheck -D nitpicky=0 -q -D plot_gallery=0 -D exclude_patterns="cited.rst,whats_new.rst,configure_git.rst,_includes,changes/dev" -d _build/doctrees . _build/linkcheck + @$(SPHINXBUILD) -b linkcheck -D nitpicky=0 -q -D plot_gallery=0 -D exclude_patterns="cited.rst,whats_new.rst,configure_git.rst,_includes,changes/dev,jupyterlite_contents,lite_extra,pypi" -d _build/doctrees . _build/linkcheck doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) _build/doctest diff --git a/doc/changes/dev/14157.apichange.rst b/doc/changes/dev/14157.apichange.rst new file mode 100644 index 00000000000..5acdb31f86a --- /dev/null +++ b/doc/changes/dev/14157.apichange.rst @@ -0,0 +1 @@ +:func:`mne.datasets.lite_data.data_path` is deprecated and will be removed in MNE 1.15; the documentation build now serves the browser notebooks from the regular datasets, by `Eric Larson`_. diff --git a/doc/changes/dev/14157.newfeature.rst b/doc/changes/dev/14157.newfeature.rst new file mode 100644 index 00000000000..879bbec54a8 --- /dev/null +++ b/doc/changes/dev/14157.newfeature.rst @@ -0,0 +1 @@ +Add an "Open in JupyterLite" badge to the tutorials and examples that can run in the browser, by `Natneal B`_. diff --git a/doc/conf.py b/doc/conf.py index 93c4faf67b6..9cb23dc0be2 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -145,7 +145,13 @@ # This pattern also affects html_static_path and html_extra_path. # NB: changes here should also be made to the linkcheck target in the Makefile -exclude_patterns = ["_includes", "changes/dev"] +exclude_patterns = [ + "_includes", + "changes/dev", + "jupyterlite_contents", + "lite_extra", + "pypi", +] # The suffix of source filenames. source_suffix = ".rst" @@ -482,6 +488,18 @@ compress_images = () sphinx_gallery_parallel = int(os.getenv("MNE_DOC_BUILD_N_JOBS", "1")) +# The JupyterLite site, and the data it serves (about 1 GB), only belong in a +# full build: `make html` turns this on, pattern and noplot builds leave it off. +build_jupyterlite = os.getenv("MNE_DOC_BUILD_JUPYTERLITE", "0") == "1" +if build_jupyterlite: + from jupyterlite_data import stage_lite_data # noqa: E402 + + extensions.append("jupyterlite_sphinx") + jupyterlite_contents = ["jupyterlite_contents"] + jupyterlite_bind_ipynb_suffix = False + # served at the docs root (/mne_data/...) through html_extra_path below + stage_lite_data(curpath / "lite_extra" / "mne_data") + sphinx_gallery_conf = { "doc_module": ("mne",), "reference_url": dict(mne=None), @@ -583,7 +601,171 @@ "copyfile_regex": r".*index\.rst", # allow custom index.rst files "parallel": sphinx_gallery_parallel, } +if build_jupyterlite: + sphinx_gallery_conf["jupyterlite"] = { + "use_jupyter_lab": True, + "jupyterlite_contents": "jupyterlite_contents", + # a dotted path rather than the function: sphinx_gallery_conf has to + # stay JSON-serializable, so sphinx-gallery imports it itself + "notebook_modification_function": ( + "jupyterlite_cell_notes.note_unrunnable_cells" + ), + } assert is_serializable(sphinx_gallery_conf) + +# --------------------------------------------------------------------------- +# Drop the "Open in JupyterLite" launch badge from gallery pages whose +# notebooks cannot run in the browser kernel at all: they need the R runtime +# (rpy2), a compiled package Pyodide does not ship (antio), or multi-GB +# datasets that cannot be bundled/slimmed. sphinx-gallery adds the badge to +# every example unconditionally, so we wrap its badge generator and return an +# empty string for these files. This only removes the badge/link; the +# notebook source is untouched (no in-code guard). Files that merely need data +# bundled, a pure-Python package installed, or pyvista 3D are NOT listed here +# (they are fixable, not impossible). +JUPYTERLITE_EXCLUDE = ( + # Tier 1, impossible: R runtime / compiled package / huge single dataset + "examples/stats/r_interop.py", # rpy2 -> needs the R runtime + "examples/io/read_impedances.py", # antio (compiled, not in Pyodide) + "examples/decoding/decoding_rsa.py", # visual_92_categories ~6 GB + "examples/decoding/decoding_spoc_CMC.py", # fieldtrip_cmc ~700 MB + "examples/decoding/ssd_spatial_filters.py", # fieldtrip_cmc ~700 MB + # Tier 2: multi-GB datasets (brainstorm / spm_face / opm / hf_sef) + "examples/datasets/brainstorm_data.py", + "examples/datasets/hf_sef_data.py", + "examples/datasets/opm_data.py", + "examples/datasets/spm_faces_dataset.py", + "examples/preprocessing/movement_detection.py", + "examples/preprocessing/muscle_detection.py", + "examples/preprocessing/otp.py", + "examples/time_frequency/source_power_spectrum_opm.py", + "examples/visualization/evoked_arrowmap.py", + "examples/visualization/meg_sensors.py", + "tutorials/inverse/80_brainstorm_phantom_elekta.py", + "tutorials/inverse/85_brainstorm_phantom_ctf.py", + "tutorials/io/60_ctf_bst_auditory.py", + "tutorials/preprocessing/80_opm_processing.py", + # Tier 3: several blockers each, none of them worth clearing on its own + # the volume inverse is ~178 MB and volume source estimates are not + # rendered in the browser + "examples/inverse/compute_mne_inverse_volume.py", + # needs aseg.mgz and the mixed source space, and calls src.plot(), which + # is the 3D SourceSpaces view + "examples/inverse/mixed_source_space_inverse.py", + # nilearn.datasets.load_mni152_template() downloads a template at runtime, + # which the browser blocks (CORS); the surrounding try only catches + # TypeError, so the failure is not survivable + "tutorials/inverse/20_dipole_fit.py", + # make_field_map(upsampling=2) subdivides the helmet mesh through VTK, and + # plot_field needs the interactive viewer that the browser renderer skips + "examples/visualization/mne_helmet.py", + # Tier 4: mne.viz.Brain features the browser renderer lacks. Brain itself + # draws (static, single time point), and the fNIRS tutorials and + # 50_background_freesurfer_mne run in full, but these lean on + # add_annotation's hover callback, brain.screenshot, legends, silhouettes + # or the flatmap, so most of their cells fail. + "examples/visualization/brain.py", + "examples/visualization/parcellation.py", + "tutorials/clinical/20_seeg.py", + "tutorials/forward/10_background_freesurfer.py", + "tutorials/inverse/60_visualize_stc.py", + # Tier 5: one-off blockers with no browser path + # plot_field needs the interactive viewer + "tutorials/evoked/20_visualize_evoked.py", + # the three-layer BEM solution alone is 237 MB + "examples/inverse/multi_dipole_model.py", + # openneuro fetches the recording at runtime, which the browser blocks + "examples/preprocessing/esg_rm_heart_artefact_pcaobs.py", + # physionet.org is not CORS-enabled and the dataset is not on the CI box + "tutorials/clinical/60_sleep.py", + # the 4D/BTi phantom dataset is not among the ones CI downloads + "tutorials/inverse/90_phantom_4DBTi.py", + # needs mne_bids as well as the epilepsy_ecog dataset and 3D sensor views + "tutorials/clinical/30_ecog.py", + # Tier 6: fetch_fsaverage. _manifest_check_download only skips the + # download when every one of its ~190 manifest entries is already present, + # so fsaverage cannot be part-bundled, and MNE-sample-data ships no + # fsaverage/bem at all. The volume forward and inverse these two want are + # 187 MB and 360 MB on top of that. + "examples/inverse/morph_volume_stc.py", + "tutorials/inverse/50_beamformer_lcmv.py", + "examples/visualization/montage.py", + # same, plus fetch_infant_template downloads a second template + "tutorials/forward/35_eeg_no_mri.py", + # snapshot_brain_montage needs a real 3D window to read pixels back from + "examples/visualization/3d_to_2d.py", + # the three-layer BEM solution is 237 MB, and T1_electrodes.mgz would pull + # in the misc dataset's MRI as well + "tutorials/inverse/70_eeg_mri_coords.py", + # mne_bids is not installable in the browser kernel + "tutorials/inverse/95_phantom_KIT.py", + # Tier 7: served size. Every file below is copied into every docs deploy, + # so a dataset that only one or two pages read has to earn its place; + # these did not (sizes are what the staging step copied). Restoring a page + # means adding what it reads to DATASET_FILES in jupyterlite_data.py. + # somato: 404 MB (the raw alone is 344 MB) for six pages + "examples/inverse/dics_epochs.py", + "examples/inverse/dics_source_power.py", + "examples/inverse/evoked_ers_source_power.py", + "examples/inverse/multidict_reweighted_tfmxne.py", + "examples/time_frequency/time_frequency_global_field_power.py", + "tutorials/time-freq/20_sensors_time_frequency.py", + # the .mff EEG recording is a 133 MB folder, for one page + "tutorials/preprocessing/90_eyetracking_data.py", + # ERP-CORE: 118 MB for two pages + "examples/preprocessing/epochs_metadata.py", + "tutorials/epochs/40_autogenerate_metadata.py", + # refmeg_noise: 93 MB for one page + "examples/preprocessing/find_ref_artifacts.py", + # testing: the SSS movement recording (38 MB) and EEGLAB folder (34 MB), + # two pages each + "tutorials/preprocessing/59_head_positions.py", + "tutorials/preprocessing/60_maxwell_filtering_sss.py", + "tutorials/intro/20_events_from_raw.py", + "examples/visualization/roi_erpimage_by_rt.py", + # single recordings well past MAX_FILE_MB, 379 MB and 251 MB, so the + # staging step skips them and the badge would have nothing to load + "examples/datasets/kernel_phantom.py", + "examples/io/elekta_epochs.py", + # These want EEGBCI runs 1 and 2, which tools/circleci_download.sh never + # fetches (it takes subject 1 runs 3/6/10/14 and run 3 for subjects 2-4), + # so the data is not on the machine that builds the docs. eeg_bridging + # alone would need run 1 for ten subjects. + "examples/visualization/onionskin.py", + "examples/preprocessing/muscle_ica.py", + "examples/preprocessing/eeg_bridging.py", + # These read a 3D scene back as pixels, and vtk.js cannot hand a + # framebuffer back to Python. Both Report tutorials build their figures by + # screenshotting (Report._itv calls backend._take_3d_screenshot), and + # 70_report additionally round-trips a report through HDF5. + "tutorials/intro/70_report.py", + "tutorials/preprocessing/14_quality_control_report.py", + # 10_publication_figure is about cropping the white margins off + # brain.screenshot(), so without a real screenshot there is no tutorial + # left; browser brain.screenshot() raises rather than return a blank image. + "tutorials/visualization/10_publication_figure.py", + # The whole page drives mne.gui.dipolefit and narrates one GUI window as + # its state evolves. The vtk.js renderer draws without a picker, so there + # is nothing for those clicks to hit; that is also why 20_source_alignment + # carries a cell note for mne.gui.coregistration. Here it is the entire + # tutorial rather than one cell, so it is excluded instead. + "tutorials/inverse/21_interactive_dipole_fit.py", +) + +import sphinx_gallery.gen_rst as _sg_gen_rst # noqa: E402 + +_orig_gen_jupyterlite_rst = _sg_gen_rst.gen_jupyterlite_rst + + +def _lite_badge_filtered(fpath, gallery_conf): + """Return the JupyterLite badge reST, or "" for excluded notebooks.""" + _p = str(fpath).replace(os.sep, "/") + if any(_p.endswith(_ex) for _ex in JUPYTERLITE_EXCLUDE): + return "" + return _orig_gen_jupyterlite_rst(fpath, gallery_conf) + + +_sg_gen_rst.gen_jupyterlite_rst = _lite_badge_filtered # Files were renamed from plot_* with: # find . -type f -name 'plot_*.py' -exec sh -c 'x="{}"; xn=`basename "${x}"`; git mv "$x" `dirname "${x}"`/${xn:5}' \; # noqa @@ -889,6 +1071,8 @@ def fix_sklearn_inherited_docstrings(app, what, name, obj, options, lines): "getting_started.html", "install_mne_python.html", ] +if build_jupyterlite: # the served data, at /mne_data/... + html_extra_path.append("lite_extra") # Custom sidebar templates, maps document names to template names. html_sidebars = { diff --git a/doc/documentation/datasets.rst b/doc/documentation/datasets.rst index 7d942fd7fb3..1b503b0a4c3 100644 --- a/doc/documentation/datasets.rst +++ b/doc/documentation/datasets.rst @@ -547,17 +547,20 @@ JupyterLite data ================ :func:`mne.datasets.lite_data.data_path` -A small curated archive holding the data files needed to run the tutorials and -examples in the browser, taken from the ``sample``, ``kiloword``, ``erp_core``, -``mtrf`` and ``eegbci`` datasets. The files are unchanged and keep the same -checksums as the full datasets. It extracts to ``MNE-lite-data/``, keeping each -file under its original dataset folder (``MNE-sample-data/``, -``MNE-kiloword-data/``, ...). -The ``somato`` dataset is not included, so the somatosensory tutorials and -examples are not available in the browser. - -This exists for the documentation build; for analysis, use the individual -dataset fetchers above. +Deprecated, and will be removed in MNE 1.15. This curated archive existed for +the documentation build, which now serves the browser notebooks from the +regular datasets above; use those fetchers instead. + +.. note:: Not every tutorial and example can run in the browser, so the + "Open in JupyterLite" badge is only shown on the pages that work there. A page + is left without a badge when it needs a non-Python runtime (for example the R + interoperability example, via ``rpy2``), a compiled reader with no WebAssembly + build (such as ``antio``), or a dataset too large to serve to a browser + (brainstorm, spm_face, opm, hf_sef, and similar). + + 3D is also limited: sensor alignment and source estimates are drawn with + vtk.js as static scenes, so the coregistration GUI and the interactive + viewers (time slider, hover, screenshots) are not available. References ========== diff --git a/doc/jupyterlite_contents/.gitkeep b/doc/jupyterlite_contents/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/doc/sphinxext/_lite_setup_cell.py b/doc/sphinxext/_lite_setup_cell.py index 32d5daa2219..ec9be131c4d 100644 --- a/doc/sphinxext/_lite_setup_cell.py +++ b/doc/sphinxext/_lite_setup_cell.py @@ -226,16 +226,13 @@ def _data_path(*args, **kwargs): for _ds, _folder, _probe in ( ("sample", "MNE-sample-data", None), - ("testing", "MNE-testing-data", None), ("ssvep", "ssvep-example-data", None), ("misc", "MNE-misc-data", None), ("eyelink", "MNE-eyelink-data", None), ("fnirs_motor", "MNE-fNIRS-motor-data", None), - ("refmeg_noise", "MNE-refmeg-noise-data", None), ("phantom_kernel", "MNE-phantom-kernel-data", None), ("multimodal", "MNE-multimodal-data", None), ("kiloword", "MNE-kiloword-data", "kword_metadata-epo.fif"), - ("erp_core", "MNE-ERP-CORE-data", "ERP-CORE_Subject-001_Task-Flankers_eeg.fif"), ("mtrf", "mTRF_1.5", "speech_data.mat"), ): getattr(mne.datasets, _ds).data_path = _lite_dataset_path(_folder, _probe) @@ -284,10 +281,12 @@ def _lite_check_fname( mne_check._check_fname = _lite_check_fname _lite_rebind("_check_fname", _orig_check_fname, _lite_check_fname) -import matplotlib.pyplot as plt # imread: the eyetracking heatmap's stimulus +import matplotlib.pyplot as plt # the eyetracking heatmap reads its stimulus +import nibabel # a few tutorials load an MRI themselves for _module, _name, _siblings in ( (plt, "imread", None), + (nibabel, "load", None), (mne.io, "read_raw_eeglab", lambda rel: [rel.removesuffix(".set") + ".fdt"]), ( mne.io, @@ -308,9 +307,7 @@ def _lite_check_fname( _lite_wrap_reader(pyxdf, "load_xdf") except Exception: pass -# folders rather than files -mne.io.read_raw_nirx = _lite_dir_reader(mne.io.read_raw_nirx) -mne.io.read_raw_egi = _lite_dir_reader(mne.io.read_raw_egi) +mne.io.read_raw_nirx = _lite_dir_reader(mne.io.read_raw_nirx) # a folder # Filesystem probes: fetch the candidates first, in the order MNE tries them, # then let it choose as it normally would. The viz modules bind these names at diff --git a/doc/sphinxext/jupyterlite_cell_notes.py b/doc/sphinxext/jupyterlite_cell_notes.py new file mode 100644 index 00000000000..17d97f2ba1e --- /dev/null +++ b/doc/sphinxext/jupyterlite_cell_notes.py @@ -0,0 +1,109 @@ +"""Per-notebook fixups applied to the JupyterLite copies only. + +:func:`note_unrunnable_cells` does two things to every notebook sphinx-gallery +copies into the JupyterLite contents: + +1. Prepends the setup cell that installs MNE and patches the browser + environment (see :mod:`jupyterlite_setup_cell`). +2. Swaps any cell that cannot run in the browser for a note that keeps the code + in view and says what it needs instead; a cell here and there is blocked + even though the rest of its notebook runs, and dropping a whole page from + the launcher over one cell costs more than it saves. + +Both are deliberately done here rather than through ``first_notebook_cell``, +which sphinx-gallery applies while *generating* the notebook and therefore also +writes into the ``.ipynb`` offered for download, where ``piplite`` does not +exist and the notebook would fail on its first cell. Doing it at copy time +keeps the download and the rendered page exactly as the docs built them. + +It lives here rather than in ``conf.py`` because ``sphinx_gallery_conf`` has to +stay JSON-serializable (``sphinx.config.is_serializable`` rejects functions), so +the config names the dotted path and sphinx-gallery imports it. +""" + +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + +import os + +import sphinx.util.logging +from jupyterlite_setup_cell import LITE_SETUP_CELL + +# not mne_doc_utils.sphinx_logger: that module pulls in mne and pyvista, which +# this one has no use for +logger = sphinx.util.logging.getLogger("mne") + +# first line of LITE_SETUP_CELL, used to spot a cell that is already there +_SETUP_MARKER = LITE_SETUP_CELL.strip().split("\n", 1)[0] + +# (notebook path suffix, substring identifying the cell, replacement markdown). +# Keep this short: a page that is mostly unavailable belongs in +# JUPYTERLITE_EXCLUDE instead of here. +CELL_NOTES = ( + ( + "forward/20_source_alignment.ipynb", + "mne.gui.coregistration", + """**This cell does not run in the browser.** + +`mne.gui.coregistration` sets the fiducials by clicking on the scalp surface, +and the vtk.js renderer used here draws scenes without a picker, so there is +nothing for those clicks to hit. Run it from a local MNE install instead: + +```python +mne.gui.coregistration(subject="sample", subjects_dir=subjects_dir) +``` + +The video above walks through the same steps, and the rest of this notebook +runs normally. +""", + ), +) + + +def note_unrunnable_cells(notebook_content, notebook_filename): + """Add the setup cell and note the cells that cannot run in the browser. + + Parameters + ---------- + notebook_content : dict + The parsed notebook, modified in place. + notebook_filename : path-like + Where the notebook will be written inside the JupyterLite contents. + """ + # setdefault, not get: a missing key would otherwise hand back a throwaway + # list and the insert would silently not stick + cells = notebook_content.setdefault("cells", []) + # stale .ipynb from an earlier build can already carry the cell; adding a + # second one would install everything twice + already = cells and _SETUP_MARKER in "".join(cells[0].get("source", [])) + if not already: + cells.insert( + 0, + { + "cell_type": "code", + "execution_count": None, + "metadata": {"collapsed": False}, + "outputs": [], + # .strip() to match what sphinx-gallery's add_code_cell wrote + # while this went through first_notebook_cell + "source": [LITE_SETUP_CELL.strip()], + }, + ) + path = str(notebook_filename).replace(os.sep, "/") + for suffix, needle, note in CELL_NOTES: + if not path.endswith(suffix): + continue + for cell in cells: + # prose mentions the same function, so only rewrite real code + if cell.get("cell_type") != "code": + continue + if needle not in "".join(cell.get("source", [])): + continue + cell["cell_type"] = "markdown" + cell["source"] = [note] + cell["metadata"] = {} + # markdown cells carry neither of these, and nbformat rejects them + cell.pop("outputs", None) + cell.pop("execution_count", None) + logger.info(f"[JupyterLite] {suffix}: noted {needle} cell") diff --git a/doc/sphinxext/jupyterlite_data.py b/doc/sphinxext/jupyterlite_data.py new file mode 100644 index 00000000000..829380b4d3d --- /dev/null +++ b/doc/sphinxext/jupyterlite_data.py @@ -0,0 +1,183 @@ +"""Stage the data the JupyterLite notebooks read, and the MNE wheel they install. + +The setup cell fetches files over HTTP, so ``conf.py`` serves this subset of +the datasets at the docs root (``/mne_data/...``), copied from ``~/mne_data``. +A badged notebook has to find everything it reads here; ``JUPYTERLITE_EXCLUDE`` +in ``conf.py`` lists the ones that do not. +""" + +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + +import os +import shutil +from pathlib import Path + +from build_lite_wheel import build_wheel, find_wheels +from mne_doc_utils import sphinx_logger + +import mne + +MNE_DATA = Path(os.path.expanduser("~/mne_data")) +# Refuse anything past this rather than bloat the deploy; the largest file that +# has to be served (sample_audvis_raw.fif) is 128 MB. +MAX_FILE_MB = 150 + +# MNE-sample-data files the badged notebooks read. Anything not here is a 404 +# in the browser, so this list follows the tutorials, not the dataset. +SAMPLE_FILES = [ + "version.txt", + "MEG/sample/sample_audvis_raw.fif", + "MEG/sample/sample_audvis_filt-0-40_raw.fif", + "MEG/sample/sample_audvis_raw-eve.fif", + "MEG/sample/sample_audvis_filt-0-40_raw-eve.fif", + "MEG/sample/sample_audvis_ecg-proj.fif", + "MEG/sample/sample_audvis-ave.fif", + "MEG/sample/sample_audvis-cov.fif", + "MEG/sample/sample_audvis-meg-eeg-oct-6-fwd.fif", + "MEG/sample/sample_audvis-meg-oct-6-meg-inv.fif", + "MEG/sample/sample_audvis-meg-oct-6-fwd.fif", + "MEG/sample/sample_audvis-meg-oct-6-meg-fixed-inv.fif", + "MEG/sample/ernoise_raw.fif", + "MEG/sample/sample_audvis-no-filter-ave.fif", + "MEG/sample/sample_audvis_raw-trans.fif", + "MEG/sample/sample_audvis-shrunk-cov.fif", + "MEG/sample/sample_audvis-meg-lh.stc", + "MEG/sample/sample_audvis-meg-rh.stc", + "MEG/sample/sample_audvis-meg-eeg-lh.stc", + "MEG/sample/sample_audvis-meg-eeg-rh.stc", + "MEG/sample/sample_audvis_ecg-eve.fif", + "subjects/sample/mri/T1.mgz", + "subjects/sample/mri/aseg.mgz", + # read_talxfm builds this path itself; plot_alignment estimates the MRI + # fiducials from it + "subjects/sample/mri/transforms/talairach.xfm", + "subjects/sample/bem/sample-oct-6-src.fif", + # head and skull surfaces for plot_alignment, in the order MNE tries them + # so the browser draws the same one the rendered docs do; there is no + # sample-head-dense.fif, and lh.seghead is the dense fallback. The .surf + # files are symlinks into bem/flash/, which copy2 follows. + "subjects/sample/bem/outer_skin.surf", + "subjects/sample/bem/outer_skull.surf", + "subjects/sample/bem/inner_skull.surf", + "subjects/sample/bem/sample-head.fif", + "subjects/sample/surf/lh.seghead", + # the single-layer BEM solution; the three-layer one is 237 MB, so the + # notebooks needing it are excluded instead + "subjects/sample/bem/sample-5120-bem-sol.fif", + # the fsaverage source space ships inside MNE-sample-data + "subjects/fsaverage/bem/fsaverage-ico-5-src.fif", + "subjects/sample/surf/rh.pial", + "subjects/sample/surf/lh.pial", + "subjects/sample/surf/rh.white", + "subjects/sample/surf/lh.white", + "subjects/sample/surf/rh.inflated", + "subjects/sample/surf/lh.inflated", + "subjects/sample/surf/rh.curv", + "subjects/sample/surf/lh.curv", + # setup_source_space reads surf/{hemi}.sphere by a path it builds itself + "subjects/sample/surf/lh.sphere", + "subjects/sample/surf/rh.sphere", + "subjects/sample/label/lh.aparc.annot", + "subjects/sample/label/rh.aparc.annot", + # the auditory/visual ROIs, whose names about nine notebooks build with an + # f-string, so a scan of the tutorial text never sees them + "MEG/sample/labels/Aud-lh.label", + "MEG/sample/labels/Aud-rh.label", + "MEG/sample/labels/Vis-lh.label", + "MEG/sample/labels/Vis-rh.label", +] + +# (dataset folder, files): each used by one or two notebooks that read only a +# couple of files out of it. A full docs build downloads all of these datasets. +DATASET_FILES = { + "MNE-sample-data": SAMPLE_FILES, + "ssvep-example-data": [ + f"sub-02/ses-01/eeg/sub-02_ses-01_task-ssvep_eeg{s}" + for s in (".vhdr", ".eeg", ".vmrk") + ], + "MNE-misc-data": [ + "xdf/sub-P001_ses-S004_task-Default_run-001_eeg_a2.xdf", + "movement/simulated_quats.pos", + "movement/simulated_movement_raw.fif", + "movement/simulated_stationary_raw.fif", + "eyetracking/eyelink/px_textpage_ws.asc", + "eyetracking/eyelink/HREF_textpage_ws.asc", + ], + "MNE-eyelink-data": [ + "freeviewing/sub-01_task-freeview_eyetrack.asc", + "freeviewing/stim/naturalistic.png", + ], + "MNE-kiloword-data": ["kword_metadata-epo.fif"], + "mTRF_1.5": ["speech_data.mat"], + # exactly the runs tools/circleci_download.sh fetches (subject 1 runs + # 3/6/10/14, run 3 for subjects 2-4); notebooks wanting runs 1 or 2 are + # excluded instead + "MNE-eegbci-data": [ + f"files/eegmmidb/1.0.0/S{s:03d}/S{s:03d}R{r:02d}.edf" + for s, r in ((1, 3), (1, 6), (1, 10), (1, 14), (2, 3), (3, 3), (4, 3)) + ], +} +# whole folders, for readers handed a directory rather than a file +# (read_raw_nirx); a manifest is left for the setup cell +DATASET_TREES = [("MNE-fNIRS-motor-data", "Participant-1")] +# somato, ERP-CORE, refmeg_noise and the rest of the testing and eyelink +# datasets are deliberately not served; see the size tiers in conf.py + + +def _copy(src, dst): + """Copy ``src`` to ``dst`` unless a same-sized copy is already there.""" + if dst.exists() and dst.stat().st_size == src.stat().st_size: + return False + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + return True + + +def stage_lite_data(dst_base): + """Copy the served data subset under ``dst_base`` and build the MNE wheel.""" + dst_base = Path(dst_base) + n_copied = n_missing = 0 + for folder, rels in DATASET_FILES.items(): + for rel in rels: + src = MNE_DATA / folder / rel + if not src.exists(): + sphinx_logger.info(f"[JupyterLite] MISSING {folder}/{rel}") + n_missing += 1 + elif src.stat().st_size / 1e6 > MAX_FILE_MB: + sphinx_logger.info(f"[JupyterLite] SKIPPED {folder}/{rel} (too big)") + else: + n_copied += _copy(src, dst_base / folder / rel) + for folder, rel_dir in DATASET_TREES: + src_dir = MNE_DATA / folder / rel_dir + if not src_dir.is_dir(): + sphinx_logger.info(f"[JupyterLite] MISSING {folder}/{rel_dir}") + n_missing += 1 + continue + # zero-byte members do not survive the artifact upload, so listing + # them would only yield 404s + names = [ + str(f.relative_to(src_dir)) + for f in sorted(src_dir.rglob("*")) + if f.is_file() and 0 < f.stat().st_size / 1e6 <= MAX_FILE_MB + ] + for name in names: + n_copied += _copy(src_dir / name, dst_base / folder / rel_dir / name) + (dst_base / folder / rel_dir / "_lite_manifest.txt").write_text( + "\n".join(names) + ) + # The logging tutorial reads a KIT file that lives inside the package under + # mne/io/kit/tests/, which the wheel leaves out, so serve it and let the + # setup cell stage it back into the path the tutorial builds. + kit = Path(mne.__file__).parent / "io" / "kit" / "tests" / "data" / "test.sqd" + n_copied += _copy(kit, dst_base / "MNE-kit-testdata" / "test.sqd") + sphinx_logger.info( + f"[JupyterLite] Served data: {n_copied} files copied, {n_missing} missing" + ) + # the development wheel, so the browser installs this MNE rather than the + # PyPI release: doc/sphinxext/build_lite_wheel.py puts it in doc/pypi, where + # the piplite addon indexes it; `make html` runs that first, so this is + # only a fallback for a bare sphinx-build + wheels = find_wheels() or build_wheel() + sphinx_logger.info(f"[JupyterLite] MNE wheel for the browser kernel: {wheels}") diff --git a/mne/datasets/config.py b/mne/datasets/config.py index 293c29bcbb3..95cd2738987 100644 --- a/mne/datasets/config.py +++ b/mne/datasets/config.py @@ -209,8 +209,7 @@ config_key="MNE_DATASETS_SAMPLE_PATH", ) -# Curated subset of sample (plus a few files from kiloword/erp_core/mtrf/eegbci) -# used by the JupyterLite browser docs; see mne/datasets/lite_data/. +# TODO VERSION: remove in 1.15 with the deprecated mne.datasets.lite_data MNE_DATASETS["lite_data"] = dict( archive_name="MNE-lite-data.tar.gz", hash="md5:5f9c4fffed32e79bc2bc2061bf22ce99", diff --git a/mne/datasets/lite_data/lite_data.py b/mne/datasets/lite_data/lite_data.py index 7e52909bafc..53e8e3fa611 100644 --- a/mne/datasets/lite_data/lite_data.py +++ b/mne/datasets/lite_data/lite_data.py @@ -2,19 +2,13 @@ # License: BSD-3-Clause # Copyright the MNE-Python contributors. -"""Curated data subset used by the JupyterLite browser documentation. - -``lite_data`` holds the data files needed to run the tutorials and examples in -the browser, taken from ``sample``, ``kiloword``, ``erp_core``, ``mtrf`` and -``eegbci``. The files are unchanged and keep the same checksums as the full -datasets. It extracts to ``MNE-lite-data/`` with each file under its original -dataset folder (``MNE-sample-data/``, ``MNE-kiloword-data/``, ...), so paths -match. -The ``somato`` dataset is not included, so the somatosensory tutorials and -examples do not run in the browser. +"""Curated data subset once used by the JupyterLite browser documentation. + +The documentation build now serves the browser notebooks from the regular +datasets, so this archive is no longer needed. """ -from ...utils import verbose +from ...utils import deprecated, verbose from ..utils import _data_path_doc, _download_mne_dataset, _get_version, _version_doc @@ -35,6 +29,11 @@ def data_path( data_path.__doc__ = _data_path_doc.format( name="lite_data", conf="MNE_DATASETS_LITE_DATA_PATH" ) +_DEPRECATED = ( + "The documentation build no longer uses the lite_data archive, so it will be " + "removed in MNE 1.15; use the individual dataset fetchers instead" +) +data_path = deprecated(_DEPRECATED)(data_path) def get_version(): # noqa: D103 @@ -42,3 +41,4 @@ def get_version(): # noqa: D103 get_version.__doc__ = _version_doc.format(name="lite_data") +get_version = deprecated(_DEPRECATED)(get_version) diff --git a/mne/datasets/tests/test_datasets.py b/mne/datasets/tests/test_datasets.py index db447b13006..3ad21a5bffb 100644 --- a/mne/datasets/tests/test_datasets.py +++ b/mne/datasets/tests/test_datasets.py @@ -358,5 +358,7 @@ def test_lite_data(): assert cfg["hash"].startswith("md5:") assert cfg["url"].startswith("https://osf.io/") assert cfg["config_key"] == "MNE_DATASETS_LITE_DATA_PATH" - assert callable(lite_data.data_path) - assert callable(lite_data.get_version) + with pytest.warns(FutureWarning, match="removed in MNE 1.15"): + lite_data.data_path(download=False) + with pytest.warns(FutureWarning, match="removed in MNE 1.15"): + lite_data.get_version()