diff --git a/pyproject.toml b/pyproject.toml index cdcb0e9..c8902fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version"] description = "A bounded-time, Pandoc-leaning Markdown parser with GFM, Extra/kramdown, math, fenced divs, and MDHTML output." license = {text = "MIT OR Apache-2.0"} requires-python = ">=3.11" -dependencies = ["fast5ever>=0.1.1", "fastcore>=2.2.3", "aidialog>=0.0.14", "pyyaml>=6.0.3", "execnb>=0.2.11"] +dependencies = ["fast5ever>=0.1.1", "fastcore>=2.2.7", "aidialog>=0.0.14", "pyyaml>=6.0.3", "execnb>=0.2.11"] readme = "README.md" authors = [{name = "Jeremy Howard", email = "j@fast.ai"}] classifiers = [ diff --git a/python/mdhtml/fill.py b/python/mdhtml/fill.py index 51437b2..b01b03d 100644 --- a/python/mdhtml/fill.py +++ b/python/mdhtml/fill.py @@ -9,10 +9,15 @@ only pair within that value. Output text is newline-normalized. `tokens` is the shared inventory every template tool builds on (fill, previews, docx field binding); `fill_md` is pure text to text; `instantiate` adds data gathering (frontmatter `formdata:` via `fastcore.xtras.frontmatter` -with `strvals=True`: structure kept, every scalar a `str`) and the one execution point for +with `strvals=True`: structure kept, every scalar a `str` except `true`/`True`/`false`/`False`, +which are `bool`) and the one execution point for `{python}` blocks (an `execnb` shell: IPython last-expression semantics, `_repr_markdown_` -preferred over `str()`, stdout discarded). Trust model: scanning and previewing untrusted -templates is safe; `instantiate` runs a template's code, so instantiating one is trusting it; data +preferred over `str()`, stdout discarded). Dialog templates (`instantiate_nb`) run code opt-in: +only cells marked `#| eval: true` participate (all but `eval: false` cells when the dialog's own +frontmatter says `eval: true`), and a cell that doesn't participate contributes nothing to the +document, not even its stored outputs. Trust model: scanning and previewing untrusted templates is +safe; `instantiate` runs a template's code, so instantiating one is trusting it, and the execution +surface is exactly the participating cells; data from untrusted sources must be sanitized upstream, since a value containing `{{other_field}}` resolves against the data (injected code never runs). A literal `{{` in prose belongs in a backtick code span, which the scanner never enters.""" @@ -22,7 +27,7 @@ from pathlib import Path from fastcore.script import call_parse -from fastcore.xtras import frontmatter +from fastcore.xtras import frontmatter, strloader from fastcore.nbio import nb_frontmatter, cell_frontmatter from execnb.shell import CaptureShell from aidialog.dialog import dlg2md @@ -257,7 +262,7 @@ def fill_md( def frontmatter_data(src): - "The `formdata:` mapping from a leading frontmatter block: real YAML, structure kept, every scalar a `str`." + "The `formdata:` mapping from a leading frontmatter block: real YAML, structure kept, scalars `str` (bools excepted)." meta, _ = frontmatter(src, strvals=True) fd = meta.get("formdata") return fd if isinstance(fd, dict) else {} @@ -314,16 +319,19 @@ async def instantiate_nb( filled=None, # Decoration callback `(name, value) -> str`, default `str(value)` templates=None, # `TemplateDelimiter`s, default `mdhtml.mustache.MUSTACHE` ) -> Md: - "Instantiate a dialog: run every code cell once (`eval: false` excluded), weave participating outputs, fill tokens" + "Instantiate a dialog: run its participating code cells (the `eval` cascade, opt-in by default), weave their outputs, fill tokens" d = read_ipynb(fname) fd = nb_frontmatter(d, strvals=True).get("formdata") merged = {**(fd if isinstance(fd, dict) else {}), **(data or {})} shell = CaptureShell() shell.user_ns["__data__"] = merged - ran = await d.execute(skip_noeval=True, shell=shell) + ran = await d.execute(default_eval=False, shell=shell) if shell.exc: shell.exc.add_note(f"in message {next(m.id for m in ran if m.has_error)}") raise shell.exc + ranids = {m.id for m in ran} + for m in d.messages: # a cell that didn't participate contributes nothing: not even stored outputs + if m.cell_type == "code" and m.id not in ranids: m.output = [] firsts = [next((m for m in d.messages if m.cell_type == ct), None) for ct in ("raw", "markdown")] fm_ids = {m.id for m in firsts if m is not None and cell_frontmatter(m.content)} body = [m for m in d.messages if m.id not in fm_ids] @@ -335,12 +343,12 @@ async def instantiate_nb( @call_parse(pos=["file"]) async def main( file: str = None, # Markdown template, or dialog/notebook `.ipynb`, to read (default: stdin) - data: str = None, # YAML file of per-matter values (`BaseLoader`: scalars stay strings) + data: str = None, # YAML file of per-matter values (scalars stay strings, bools excepted) out: str = None, # Write the filled document here (default: stdout) lenient: bool = False, # Defer unresolved tokens and warn, instead of raising ): "Instantiate a Markdown template (or dialog notebook): execute its `{python}` blocks (or code cells) and fill its tokens" - values = yaml.load(open(data, encoding="utf-8"), Loader=yaml.BaseLoader) if data else {} + values = yaml.load(open(data, encoding="utf-8"), Loader=strloader()) if data else {} if file and file.endswith(".ipynb"): res = await instantiate_nb(file, values, strict=not lenient, dest=out) else: res = await instantiate(read_src(file), values, strict=not lenient, dest=out) for w in res.warnings: print(w, file=sys.stderr) diff --git a/tests/test_fill.py b/tests/test_fill.py index 1ef5ae6..991b41e 100644 --- a/tests/test_fill.py +++ b/tests/test_fill.py @@ -143,9 +143,9 @@ def test_pill_and_cli(tmp_path): assert '{{d}}' in h # cell var: plain pill import subprocess tpl = tmp_path / "t.md" - tpl.write_text("---\nformdata:\n who: Sam\n---\n\nHi {{who}}, {{n}} shares.\n") + tpl.write_text("---\nformdata:\n who: Sam\n---\n\nHi {{who}}, {{n}} shares.{{#paid}} Paid.{{/paid}}\n") vals = tmp_path / "v.yml" - vals.write_text("n: 1000\n") + vals.write_text("n: 1000\npaid: false\n") res = subprocess.run(["fillmd", str(tpl), "--data", str(vals)], text=True, capture_output=True, check=True) assert res.stdout == "Hi Sam, 1000 shares.\n" and res.stderr == "" lenient = subprocess.run(["fillmd", str(tpl)], text=True, capture_output=True) @@ -166,7 +166,7 @@ def test_instantiate_nb(tmp_path): from aidialog.dialog import Message, snote, sraw from mdhtml.fill import instantiate_nb p = mk_dlg(tmp_path, [ - Message("---\nformdata:\n who: Alice\n---", msg_type=sraw), + Message("---\neval: true\nformdata:\n who: Alice\n---", msg_type=sraw), Message("# Report for {{who}}", msg_type=snote), Message("x = 6*7"), Message('f"Result: {x}"'), @@ -195,10 +195,23 @@ def test_instantiate_nb_participation(tmp_path): assert "only me A" in res and "kept" not in res +def test_instantiate_nb_optin(tmp_path): + from aidialog.dialog import Message, snote, sraw + from mdhtml.fill import instantiate_nb + fm = Message("---\nformdata:\n who: A\n---", msg_type=sraw) + note = Message("Hi {{who}}:", msg_type=snote) + stale = Message("'lawyer scratch'") + stale.output = [dict(output_type="stream", name="stdout", text="stale output\n")] + live = Message('#| eval: true\n"fresh"') + res = run_sync(instantiate_nb(mk_dlg(tmp_path, [fm, note, stale, live]))) + assert "Hi A:" in res and "fresh" in res + assert "stale output" not in res and "lawyer scratch" not in res + + def test_instantiate_nb_error(tmp_path): from aidialog.dialog import Message from mdhtml.fill import instantiate_nb - bad = Message("1/0") + bad = Message("#| eval: true\n1/0") p = mk_dlg(tmp_path, [bad]) with pytest.raises(ZeroDivisionError) as ei: run_sync(instantiate_nb(p)) assert any(bad.id in n for n in ei.value.__notes__)