ci: fail closed when README command discovery is empty

tests/documentation_commands.py

@@ -0,0 +1,29 @@
+"""Shared extraction of executable shell examples from Markdown documents."""
+
+from __future__ import annotations
+
+import re
+import shlex
+
+
+_FENCE_PATTERN = re.compile(r"^\s*```")
+_COMMAND_PATTERN = re.compile(r"^northstar(?:-[a-z0-9-]+)?(?:\s|$)")
+
+
+def parse_documented_commands(markdown: str) -> list[list[str]]:
+    """Return Northstar commands from any fenced Markdown code block."""
+
+    commands: list[list[str]] = []
+    in_code_block = False
+    for line in markdown.splitlines():
+        if _FENCE_PATTERN.match(line):
+            in_code_block = not in_code_block
+            continue
+        if not in_code_block:
+            continue
+        command_line = line.strip()
+        if command_line.startswith("$ "):
+            command_line = command_line[2:].lstrip()
+        if _COMMAND_PATTERN.match(command_line):
+            commands.append(shlex.split(command_line))
+    return commands

tests/test_documentation.py

@@ -1,27 +1,23 @@
 from __future__ import annotations
 
-import re
-import shlex
 import tempfile
 from pathlib import Path
 
 import base_cli
 import pytest
 
 from base_cli_demo.cli import command
+from documentation_commands import parse_documented_commands
 
 
-COMMAND_PATTERN = re.compile(r"^\$\s+(northstar(?:\s+.*)?)$")
 DOCUMENTS = (Path("README.md"), Path("docs/learning-path.md"))
 
 
 def documented_commands() -> list[tuple[str, list[str]]]:
     examples: list[tuple[str, list[str]]] = []
     for document in DOCUMENTS:
-        for line in document.read_text(encoding="utf-8").splitlines():
-            match = COMMAND_PATTERN.match(line)
-            if match:
-                examples.append((str(document), shlex.split(match.group(1))))
+        commands = parse_documented_commands(document.read_text(encoding="utf-8"))
+        examples.extend((str(document), command) for command in commands)
     return examples
 
 

tests/test_readme_examples.py

@@ -2,24 +2,30 @@
 
 import json
 import os
-import shlex
 import shutil
 import subprocess
 import sys
 from pathlib import Path
 
 import pytest
 
+from documentation_commands import parse_documented_commands
+
+
+README_PATH = Path(__file__).parents[1] / "README.md"
+EXAMPLES_PATH = Path(__file__).parents[1] / "examples"
+
+
+def parse_readme_commands(markdown: str) -> list[list[str]]:
+    return parse_documented_commands(markdown)
+
 
 def readme_commands() -> list[list[str]]:
-    commands: list[list[str]] = []
-    in_shell_block = False
-    for line in Path("README.md").read_text(encoding="utf-8").splitlines():
-        if line.strip().startswith("```"):
-            in_shell_block = not in_shell_block
-            continue
-        if in_shell_block and line.strip().startswith("northstar"):
-            commands.append(shlex.split(line.strip()))
+    commands = parse_readme_commands(README_PATH.read_text(encoding="utf-8"))
+    if not commands:
+        raise AssertionError(
+            "README command discovery found no Northstar commands."
+        )
     return commands
 
 
@@ -40,6 +46,8 @@ def run_installed_command(
             "LOCALAPPDATA": str(home / "home" / "AppData" / "Local"),
         }
     )
+    if any(argument.startswith("examples/") for argument in args):
+        shutil.copytree(EXAMPLES_PATH, home / "examples")
     return subprocess.run(
         [executable, *args[1:]],
         capture_output=True,
@@ -61,6 +69,22 @@ def test_readme_northstar_commands_run_from_the_installed_wheel(
     )
 
 
+def test_readme_command_parser_fails_closed_on_an_invalid_documented_command(
+    tmp_path: Path,
+) -> None:
+    markdown = README_PATH.read_text(encoding="utf-8")
+    assert "$ northstar --help" in markdown
+    markdown = markdown.replace(
+        "$ northstar --help", "$ northstar --definitely-invalid", 1
+    )
+    commands = parse_readme_commands(markdown)
+    invalid = next(args for args in commands if "--definitely-invalid" in args)
+    result = run_installed_command(invalid, tmp_path)
+
+    assert result.returncode != 0
+    assert commands
+
+
 def test_readme_json_output_is_machine_readable(tmp_path: Path) -> None:
     result = run_installed_command(
         ["northstar", "--quiet", "--environment", "dev", "status", "--format", "json"],