diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b4cf79..c4957d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ All notable changes to lcode are documented here. The format follows like it over four nights, the extra work also fixed a test the model had called pre-existing. - **Checks that don't finish in 10 minutes are no longer called a failure.** The model is told that they don't say whether its change works, instead of to fix what it broke. +- **A SPARQL query it never ran** now counts like code it never ran (#242): on the dev slice, 2 of 3 + `sparql-university` attempts on 0.61.0 wrote `solution.sparql` and said they were done without running it. + lcode sends the model back once, and the note names `rdflib`. SQL files stay out, because one can be a + migration, and asking to run that could change a real database. ### Fixed diff --git a/docs/usage.md b/docs/usage.md index c97df0d..ebefe29 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -273,9 +273,10 @@ or reading it doesn't count as running it. `checks = "off"` turns this off too. **Code it never ran.** A model that writes a new module and finds no Python, or checks only its syntax, tends to call it done, and the first run shows the error. When a request wrote a new code -file (Python, JavaScript, TypeScript, Rust, Go, C, Java and other languages that run) and no command -since its last change ran it, lcode sends it back once: run it (import a module and call what it -wrote; start a server in the background and try it) and fix what fails, or say plainly that it's +file (Python, JavaScript, TypeScript, Rust, Go, C, Java and other languages that run, or a SPARQL +query) and no command since its last change ran it, lcode sends it back once: run it (import a module +and call what it wrote; run a query on its data; start a server in the background and try it) and fix +what fails, or say plainly that it's untested and why. A command counts when it names the file or its module, or runs tests, and its program was there: a syntax, lint or type check, `python3: command not found` or a command that only reads doesn't count. When `uv` is installed, the note adds that `uv run --with PACKAGE python FILE` runs diff --git a/src/lcode/checks.py b/src/lcode/checks.py index f6857ed..eae2d5f 100644 --- a/src/lcode/checks.py +++ b/src/lcode/checks.py @@ -338,10 +338,12 @@ def unrun_note(scripts: list[str]) -> str: # ----------------------------------------------------------------------------- code written, never run +# Not .sql: one can be a migration, and asking the model to run that could change a real database. CODE_SUFFIXES = frozenset( {".py", ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".rs", ".go", ".c", ".cc", ".cpp", ".java", ".kt", ".rb", - ".php", ".pl", ".lua", ".jl", ".r", ".scala", ".swift", ".cs"} + ".php", ".pl", ".lua", ".jl", ".r", ".scala", ".swift", ".cs", ".sparql", ".rq"} ) # fmt: skip +SPARQL_SUFFIXES = (".sparql", ".rq") # Checks that read the code without running it (syntax, lint, types): they don't show that it works. STATIC_ONLY = re.compile( r"\bast\.parse\b|\bpy_compile\b|\bcompileall\b|(?:^|\s)-fsyntax-only\b|\b(?:node|deno)\s+(?:-c|--check)\b" @@ -359,7 +361,7 @@ def unrun_note(scripts: list[str]) -> str: def is_code(rel: str) -> bool: - """Whether a file is a program or module in a language that runs (shell scripts have their own check).""" + """Whether a file is a program, module or query in a language that runs (shell scripts have their own check).""" return Path(rel).suffix.lower() in CODE_SUFFIXES @@ -390,16 +392,19 @@ def untested_note(files: list[str], missing: list[str], uv: bool) -> str: note = ( f"[lcode] You wrote {names} in this request but haven't run {it} since your last change. Run {it} now and fix " f"what fails, so you know {it} works before you say so: for a module, import it and call what you wrote; " - "for a server or anything else that keeps running, start it in the background and try it. A syntax check " - "doesn't show that it works." + "for a query, run it on its data; for a server or anything else that keeps running, start it in the " + "background and try it. A syntax check doesn't show that it works." ) if missing: note += f" {', '.join(f'`{m}`' for m in missing)} isn't installed here." - if uv and any(f.endswith(".py") for f in files): + sparql = any(f.lower().endswith(SPARQL_SUFFIXES) for f in files) + if uv and (sparql or any(f.endswith(".py") for f in files)): note += ( " To run Python code without changing this machine's Python, `uv run --with PACKAGE python FILE` fetches " "Python and the packages it needs into a throwaway environment." ) + if sparql: + note += " For a SPARQL query, the package is `rdflib`." return note + f" If you can't run {it}, say plainly in your answer that {it} is untested, and why." diff --git a/tests/test_stuck.py b/tests/test_stuck.py index cbe0821..59011e7 100644 --- a/tests/test_stuck.py +++ b/tests/test_stuck.py @@ -325,8 +325,27 @@ def test_what_counts_as_running_new_code(): no_torch = "ModuleNotFoundError: No module named 'torch'\n\n[exit code: 1]" assert checks.exercised("python3 -c 'import geometry'", no_torch, files) == [] assert checks.not_found(missing + "\n/bin/sh: 1: node: not found\n") == ["python3", "node"] - assert checks.is_code("geometry.py") and checks.is_code("src/Main.JAVA") - assert not any(checks.is_code(f) for f in ("build.sh", "notes.md", "meeting.ics", "query.sparql")) + assert all(checks.is_code(f) for f in ("geometry.py", "src/Main.JAVA", "query.sparql", "ask.rq")) + assert not any(checks.is_code(f) for f in ("build.sh", "notes.md", "meeting.ics", "schema.sql")) + + +def test_a_query_written_and_never_run_is_sent_back_once(make_agent, repo, monkeypatch): + """2 of 3 sparql-university attempts on 0.61.0 wrote solution.sparql and said they were done without running it.""" + monkeypatch.setattr(checks, "uv_available", lambda: True) + agent = make_agent( + [ + reply(tool_calls=[call("write_file", path="solution.sparql", content="SELECT ?s WHERE { ?s ?p ?o }\n")]), + reply("Done: solution.sparql has the query."), + reply(tool_calls=[call("bash", command="uv run --with rdflib python -c 'open(\"solution.sparql\")'")]), + reply("It runs, and returns 3 rows."), + ], + checks="auto", + ) + agent.run_turn("write the query") + notes = untested_notes(agent) + assert len(notes) == 1 and "You wrote solution.sparql" in notes[0] and "for a query, run it on its data" in notes[0] + assert "the package is `rdflib`" in notes[0] + assert agent.messages[-1]["content"] == "It runs, and returns 3 rows." def test_code_written_and_never_run_is_sent_back_once(make_agent, repo, monkeypatch):