From 4708fe9c522449bd03edcdb5ee4f3bc0cc0bdd2c Mon Sep 17 00:00:00 2001 From: eimexdev Date: Wed, 21 Oct 2026 20:21:45 -0700 Subject: [PATCH] Pass non-ASCII paths to the helper on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve's Lua 5.1 opens files and starts processes through the ANSI code page, while Resolve and the installer's config use UTF-8, so a user name like José or media in Música broke the runtime lookup and garbled the audio path on the command line. The Windows installer now also records python_bin_ansi, the runtime path as an 8.3 short path or, without short names, in the ANSI code page; the config file itself is written as ASCII with Lua escapes. On Windows the scripts pass Resolve's arguments in a UTF-8 JSON file that the CLI reads with a new leading --args-file option, while temporary paths taken from the environment stay on the command line. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + openbeat/cli.py | 16 ++- .../Modules/OpenBeat/OpenBeatCommon.lua | 78 +++++++++--- scripts/staged_install.py | 62 ++++++++- tests/lua/test_helper.lua | 2 +- tests/lua/test_windows_paths.lua | 103 +++++++++++++++ tests/test_windows_paths.py | 119 ++++++++++++++++++ 7 files changed, 362 insertions(+), 19 deletions(-) create mode 100644 tests/lua/test_windows_paths.lua create mode 100644 tests/test_windows_paths.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b47f24..1e47eac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable OpenBeat changes should be summarized here for users. GitHub Release - Installer updates now prepare and verify the new files before replacing the installed ones. If an update fails or is interrupted, the previous installation is kept or restored by running the installer again. - Release builds use one resolved commit for testing, building, and tagging, and pinned build dependencies. - When beat analysis or click-track rendering fails, OpenBeat now shows the helper's error instead of a missing-file error, and no longer imports an older click track as if it were new. +- On Windows, OpenBeat should now work when the Windows user name or the media path contains non-English characters, such as `José` or `Música`. Subtitle export to such folders is not fixed yet. ## 0.0.2 - 2026-04-19 diff --git a/openbeat/cli.py b/openbeat/cli.py index 79041be..35f6217 100644 --- a/openbeat/cli.py +++ b/openbeat/cli.py @@ -2,6 +2,7 @@ import argparse import json +import sys from dataclasses import asdict from pathlib import Path @@ -59,9 +60,22 @@ def build_parser() -> argparse.ArgumentParser: return parser +def expand_args_file(argv: list[str]) -> list[str]: + """Replace a leading ``--args-file PATH`` with the arguments listed in that UTF-8 JSON file. + + The Resolve scripts use it on Windows, where command lines pass through the ANSI code page. + """ + if len(argv) < 2 or argv[0] != "--args-file": + return argv + listed = json.loads(Path(argv[1]).read_text(encoding="utf-8")) + if not isinstance(listed, list) or not all(isinstance(item, str) for item in listed): + raise SystemExit(f"{argv[1]} must contain a JSON list of strings") + return listed + argv[2:] + + def main() -> int: parser = build_parser() - args = parser.parse_args() + args = parser.parse_args(expand_args_file(sys.argv[1:])) return args.func(args) diff --git a/resolve/Fusion/Modules/OpenBeat/OpenBeatCommon.lua b/resolve/Fusion/Modules/OpenBeat/OpenBeatCommon.lua index c3177e5..7f67ecb 100644 --- a/resolve/Fusion/Modules/OpenBeat/OpenBeatCommon.lua +++ b/resolve/Fusion/Modules/OpenBeat/OpenBeatCommon.lua @@ -235,6 +235,11 @@ local function file_size(path) end local function python_bin() + -- Lua 5.1 on Windows opens files and starts processes through the ANSI code page, so the + -- installer also records the runtime path spelled for it when the UTF-8 path is not ASCII. + if is_windows() and local_config.python_bin_ansi and file_exists(local_config.python_bin_ansi) then + return local_config.python_bin_ansi + end if local_config.python_bin and file_exists(local_config.python_bin) then return local_config.python_bin end @@ -260,6 +265,59 @@ local function command_prefix() return shell_quote(bin) end +local function is_ascii(value) + return not tostring(value):find("[\128-\255]") +end + +local JSON_ESCAPES = { ['"'] = '\\"', ["\\"] = "\\\\", ["\n"] = "\\n", ["\r"] = "\\r", ["\t"] = "\\t" } + +local function json_string(value) + local escaped = tostring(value):gsub('[%c"\\]', function(char) + return JSON_ESCAPES[char] or string.format("\\u%04x", char:byte()) + end) + return '"' .. escaped .. '"' +end + +local function command_arguments(values) + local quoted = {} + for _, value in ipairs(values or {}) do + -- Subcommands, options and modes stay readable in the log. + table.insert(quoted, " " .. (value:match("^[%w%-]+$") and value or shell_quote(value))) + end + return table.concat(quoted) +end + +-- Runs the helper with `resolve_args`, which include text from Resolve such as media paths, +-- followed by `local_args`, which OpenBeat builds from the environment (temporary files). +-- Resolve's strings are UTF-8, but Lua 5.1 on Windows starts processes through the ANSI +-- code page, which garbles any non-ASCII character. On Windows the Resolve arguments are +-- therefore written to a UTF-8 file that the helper reads; environment-derived paths are +-- already in the ANSI code page and stay on the command line. +local function run_helper(resolve_args, local_args) + if not is_windows() then + return run_command(command_prefix() .. command_arguments(resolve_args) .. command_arguments(local_args)) + end + + local args_path = temp_path("openbeat_args", ".json") + local encoded = {} + for _, value in ipairs(resolve_args) do + table.insert(encoded, json_string(value)) + end + local handle, open_error = io.open(args_path, "wb") + if not handle then + error("OpenBeat could not write its helper arguments: " .. tostring(open_error), 0) + end + handle:write("[", table.concat(encoded, ", "), "]\n") + handle:close() + local command = command_prefix() .. " --args-file " .. shell_quote(args_path) .. command_arguments(local_args) + local ok, result = pcall(run_command, command) + os.remove(args_path) + if not ok then + error(result, 0) + end + return result +end + local function project_context() local resolve_app = resolve or (app and app:GetResolve()) if not resolve_app then @@ -319,12 +377,7 @@ end local function analyze_source(source_path) local output = temp_path("openbeat_analysis", ".lua") - local final_command = command_prefix() - .. " analyze --audio " - .. shell_quote(source_path) - .. " --format lua --output " - .. shell_quote(output) - local ran, run_error = pcall(run_command, final_command) + local ran, run_error = pcall(run_helper, { "analyze", "--audio", source_path }, { "--format", "lua", "--output", output }) local analysis, load_error if ran then analysis, load_error = load_analysis(output) @@ -402,15 +455,10 @@ end local function render_click_track(source_path, mode) local suffix = mode == "raw" and ".openbeat-raw-clicks" or ".openbeat-clicks" local output = output_path_for(source_path, suffix, ".wav") - local command = command_prefix() - .. " click-track --audio " - .. shell_quote(source_path) - .. " --mode " - .. shell_quote(mode) - .. " --output " - .. shell_quote(output) - run_command(command) - if (file_size(output) or 0) == 0 then + run_helper({ "click-track", "--audio", source_path, "--mode", mode, "--output", output }) + -- On Windows, Lua cannot open a non-ASCII path from Resolve; the helper's success stands. + local checkable = is_ascii(output) or not is_windows() + if checkable and (file_size(output) or 0) == 0 then error("OpenBeat's helper finished but did not write the click track " .. output, 0) end return output diff --git a/scripts/staged_install.py b/scripts/staged_install.py index c15c182..f9256d0 100644 --- a/scripts/staged_install.py +++ b/scripts/staged_install.py @@ -54,6 +54,64 @@ def runtime_executable(modules: Path) -> Path: return modules / "bin" / "openbeat.exe" +def windows_short_path(path: str) -> str | None: + """Return the 8.3 short form of an existing ``path``, or ``None`` where unavailable.""" + try: + import ctypes + from ctypes import wintypes + + get_short_path = ctypes.windll.kernel32.GetShortPathNameW + except Exception: + return None + get_short_path.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD] + get_short_path.restype = wintypes.DWORD + size = get_short_path(path, None, 0) + if not size: + return None + buffer = ctypes.create_unicode_buffer(size) + if not 0 < get_short_path(path, buffer, size) < size: + return None + return buffer.value + + +def ansi_path(path: Path, anchor: Path, encoding: str = "mbcs") -> bytes | None: + """Spell a non-ASCII ``path`` for Lua 5.1 on Windows, or return ``None`` if it cannot. + + Resolve's Lua opens files and starts processes through the ANSI code page, so the + UTF-8 path never matches. The 8.3 short path is plain ASCII where the volume has + short names; otherwise the path encoded in the ANSI code page works if each of its + characters exists there. ``path`` need not exist yet: it is spelled as the short + form of the existing folder ``anchor`` followed by the rest of ``path``. + """ + text = path.as_posix() + if text.isascii(): + return None + rest = path.relative_to(anchor).parts + short = windows_short_path(str(anchor)) + if short and short.isascii() and all(name.isascii() for name in rest): + return "/".join([short.replace("\\", "/").rstrip("/"), *rest]).encode("ascii") + try: + return text.encode(encoding) + except (UnicodeError, LookupError): + return None + + +def lua_string(value: bytes) -> str: + """Quote ``value`` as a Lua string literal made of ASCII characters only.""" + return '"' + "".join( + chr(byte) if 32 <= byte < 127 and byte not in (34, 92) else f"\\{byte:03d}" for byte in value + ) + '"' + + +def runtime_config(runtime: Path, anchor: Path, encoding: str = "mbcs") -> str: + """The ``OpenBeatConfig.local.lua`` that points the Resolve scripts at ``runtime``.""" + lines = [f" python_bin = {lua_string(runtime.as_posix().encode('utf-8'))},"] + ansi = ansi_path(runtime, anchor, encoding) + if ansi is not None: + lines.append(f" python_bin_ansi = {lua_string(ansi)},") + return "return {\n" + "\n".join(lines) + "\n}\n" + + def manifest_path(directory: Path) -> Path: return directory.with_name(f"{directory.name}.sha256") @@ -189,8 +247,8 @@ def _stage(payload_root: Path, targets: list[Path]) -> Path: if not staged_runtime.is_file(): raise InstallError("The installer payload does not contain the OpenBeat runtime.") runtime = modules / staged_runtime.relative_to(staged_modules) - config = f'return {{\n python_bin = "{runtime.as_posix()}",\n}}\n' - (staged_modules / CONFIG_NAME).write_text(config, encoding="utf-8") + config = runtime_config(runtime, anchor=modules.parent) + (staged_modules / CONFIG_NAME).write_text(config, encoding="ascii") return runtime diff --git a/tests/lua/test_helper.lua b/tests/lua/test_helper.lua index 1585d27..09b123c 100644 --- a/tests/lua/test_helper.lua +++ b/tests/lua/test_helper.lua @@ -18,7 +18,7 @@ local function upvalue(fn, name) end local analyze_source = upvalue(upvalue(OpenBeat.run, "create_timeline_markers"), "analyze_source") local render_click_track = upvalue(upvalue(OpenBeat.run, "create_click_audio"), "render_click_track") -local run_command = upvalue(analyze_source, "run_command") +local run_command = upvalue(upvalue(analyze_source, "run_helper"), "run_command") local temp_path = upvalue(analyze_source, "temp_path") local output_path_for = upvalue(render_click_track, "output_path_for") diff --git a/tests/lua/test_windows_paths.lua b/tests/lua/test_windows_paths.lua new file mode 100644 index 0000000..78b29b1 --- /dev/null +++ b/tests/lua/test_windows_paths.lua @@ -0,0 +1,103 @@ +-- On Windows, Resolve's Lua starts processes through the ANSI code page, so UTF-8 text from +-- Resolve must reach the helper in a file, and the runtime is found by its ANSI spelling. +local directory = assert(arg[1]) +local open_file = io.open +io.open = function(path, mode) + if tostring(path):match("OpenBeat%.log$") then return nil end + return open_file(path, mode) +end + +local OpenBeat = dofile("resolve/Fusion/Modules/OpenBeat/OpenBeatCommon.lua") +local function upvalue(fn, name) + for index = 1, 200 do + local key, value = debug.getupvalue(fn, index) + if key == name then return value, index end + if not key then break end + end + error("Missing upvalue " .. name) +end +local analyze_source = upvalue(upvalue(OpenBeat.run, "create_timeline_markers"), "analyze_source") +local render_click_track = upvalue(upvalue(OpenBeat.run, "create_click_audio"), "render_click_track") +local run_helper = upvalue(analyze_source, "run_helper") +local python_bin = upvalue(upvalue(run_helper, "command_prefix"), "python_bin") +local function exists(path) + local handle = open_file(path, "rb") + if handle then handle:close() end + return handle ~= nil +end +local function read(path) + local handle = assert(open_file(path, "rb")); local text = handle:read("*a"); handle:close(); return text +end + +-- Installed configuration: the UTF-8 runtime path does not open through the ANSI code page, +-- so the ANSI spelling the installer recorded is used. +local ansi_runtime = directory .. "/openbeat~1.exe" +local utf8_runtime = directory .. "/Jos\195\169/openbeat.exe" +local _, config_index = upvalue(python_bin, "local_config") +debug.setupvalue(python_bin, config_index, { python_bin = utf8_runtime, python_bin_ansi = ansi_runtime }) +local handle = assert(open_file(ansi_runtime, "wb")) +handle:close() + +local posix_config = package.config +local getenv = os.getenv +package.config = "\\" .. posix_config:sub(2) +-- On Linux the backslash makes "tmp\\" a file inside the test folder. +os.getenv = function(name) if name == "TEMP" then return directory .. "/tmp" end return getenv(name) end +assert(python_bin() == ansi_runtime) + +local media = "C:\\M\195\186sica\\canci\195\179n.wav" +local behavior +local seen = {} +io.popen = function(command) + assert(not command:find("[\128-\255]"), "Non-ASCII text on the command line: " .. command) + local prefix = '""' .. ansi_runtime .. '" --args-file "' + assert(command:sub(1, #prefix) == prefix, command) + local args_path = assert(command:match('%-%-args%-file "([^"]+)"')) + local marker = assert(command:match('&& echo (OPENBEAT_HELPER_OK_%x+)"$'), command) + seen.args_path, seen.args, seen.command = args_path, read(args_path), command + local result = behavior(command) + local output = result.output or "" + if not result.fail then output = output .. marker .. "\r\n" end + return { read = function() return output end, close = function() return true end } +end + +-- Analysis: the media path travels as UTF-8 JSON; the temporary output stays on the command line. +behavior = function(command) + local output = assert(command:match('%-%-format lua %-%-output "([^"]+)" 2>&1'), command) + assert(output:sub(1, #directory + 5) == directory .. "/tmp\\", output) + local file = assert(open_file(output, "wb")) + file:write("return { raw_beats = { 1 }, quantized_beats = { 2 } }") + file:close() + return {} +end +assert(analyze_source(media).quantized_beats[1] == 2) +assert(seen.args == '["analyze", "--audio", "C:\\\\M\195\186sica\\\\canci\195\179n.wav"]\n', seen.args) +assert(not exists(seen.args_path), "The arguments file must be removed") + +-- Click track: every argument comes from Resolve, and Lua cannot check the non-ASCII output path. +behavior = function(command) + assert(command:match('%-%-args%-file "[^"]+" 2>&1'), command) + return { output = '{"output_path": "C:\\\\M\\u00fasica\\\\canci\\u00f3n.openbeat-clicks.wav"}' } +end +local click = render_click_track(media, "raw") +assert(click == "C:\\M\195\186sica\\canci\195\179n.openbeat-raw-clicks.wav", click) +assert(seen.args == '["click-track", "--audio", "C:\\\\M\195\186sica\\\\canci\195\179n.wav", "--mode", "raw", "--output", ' + .. '"C:\\\\M\195\186sica\\\\canci\195\179n.openbeat-raw-clicks.wav"]\n', seen.args) +assert(not exists(seen.args_path)) + +-- ...but an ASCII output path is still checked, and failures still remove the arguments file. +assert(not pcall(render_click_track, "C:\\Music\\missing.wav", "raw")) +behavior = function() return { output = "LibsndfileError: Error opening", fail = true } end +local ok, message = pcall(analyze_source, media) +assert(not ok and message:find("LibsndfileError", 1, true), message) +assert(not exists(seen.args_path)) + +-- JSON escaping of quotes and control characters. +behavior = function() return {} end +pcall(run_helper, { "a\"b", "tab\tnew\nline\1" }, {}) +assert(seen.args == '["a\\"b", "tab\\tnew\\nline\\u0001"]\n', seen.args) + +-- Elsewhere the ANSI spelling is ignored and arguments stay on the command line. +package.config = posix_config +os.getenv = getenv +assert(python_bin() ~= ansi_runtime) diff --git a/tests/test_windows_paths.py b/tests/test_windows_paths.py new file mode 100644 index 0000000..05c39be --- /dev/null +++ b/tests/test_windows_paths.py @@ -0,0 +1,119 @@ +"""Non-ASCII paths on Windows, where Resolve's Lua uses the ANSI code page. + +These run on any platform: the Windows API and code page are replaced with stand-ins. +""" +import json +import re +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest import mock + +from openbeat import cli +from scripts import build_installers, staged_install + + +def lua_bytes(literal: str) -> bytes: + """Decode a Lua string literal written by ``staged_install.lua_string``.""" + assert literal.startswith('"') and literal.endswith('"') and literal.isascii() + return re.sub(rb"\\(\d{3})", lambda match: bytes([int(match.group(1))]), literal[1:-1].encode("ascii")) + + +def config_values(config: str) -> dict[str, bytes]: + return {key: lua_bytes(value) for key, value in re.findall(r'^ (\w+) = (".*"),$', config, re.MULTILINE)} + + +class RuntimeConfigTests(unittest.TestCase): + anchor = Path("C:/Users/José/AppData/Roaming/Blackmagic Design/DaVinci Resolve/Support/Fusion/Modules") + runtime = anchor / "OpenBeat" / "bin" / "openbeat" / "openbeat.exe" + + def test_ascii_runtime_needs_no_other_spelling(self) -> None: + runtime = Path("C:/Users/Jo/Fusion/Modules/OpenBeat/bin/openbeat.exe") + with mock.patch.object(staged_install, "windows_short_path", side_effect=AssertionError): + config = staged_install.runtime_config(runtime, runtime.parents[2]) + self.assertEqual(config, f'return {{\n python_bin = "{runtime.as_posix()}",\n}}\n') + + def test_short_path_spells_runtime_in_ascii(self) -> None: + short = "C:\\Users\\JOS~1\\AppData\\Roaming\\BLACKM~1\\DAVINC~1\\Support\\Fusion\\Modules" + with mock.patch.object(staged_install, "windows_short_path", return_value=short) as short_path: + config = staged_install.runtime_config(self.runtime, self.anchor, encoding="cp1252") + short_path.assert_called_once_with(str(self.anchor)) + self.assertTrue(config.isascii()) + values = config_values(config) + self.assertEqual(values["python_bin"].decode("utf-8"), self.runtime.as_posix()) + self.assertEqual( + values["python_bin_ansi"], + b"C:/Users/JOS~1/AppData/Roaming/BLACKM~1/DAVINC~1/Support/Fusion/Modules/OpenBeat/bin/openbeat/openbeat.exe", + ) + + def test_without_short_names_the_ansi_code_page_is_used(self) -> None: + # With 8.3 names disabled, GetShortPathNameW returns the long path unchanged. + for short in (None, str(self.anchor)): + with self.subTest(short=short), mock.patch.object(staged_install, "windows_short_path", return_value=short): + values = config_values(staged_install.runtime_config(self.runtime, self.anchor, encoding="cp1252")) + self.assertEqual(values["python_bin_ansi"], self.runtime.as_posix().encode("cp1252")) + self.assertIn(b"Jos\xe9", values["python_bin_ansi"]) + + def test_unrepresentable_path_falls_back_to_utf8_only(self) -> None: + anchor = Path("C:/Users/用户/Fusion/Modules") + with mock.patch.object(staged_install, "windows_short_path", return_value=None): + config = staged_install.runtime_config(anchor / "OpenBeat" / "bin" / "openbeat.exe", anchor, encoding="cp1252") + self.assertEqual(set(config_values(config)), {"python_bin"}) + # mbcs only exists on Windows; elsewhere the lookup fails the same way. + with mock.patch.object(staged_install, "windows_short_path", return_value=None): + config = staged_install.runtime_config(self.runtime, self.anchor, encoding="no-such-codec") + self.assertEqual(set(config_values(config)), {"python_bin"}) + + def test_short_path_lookup(self) -> None: + short = staged_install.windows_short_path(str(Path.cwd())) + if sys.platform == "win32": + self.assertIsInstance(short, str) + else: + self.assertIsNone(short) + + def test_install_records_ansi_runtime_path(self) -> None: + with TemporaryDirectory(prefix="openbeat-unicode-") as temporary: + work = Path(temporary) + built = work / "build" / "openbeat" / "openbeat.exe" + built.parent.mkdir(parents=True) + built.write_bytes(b"runtime") + build_installers.build_payload(work / "payload", built, "windows") + root = work / "José" / "Fusion" + short = str(work / "JOS~1" / "Fusion" / "Modules") + with mock.patch.object(staged_install, "windows_short_path", return_value=short) as short_path: + runtime = staged_install.install_payload(work / "payload", root) + short_path.assert_called_once_with(str(root / "Modules")) + values = config_values((root / "Modules" / "OpenBeat" / staged_install.CONFIG_NAME).read_text(encoding="ascii")) + self.assertEqual(values["python_bin"].decode("utf-8"), runtime.as_posix()) + relative = runtime.relative_to(root / "Modules").as_posix() + self.assertEqual(values["python_bin_ansi"].decode("ascii"), Path(short).as_posix() + "/" + relative) + + +class ArgsFileTests(unittest.TestCase): + def test_arguments_without_args_file_are_unchanged(self) -> None: + argv = ["analyze", "--audio", "song.wav"] + self.assertIs(cli.expand_args_file(argv), argv) + + def test_args_file_arguments_come_first(self) -> None: + with TemporaryDirectory(prefix="openbeat-args-") as temporary: + path = Path(temporary) / "args.json" + # Written by the Resolve scripts: UTF-8, with JSON escapes for backslashes and quotes. + path.write_bytes('["analyze", "--audio", "C:\\\\Música\\\\canción \\"1\\".wav"]\n'.encode("utf-8")) + argv = cli.expand_args_file(["--args-file", str(path), "--format", "lua", "--output", "C:\\Temp\\a.lua"]) + args = cli.build_parser().parse_args(argv) + self.assertEqual(args.command, "analyze") + self.assertEqual(args.audio, 'C:\\Música\\canción "1".wav') + self.assertEqual((args.format, args.output), ("lua", "C:\\Temp\\a.lua")) + + def test_args_file_must_list_strings(self) -> None: + with TemporaryDirectory(prefix="openbeat-args-") as temporary: + path = Path(temporary) / "args.json" + for content in ({"audio": "x"}, ["analyze", 1]): + path.write_text(json.dumps(content), encoding="utf-8") + with self.subTest(content=content), self.assertRaises(SystemExit): + cli.expand_args_file(["--args-file", str(path)]) + + +if __name__ == "__main__": + unittest.main()