diff --git a/.github/workflows/moocore.yml b/.github/workflows/moocore.yml index 0387791..f18e657 100644 --- a/.github/workflows/moocore.yml +++ b/.github/workflows/moocore.yml @@ -101,3 +101,45 @@ jobs: name: failed-tests-${{ matrix.os }}-${{ matrix.cc }}-${{ github.run_id }} path: failed-test-outputs/ if-no-files-found: ignore + + test-windows: + if: "! contains(github.event.head_commit.message, '[skip ci]')" + name: Run testsuite on windows-latest with MSVC + runs-on: windows-latest + env: + PYTHONUTF8: 1 + steps: + - name: Checkout moocore + uses: actions/checkout@v6 + with: + repository: multi-objective/moocore + sparse-checkout: c + + - name: Checkout testsuite + uses: actions/checkout@v6 + with: + path: testsuite + + - uses: ilammy/msvc-dev-cmd@v1 + + - uses: actions/setup-python@v6 + with: + python-version: '3.x' + + - name: Install dependencies + run: python -m pip install -r testsuite/requirements.txt + + - name: Build moocore (MSVC) + shell: bash + run: make -C c all CC=cl DEBUG=1 + + - name: Run testsuite + # Default shell (PowerShell) on purpose: no Unix tools are required. + run: | + foreach ($t in "dominatedsets","eaf","epsilon","hv","hvapprox","igd","ndsort","nondominated") { + Push-Location "testsuite/$t" + python ../regtest.py "$env:GITHUB_WORKSPACE/bin/$t" + $rc = $LASTEXITCODE + Pop-Location + if ($rc -ne 0) { exit $rc } + } diff --git a/README.md b/README.md index 2e2ab99..7819f31 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,38 @@ This is a testsuite for [`moocore`](https://github.com/multi-objective/moocore). [testsuite-badge]: https://github.com/multi-objective/testsuite/actions/workflows/moocore.yml/badge.svg [testsuite-link]: https://github.com/multi-objective/testsuite/actions/workflows/moocore.yml + +Running the testsuite +--------------------- + +```bash +pip install -r requirements.txt +cd hv && ../regtest.py ../../moocore/bin/hv +``` + +`regtest.py PROGRAM [TESTS]` runs the `*.test` files in the current directory (recursively) using `PROGRAM`. +The `*.test` files do not call platform-specific utilities directly but use the environment variables +`$XZCAT`, `$SORT`, `$CAT_SQUEEZE` and `$MKTEMP`. On Linux/macOS they default to `xzcat`, `sort`, `cat -s` and `mktemp` +and can be overridden from the environment. + +Windows +------- + +The testsuite runs on Windows with the executables of `moocore` built with MSVC. +`regtest.py` only needs Python and `joblib` (see `requirements.txt`); no Unix utilities are required. +On Windows, `regtest.py`: + + * runs the `*.test` files with a built-in minimal interpreter (variables, `|`, `<`, `>`, `echo`, `rm`) instead of a shell, and + * uses the portable Python scripts in [`tools/`](tools) for `$XZCAT`, `$SORT`, `$CAT_SQUEEZE` and `$MKTEMP`. + +moocore's makefile adds the `.exe` extension to the executables, and `regtest.py` appends it if missing, so these are equivalent: + +```bat +rem From a Visual Studio developer prompt (or after ilammy/msvc-dev-cmd) +make -C moocore\c all CC=cl +cd testsuite\hv +python ..\regtest.py ..\..\moocore\bin\hv +python ..\regtest.py ..\..\moocore\bin\hv.exe +``` + +Set `REGTEST_PORTABLE=1` to use this Windows code path on Linux/macOS (useful for debugging). diff --git a/eaf/eafdiff-ALG_1_dat_ALG_2_dat.test b/eaf/eafdiff-ALG_1_dat_ALG_2_dat.test index 1d6cb3d..59e4da2 100644 --- a/eaf/eafdiff-ALG_1_dat_ALG_2_dat.test +++ b/eaf/eafdiff-ALG_1_dat_ALG_2_dat.test @@ -1,6 +1,6 @@ -file1=$(mktemp) -file2=$(mktemp) -xzcat ../data/ALG_1_dat.xz > $file1 -xzcat ../data/ALG_2_dat.xz > $file2 +file1=$($MKTEMP) +file2=$($MKTEMP) +${XZCAT} ../data/ALG_1_dat.xz > $file1 +${XZCAT} ../data/ALG_2_dat.xz > $file2 ${PROGRAM} -d $file1 $file2 rm -f $file1 $file2 diff --git a/eaf/input1.test b/eaf/input1.test index bd06482..929ad66 100644 --- a/eaf/input1.test +++ b/eaf/input1.test @@ -1 +1 @@ -${PROGRAM} --quiet -i- ../data/input1.dat | sort | cat -s +${PROGRAM} --quiet -i- ../data/input1.dat | ${SORT} | ${CAT_SQUEEZE} diff --git a/eaf/ran.1000pts.3d.10.test b/eaf/ran.1000pts.3d.10.test index a67b5b1..c143427 100644 --- a/eaf/ran.1000pts.3d.10.test +++ b/eaf/ran.1000pts.3d.10.test @@ -1 +1 @@ -${PROGRAM} --quiet ../data/ran.1000pts.3d.10 | sort | cat -s +${PROGRAM} --quiet ../data/ran.1000pts.3d.10 | ${SORT} | ${CAT_SQUEEZE} diff --git a/eaf/spherical-3d-2000pts-10.test b/eaf/spherical-3d-2000pts-10.test index 5865316..56076f8 100644 --- a/eaf/spherical-3d-2000pts-10.test +++ b/eaf/spherical-3d-2000pts-10.test @@ -1 +1 @@ -xzcat ../data/${TESTNAME}.dat.xz | $PROGRAM --quiet | sort -s | cat -s +${XZCAT} ../data/${TESTNAME}.dat.xz | $PROGRAM --quiet | ${SORT} -s | ${CAT_SQUEEZE} diff --git a/epsilon/rmnk_0.0_10_16_1_0_random_search_1.txt.test b/epsilon/rmnk_0.0_10_16_1_0_random_search_1.txt.test index f12b9a0..744366c 100644 --- a/epsilon/rmnk_0.0_10_16_1_0_random_search_1.txt.test +++ b/epsilon/rmnk_0.0_10_16_1_0_random_search_1.txt.test @@ -1,5 +1,5 @@ -reference=$(mktemp) -xzcat ../data/rmnk_0.0_10_16_1_0_ref.txt.xz > $reference -xzcat ../data/${TESTNAME}.xz | ${PROGRAM} -r $reference --additive -xzcat ../data/${TESTNAME}.xz | ${PROGRAM} -r $reference --additive --maximise +reference=$($MKTEMP) +${XZCAT} ../data/rmnk_0.0_10_16_1_0_ref.txt.xz > $reference +${XZCAT} ../data/${TESTNAME}.xz | ${PROGRAM} -r $reference --additive +${XZCAT} ../data/${TESTNAME}.xz | ${PROGRAM} -r $reference --additive --maximise rm -f $reference diff --git a/hv/ran.40000pts.3d.1.test b/hv/ran.40000pts.3d.1.test index a7c86c0..bbc7885 100644 --- a/hv/ran.40000pts.3d.1.test +++ b/hv/ran.40000pts.3d.1.test @@ -1 +1 @@ -xzcat ../data/${TESTNAME}.xz | ${PROGRAM} -r "10 10 10" --union +${XZCAT} ../data/${TESTNAME}.xz | ${PROGRAM} -r "10 10 10" --union diff --git a/hv/ran.40001pts.3d.1.test b/hv/ran.40001pts.3d.1.test index a7c86c0..bbc7885 100644 --- a/hv/ran.40001pts.3d.1.test +++ b/hv/ran.40001pts.3d.1.test @@ -1 +1 @@ -xzcat ../data/${TESTNAME}.xz | ${PROGRAM} -r "10 10 10" --union +${XZCAT} ../data/${TESTNAME}.xz | ${PROGRAM} -r "10 10 10" --union diff --git a/hv/rep5d.test b/hv/rep5d.test index 98e46cb..15c1c3e 100644 --- a/hv/rep5d.test +++ b/hv/rep5d.test @@ -1 +1 @@ -xzcat ${TESTNAME}.inp.xz | $PROGRAM -r "-14300 -14300 -14300 -14300 -14300" +${XZCAT} ${TESTNAME}.inp.xz | $PROGRAM -r "-14300 -14300 -14300 -14300 -14300" diff --git a/hv/test2D-200k.test b/hv/test2D-200k.test index 45c25f1..0e47668 100644 --- a/hv/test2D-200k.test +++ b/hv/test2D-200k.test @@ -1 +1 @@ -xzcat ../data/${TESTNAME}.inp.xz | $PROGRAM -r "-117751 -117751" +${XZCAT} ../data/${TESTNAME}.inp.xz | $PROGRAM -r "-117751 -117751" diff --git a/igd/ALG_1_ALG_2.test b/igd/ALG_1_ALG_2.test index ba1af0a..438a14f 100644 --- a/igd/ALG_1_ALG_2.test +++ b/igd/ALG_1_ALG_2.test @@ -1,7 +1,7 @@ -tmpfile1=$(mktemp -u) -tmpfile2=$(mktemp -u) -xzcat ../data/ALG_1_dat.xz > "$tmpfile1" -xzcat ../data/ALG_2_dat.xz > "$tmpfile2" +tmpfile1=$($MKTEMP -u) +tmpfile2=$($MKTEMP -u) +${XZCAT} ../data/ALG_1_dat.xz > "$tmpfile1" +${XZCAT} ../data/ALG_2_dat.xz > "$tmpfile2" echo "# (1,2)" ${PROGRAM} --all -r "$tmpfile1" "$tmpfile2" ${PROGRAM} --all --maximise -r "$tmpfile1" "$tmpfile2" diff --git a/igd/spherical-3d-2000pts-10.test b/igd/spherical-3d-2000pts-10.test index caeabcd..34e5183 100644 --- a/igd/spherical-3d-2000pts-10.test +++ b/igd/spherical-3d-2000pts-10.test @@ -1,4 +1,4 @@ -tmpfile=$(mktemp -u) -xzcat ../data/${TESTNAME}.dat.xz > "$tmpfile" +tmpfile=$($MKTEMP -u) +${XZCAT} ../data/${TESTNAME}.dat.xz > "$tmpfile" ${PROGRAM} --all -r "$tmpfile" "$tmpfile" rm -f "$tmpfile" diff --git a/nondominated/DTLZDiscontinuousShape.3d.front.1000pts.10.test b/nondominated/DTLZDiscontinuousShape.3d.front.1000pts.10.test index 35e48ff..c6e216b 100644 --- a/nondominated/DTLZDiscontinuousShape.3d.front.1000pts.10.test +++ b/nondominated/DTLZDiscontinuousShape.3d.front.1000pts.10.test @@ -1 +1 @@ -$PROGRAM < ../data/${TESTNAME} --union --quiet --filter | sort -b +$PROGRAM < ../data/${TESTNAME} --union --quiet --filter | ${SORT} -b diff --git a/nondominated/DTLZLinearShape.3d.front.1000pts.10.test b/nondominated/DTLZLinearShape.3d.front.1000pts.10.test index 35e48ff..c6e216b 100644 --- a/nondominated/DTLZLinearShape.3d.front.1000pts.10.test +++ b/nondominated/DTLZLinearShape.3d.front.1000pts.10.test @@ -1 +1 @@ -$PROGRAM < ../data/${TESTNAME} --union --quiet --filter | sort -b +$PROGRAM < ../data/${TESTNAME} --union --quiet --filter | ${SORT} -b diff --git a/nondominated/DTLZLinearShape.6d.front.700pts.10.test b/nondominated/DTLZLinearShape.6d.front.700pts.10.test index 22d5968..af2fc94 100644 --- a/nondominated/DTLZLinearShape.6d.front.700pts.10.test +++ b/nondominated/DTLZLinearShape.6d.front.700pts.10.test @@ -1 +1 @@ -xzcat ../data/${TESTNAME}.xz | $PROGRAM --union --quiet --filter | sort -b +${XZCAT} ../data/${TESTNAME}.xz | $PROGRAM --union --quiet --filter | ${SORT} -b diff --git a/nondominated/DTLZSphereShape.3d.front.1000pts.10.test b/nondominated/DTLZSphereShape.3d.front.1000pts.10.test index 35e48ff..c6e216b 100644 --- a/nondominated/DTLZSphereShape.3d.front.1000pts.10.test +++ b/nondominated/DTLZSphereShape.3d.front.1000pts.10.test @@ -1 +1 @@ -$PROGRAM < ../data/${TESTNAME} --union --quiet --filter | sort -b +$PROGRAM < ../data/${TESTNAME} --union --quiet --filter | ${SORT} -b diff --git a/nondominated/DTLZSphereShape.5d.front.1000pts.10.test b/nondominated/DTLZSphereShape.5d.front.1000pts.10.test index 35e48ff..c6e216b 100644 --- a/nondominated/DTLZSphereShape.5d.front.1000pts.10.test +++ b/nondominated/DTLZSphereShape.5d.front.1000pts.10.test @@ -1 +1 @@ -$PROGRAM < ../data/${TESTNAME} --union --quiet --filter | sort -b +$PROGRAM < ../data/${TESTNAME} --union --quiet --filter | ${SORT} -b diff --git a/nondominated/DTLZSphereShape.6d.front.1000pts.10.test b/nondominated/DTLZSphereShape.6d.front.1000pts.10.test index dc423ba..bab99d6 100644 --- a/nondominated/DTLZSphereShape.6d.front.1000pts.10.test +++ b/nondominated/DTLZSphereShape.6d.front.1000pts.10.test @@ -1 +1 @@ -xzcat ../data/${TESTNAME}.xz | $PROGRAM --union --quiet --filter | sort -b +${XZCAT} ../data/${TESTNAME}.xz | $PROGRAM --union --quiet --filter | ${SORT} -b diff --git a/nondominated/ran.1000pts.3d.10.test b/nondominated/ran.1000pts.3d.10.test index 321a9ac..2d02fc4 100644 --- a/nondominated/ran.1000pts.3d.10.test +++ b/nondominated/ran.1000pts.3d.10.test @@ -1 +1 @@ -xzcat ../eaf/${TESTNAME}.exp.xz | $PROGRAM --union --quiet --filter | sort -b +${XZCAT} ../eaf/${TESTNAME}.exp.xz | $PROGRAM --union --quiet --filter | ${SORT} -b diff --git a/nondominated/ran.1300pts.5d.10.test b/nondominated/ran.1300pts.5d.10.test index d5b92ba..5f8aaa9 100644 --- a/nondominated/ran.1300pts.5d.10.test +++ b/nondominated/ran.1300pts.5d.10.test @@ -1 +1 @@ -xzcat ../data/${TESTNAME}.xz | $PROGRAM --maximise --union --quiet --filter | sort -b +${XZCAT} ../data/${TESTNAME}.xz | $PROGRAM --maximise --union --quiet --filter | ${SORT} -b diff --git a/nondominated/ran.40000pts.3d.1.test b/nondominated/ran.40000pts.3d.1.test index 2ce61bf..5ef29e2 100644 --- a/nondominated/ran.40000pts.3d.1.test +++ b/nondominated/ran.40000pts.3d.1.test @@ -1 +1 @@ -xzcat ../data/${TESTNAME}.xz | $PROGRAM --verbose +${XZCAT} ../data/${TESTNAME}.xz | $PROGRAM --verbose diff --git a/nondominated/ran.9000pts.4d.10.test b/nondominated/ran.9000pts.4d.10.test index d5b92ba..5f8aaa9 100644 --- a/nondominated/ran.9000pts.4d.10.test +++ b/nondominated/ran.9000pts.4d.10.test @@ -1 +1 @@ -xzcat ../data/${TESTNAME}.xz | $PROGRAM --maximise --union --quiet --filter | sort -b +${XZCAT} ../data/${TESTNAME}.xz | $PROGRAM --maximise --union --quiet --filter | ${SORT} -b diff --git a/nondominated/rmnk_0.0_10_16_1_0_random_search_1.txt.test b/nondominated/rmnk_0.0_10_16_1_0_random_search_1.txt.test index 5e7536b..9130259 100644 --- a/nondominated/rmnk_0.0_10_16_1_0_random_search_1.txt.test +++ b/nondominated/rmnk_0.0_10_16_1_0_random_search_1.txt.test @@ -1 +1 @@ -xzcat ../data/${TESTNAME}.xz | $PROGRAM --union --verbose +${XZCAT} ../data/${TESTNAME}.xz | $PROGRAM --union --verbose diff --git a/nondominated/test2D-200k.test b/nondominated/test2D-200k.test index 2058d84..2faf410 100644 --- a/nondominated/test2D-200k.test +++ b/nondominated/test2D-200k.test @@ -1 +1 @@ -xzcat ../data/${TESTNAME}.inp.xz | $PROGRAM --quiet --filter | sort -b -s +${XZCAT} ../data/${TESTNAME}.inp.xz | $PROGRAM --quiet --filter | ${SORT} -b -s diff --git a/regtest.py b/regtest.py index 5b2bb05..da54ff1 100755 --- a/regtest.py +++ b/regtest.py @@ -5,10 +5,13 @@ import glob import time import tempfile +import shlex from itertools import zip_longest -from joblib import Parallel, delayed +from pathlib import Path from rich import print as print_rich +from joblib import Parallel, delayed + import re import difflib import lzma @@ -16,6 +19,53 @@ # debug = True will use the command-line diff (or xzdiff) to check the results of difflib. debug = False +IS_WINDOWS = sys.platform == "win32" +# Set REGTEST_PORTABLE=1 to use the Windows code path on other platforms. +USE_PORTABLE = IS_WINDOWS or os.getenv("REGTEST_PORTABLE") == "1" + +# Make sure that Python subprocesses and joblib workers use UTF-8. +os.environ.setdefault("PYTHONUTF8", "1") +os.environ.setdefault("PYTHONIOENCODING", "utf-8") + + +def setup_utf8_output(): + """Avoid console-encoding crashes when printing unicode symbols.""" + for stream in (sys.stdout, sys.stderr): + if hasattr(stream, "reconfigure"): + try: + stream.reconfigure(encoding="utf-8", errors="replace") + except (OSError, ValueError): + pass + + +setup_utf8_output() + +TOOLS_DIR = Path(__file__).resolve().parent / "tools" + + +def _python_tool(script): + return shlex.join([Path(sys.executable).as_posix(), (TOOLS_DIR / script).as_posix()]) + + +# Commands used by the *.test files. Unix defaults are the standard utilities, +# on Windows they are portable Python scripts from tools/. They can be +# overridden by setting the corresponding environment variable. +if USE_PORTABLE: + _TOOL_DEFAULTS = { + "XZCAT": _python_tool("xzcat.py"), + "SORT": _python_tool("sort.py"), + "CAT_SQUEEZE": _python_tool("cat_squeeze.py"), + "MKTEMP": _python_tool("mktemp.py"), + } +else: + _TOOL_DEFAULTS = { + "XZCAT": "xzcat", + "SORT": "sort", + "CAT_SQUEEZE": "cat -s", + "MKTEMP": "mktemp", + } +TOOL_ENV = {k: os.environ.get(k, v) for k, v in _TOOL_DEFAULTS.items()} + _RE_COMBINE_WHITESPACE = re.compile(r"\s+") ASAN_OPTIONS = os.getenv("ASAN_OPTIONS", "") @@ -112,6 +162,140 @@ def truncate_lines(output: str, max_lines: int) -> str: return "\n".join(lines[:max_lines] + ["...truncated"]) +_RE_VAR = re.compile(r"\$(?:(\w+)|\{(\w+)\})") +_RE_ASSIGN = re.compile(r"^(\w+)=(.*)$") +_RE_SUBST = re.compile(r"^\$\((.*)\)$") + + +def _expand(word, variables): + return _RE_VAR.sub(lambda m: variables.get(m.group(1) or m.group(2), ""), word) + + +def _split_line(line): + lex = shlex.shlex(line, posix=True, punctuation_chars="|<>") + lex.whitespace_split = True + lex.commenters = "" + return list(lex) + + +def _run_pipeline(line, variables, cwd, env): + """Run one simple pipeline of a test file (a tiny POSIX shell subset). + + Supports variable expansion, '|', '<', '>', and the builtins echo and rm. + Returns the combined stdout/stderr of the pipeline as bytes. + """ + stages = [[]] + stdin_file = stdout_file = None + tokens = _split_line(line) + i = 0 + while i < len(tokens): + tok = tokens[i] + if tok == "|": + stages.append([]) + elif tok in ("<", ">"): + i += 1 + target = _expand(tokens[i], variables) + if tok == "<": + stdin_file = target + else: + stdout_file = target + else: + stages[-1].append(tok) + i += 1 + + argvs = [] + for words in stages: + argv = [] + for j, w in enumerate(words): + expanded = _expand(w, variables) + if j == 0 and w.startswith("$") and expanded.strip(): + argv.extend(shlex.split(expanded)) # command stored in a variable + else: + argv.append(expanded) + argvs.append(argv) + + if len(argvs) == 1 and argvs[0] and argvs[0][0] in ("echo", "rm"): + argv = argvs[0] + if argv[0] == "echo": + out = (" ".join(argv[1:]) + "\n").encode("utf-8") + if stdout_file: + Path(cwd or ".", stdout_file).write_bytes(out) + return b"" + return out + for name in (a for a in argv[1:] if not a.startswith("-")): + try: + os.remove(os.path.join(cwd or ".", name)) + except OSError: + pass + return b"" + + opened = [] + procs = [] + errs = [] + try: + prev_out = None + if stdin_file: + prev_out = open(os.path.join(cwd or ".", stdin_file), "rb") + opened.append(prev_out) + for k, argv in enumerate(argvs): + last = k == len(argvs) - 1 + if last and stdout_file: + stdout = open(os.path.join(cwd or ".", stdout_file), "wb") + opened.append(stdout) + else: + stdout = subprocess.PIPE + if last: + stderr = subprocess.STDOUT + else: + stderr = tempfile.TemporaryFile() + errs.append(stderr) + p = subprocess.Popen( + argv, stdin=prev_out, stdout=stdout, stderr=stderr, cwd=cwd, env=env + ) + if k > 0: + procs[-1].stdout.close() + procs.append(p) + prev_out = p.stdout + out = procs[-1].communicate()[0] or b"" + for p in procs[:-1]: + p.wait() + extra = b"" + for e in errs: + e.seek(0) + extra += e.read() + e.close() + return extra + out + finally: + for f in opened: + f.close() + + +def run_script_portable(script, cwd, env): + """Execute a *.test file without a POSIX shell (used on Windows).""" + variables = {k: env[k] for k in env} + output = b"" + with open(os.path.join(cwd or ".", script), "r", encoding="utf-8") as fh: + lines = fh.read().splitlines() + for line in lines: + line = line.strip() + if not line or line.startswith("#"): + continue + m = _RE_ASSIGN.match(line) + if m and " " not in m.group(1): + name, value = m.groups() + sub = _RE_SUBST.match(value) + if sub: + value = _run_pipeline(sub.group(1), variables, cwd, env) + value = value.decode("utf-8").rstrip("\r\n") + else: + value = "".join(_split_line(value)) if value else "" + value = _expand(value, variables) + variables[name] = value + continue + output += _run_pipeline(line, variables, cwd, env) + return output + + def runcmd(command, cwd=None, env=None, outfile=subprocess.DEVNULL): # Merge with os.environ to preserve PATH and other environment variables full_env = os.environ.copy() @@ -120,6 +304,16 @@ def runcmd(command, cwd=None, env=None, outfile=subprocess.DEVNULL): full_env["LC_ALL"] = "C" # To avoid problems with sorting. full_env["ASAN_OPTIONS"] = ASAN_OPTIONS full_env["UBSAN_OPTIONS"] = UBSAN_OPTIONS + full_env.update(TOOL_ENV) + full_env["PYTHONUTF8"] = "1" + full_env["PYTHONIOENCODING"] = "utf-8" + if USE_PORTABLE and command.startswith(". ./"): + data = run_script_portable(command[4:], cwd, full_env) + if outfile != subprocess.DEVNULL: + open_func = lzma.open if outfile.endswith(".xz") else open + with open_func(outfile, "wb") as fh: + fh.write(data) + return 0 stdout = outfile if outfile == subprocess.DEVNULL else subprocess.PIPE result = subprocess.run( command, @@ -139,6 +333,18 @@ def runcmd(command, cwd=None, env=None, outfile=subprocess.DEVNULL): return result.returncode +def find_exe(fpath): + """Return the path of the executable, adding '.exe' on Windows if needed. + + moocore's makefile appends '.exe' to the executables it builds. + """ + fpath = os.path.realpath(os.path.expanduser(fpath)) + if IS_WINDOWS and not fpath.lower().endswith(".exe"): + if os.path.isfile(fpath + ".exe") or not os.path.isfile(fpath): + fpath += ".exe" + return fpath + + def is_exe(fpath): fpath = os.path.expanduser(fpath) return ( @@ -149,6 +355,9 @@ def is_exe(fpath): def run_test(test, program): + setup_utf8_output() + if USE_PORTABLE: + program = Path(program).as_posix() print("{:<60}".format("Running " + test + " :"), end=" ") testdirname = os.path.dirname(test) if testdirname == "": @@ -194,7 +403,7 @@ def main(): print("\t for example:", sys.argv[0], "../bin/hv") sys.exit(1) - program = os.path.realpath(os.path.expanduser(sys.argv[1])) + program = find_exe(sys.argv[1]) if not is_exe(program): print(f"error: '{program}' not found or not executable!") sys.exit(1) @@ -214,7 +423,13 @@ def main(): ntotal = len(tests) elapsed_time = time.time() - ok = Parallel(n_jobs=-2)(delayed(run_test)(test, program=program) for test in tests) + # The loky backend starts worker processes with spawn semantics on every + # platform (as required on Windows), so run_test is a top-level function + # and all its arguments are picklable. Workers inherit PYTHONUTF8 and + # PYTHONIOENCODING from the environment set at import time. + ok = Parallel(n_jobs=-2, backend="loky")( + delayed(run_test)(test, program=program) for test in tests + ) elapsed_time = time.time() - elapsed_time npassed = sum(ok) nfailed = ntotal - npassed diff --git a/tools/cat_squeeze.py b/tools/cat_squeeze.py new file mode 100644 index 0000000..a9af2fd --- /dev/null +++ b/tools/cat_squeeze.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Portable replacement for ``cat --squeeze-blank`` (``cat -s``).""" +import sys + + +def main(files): + out = sys.stdout.buffer + previous_blank = False + for name in files or ["-"]: + if name == "-": + f = sys.stdin.buffer + else: + f = open(name, "rb") + for line in f: + blank = line.rstrip(b"\r\n") == b"" + if blank and previous_blank: + continue + previous_blank = blank + out.write(line) + if f is not sys.stdin.buffer: + f.close() + out.flush() + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/mktemp.py b/tools/mktemp.py new file mode 100644 index 0000000..26b9872 --- /dev/null +++ b/tools/mktemp.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Portable replacement for ``mktemp [-u]``: print a temporary file name.""" +import os +import sys +import tempfile +from pathlib import Path + + +def main(args): + if args and args[0] not in ("-u", "--dry-run"): + sys.exit(f"mktemp.py: unsupported option {args[0]}") + if args: + # Name only: do not create the file. + path = Path(tempfile.gettempdir()) / ("tmp." + os.urandom(6).hex()) + else: + fd, name = tempfile.mkstemp() + os.close(fd) + path = Path(name) + print(path.as_posix()) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/sort.py b/tools/sort.py new file mode 100644 index 0000000..470c685 --- /dev/null +++ b/tools/sort.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Portable replacement for ``sort`` (LC_ALL=C order) supporting +``-b/--ignore-leading-blanks`` and ``-s/--stable``.""" +import sys + + +def main(args): + blanks = stable = False + files = [] + for a in args: + if a == "--ignore-leading-blanks": + blanks = True + elif a == "--stable": + stable = True + elif a.startswith("-") and not a.startswith("--") and len(a) > 1: + for c in a[1:]: + if c == "b": + blanks = True + elif c == "s": + stable = True + else: + sys.exit(f"sort.py: unsupported option -{c}") + else: + files.append(a) + + data = b"" + if not files: + files = ["-"] + for name in files: + if name == "-": + data += sys.stdin.buffer.read() + else: + with open(name, "rb") as f: + data += f.read() + + lines = data.split(b"\n") + if lines and lines[-1] == b"": + lines.pop() + + def key(line): + k = line.rstrip(b"\r") + return k.lstrip(b" \t") if blanks else k + + if stable: + lines.sort(key=key) + else: + # Like GNU sort, fall back to comparing the whole line. + lines.sort(key=lambda x: (key(x), x.rstrip(b"\r"))) + out = sys.stdout.buffer + out.write(b"".join(line + b"\n" for line in lines)) + out.flush() + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/xzcat.py b/tools/xzcat.py new file mode 100644 index 0000000..2d537ec --- /dev/null +++ b/tools/xzcat.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Portable replacement for ``xzcat``: decompress FILEs (or stdin) to stdout.""" +import lzma +import shutil +import sys + + +def main(files): + out = sys.stdout.buffer + if not files: + files = ["-"] + for name in files: + if name == "-": + shutil.copyfileobj(lzma.LZMAFile(sys.stdin.buffer), out) + else: + with lzma.open(name, "rb") as f: + shutil.copyfileobj(f, out) + out.flush() + + +if __name__ == "__main__": + main(sys.argv[1:])