diff --git a/.github/workflows/python-linux.yml b/.github/workflows/python-linux.yml index 3aa7d58d..c6443d1e 100644 --- a/.github/workflows/python-linux.yml +++ b/.github/workflows/python-linux.yml @@ -131,8 +131,13 @@ jobs: echo "No pkg-config-visible system Abseil" fi - - name: Install locked environment (T1 + T2 voice) - run: uv sync --locked --extra voice --extra dev + - name: Qualify cold locked Voice install and standalone AEC wheel + # Builds an sdist/wheel as well as the locked install, verifies actual + # Meson resolution, and detects ignored build/install source pollution. + env: + # Regression: the qualifier must prevent an inherited uv log filter. + RUST_LOG: warn + run: python tools/verify_aec_source_build.py - name: Verify environment (voice profile) run: uv run --locked --no-sync python tools/verify_python_environment.py --profile voice @@ -147,25 +152,6 @@ jobs: for n in ('torch', 'torchaudio', 'silero_vad', 'onnxruntime')); print('AEC 1.0.1 import and model-free Voice: PASS')" - - name: Assert bundled Abseil build - shell: bash - run: | - uv run --locked --no-sync python - <<'PYTHON' - import json - from pathlib import Path - - info = Path("vendor/aec-audio-processing/webrtc-audio-processing/build/meson-info") - options = { - item["name"]: item["value"] - for item in json.loads((info / "intro-buildoptions.json").read_text()) - } - assert "abseil-cpp" in options["force_fallback_for"], options["force_fallback_for"] - project = json.loads((info / "intro-projectinfo.json").read_text()) - subprojects = {item["name"]: item["version"] for item in project["subprojects"]} - assert subprojects.get("abseil-cpp") == "20240722.0", subprojects - print("AEC build: forced bundled Abseil 20240722.0") - PYTHON - - name: Check third-party provenance run: uv run --locked --no-sync python tools/check_third_party_provenance.py --release-ready @@ -179,6 +165,7 @@ jobs: tests/test_voice_backend_registry.py tests/test_mic_descriptor.py tests/test_aec_native_smoke.py + tests/test_aec_build_contract.py tests/test_playback_device_recovery.py tests/test_tts_contract.py tests/test_tts_fish_audio_backend.py diff --git a/LICENSES/provenance.json b/LICENSES/provenance.json index 28671d21..aa4e7acd 100644 --- a/LICENSES/provenance.json +++ b/LICENSES/provenance.json @@ -385,7 +385,7 @@ "modified": true, "release_action": "include", "gate_status": "ready", - "notes": "Linux-only path source. The sole upstream source patch forces bundled Abseil in Meson setup. Original nested notices are retained and copied into wheel package data. See the adjacent provenance and patch files for verification and removal conditions." + "notes": "Linux-only path source. The build-only patch forces and verifies bundled Abseil, isolates setuptools/Meson writes in a disposable source copy, checks wrapper/proxy version consistency before native work, and compiles the committed SWIG bindings. Original generated bindings, runtime sources and nested notices are retained. See the adjacent provenance and complete patch for verification and removal conditions." }, { "id": "bokuweb-zstd-wasm", diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 98e8b6c8..b2694082 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -19,7 +19,10 @@ the cu124 dependency audit and future SBOM rather than duplicated here. ### aec-audio-processing (Linux source build) - Baseline: official PyPI `aec-audio-processing` 1.0.1 sdist. -- Local change: one Meson argument forces bundled Abseil for this build. +- Local build changes: force and verify bundled Abseil, stage setuptools/Meson + writes in a disposable source copy, and compile the committed SWIG bindings. + A pre-build check rejects inconsistent or unreadable wrapper/proxy versions, + including local files regenerated by an earlier installation. - Provenance and removal: `vendor/aec-audio-processing.PROVENANCE.md`. - Complete bundled license/attribution texts are preserved in the source tree and in `vendor/aec-audio-processing/src/files/THIRD_PARTY_NOTICES.txt`, which diff --git a/docs/alpha-0.17.1.md b/docs/alpha-0.17.1.md index 0a83beb0..eb661ba5 100644 --- a/docs/alpha-0.17.1.md +++ b/docs/alpha-0.17.1.md @@ -5,6 +5,22 @@ Baseline: `v0.17.0-alpha.0`. This remains a **source Alpha release**. Desktop installers, voice/model weights, character media, reference recordings and credentials are not included in the source ZIP. +## Post-release Linux Voice upgrade note (#249) + +The AEC build repair in #249 is a follow-up and is **not included in the +`v0.17.1-alpha.0` tag**. Older Linux builds may have left locally regenerated +SWIG files and `vendor/aec-audio-processing/webrtc-audio-processing/install/`. +After updating to the fixed revision, back up intentional vendor source edits, +restore the committed `vendor/aec-audio-processing/src` bindings, and remove +that obsolete install directory after checking its contents. Mixed wrapper/proxy +versions now fail before native build work; no files are restored or deleted +automatically. See the [Linux upgrade steps](install_profiles.md#upgrading-an-existing-linux-voice-checkout). +The reporter completed the original Arch/system-Abseil-20260817 acceptance on +`94b584c`, including a separate clean-source/cache/environment retest. See the +[Arch acceptance report](https://github.com/Code-Amadeus/Amadeus/issues/247#issuecomment-6310492090) +for the tested environment and scope; this does not establish all-distribution +or audio-device qualification. + ## Application and dictionary in one Release The application Release contains its source ZIP, source manifest, checksums and diff --git a/docs/install_profiles.md b/docs/install_profiles.md index ea8b0402..69cc2b5e 100644 --- a/docs/install_profiles.md +++ b/docs/install_profiles.md @@ -104,15 +104,47 @@ code, weights, configured interpreter selection, or GPU qualification. On Ubuntu 24.04, install `build-essential`, `pkg-config` and `portaudio19-dev`, then run `uv sync --locked --extra voice` and the `--profile voice` verifier. -The Linux-only AEC source override uses the official 1.0.1 sdist with one Meson -argument forcing bundled Abseil 20240722.0. It does not modify system Abseil or -change the Windows/macOS registry source. See [AEC provenance](../vendor/aec-audio-processing.PROVENANCE.md) +The Linux-only AEC source override uses the official 1.0.1 sdist, forces and +verifies bundled Abseil 20240722.0, and stages setuptools/Meson writes in a +disposable source copy. It compiles the committed SWIG wrapper/proxies after +checking their version consistency; installation no longer invokes SWIG. +It does not modify system Abseil or change the Windows/macOS registry source. +See [AEC provenance](../vendor/aec-audio-processing.PROVENANCE.md) for the artifact hash, patch, notices and removal conditions. The Linux Voice CI uses a fresh uv cache, builds the path dependency and checks the actual Meson options/subproject version. It qualifies this source-build path on Ubuntu 24.04, not every Linux toolchain or real audio-device behavior. +### Upgrading an existing Linux Voice checkout + +An older build may have regenerated only `audio_processing_wrap.cpp` and +`webrtc_audio_processing.py`, leaving the public `audio_processing.py` at a +different SWIG version. Updating Git preserves those local edits. The new +build rejects this mismatch before native library lookup or compilation, +rather than overwriting the user's files. + +Inspect and back up any intentional changes under the vendor `src/` directory. +From the repository root **after updating to the fixed revision**, restore the +committed bindings: + +```bash +git restore --source=HEAD -- vendor/aec-audio-processing/src +``` + +Remove the old generated +`vendor/aec-audio-processing/webrtc-audio-processing/install/` directory after +confirming it contains only the previous build's output. The new backend +excludes old build/install directories from staging and does not delete them +from the checkout. Then rerun `uv sync --locked --extra voice` and the Voice +verifier. For a source archive, unpack a fresh fixed archive instead of using +`git restore`. + +For source-build qualification, run +`UV_CACHE_DIR="$(mktemp -d)" python3.12 tools/verify_aec_source_build.py`. +The tool requires Python 3.12 and pins the locked install, both distribution +builds, and the isolated wheel environment to that interpreter. + ## Optional model interpreters and community configurations A single default environment does not prohibit isolated model processes. Qwen ASR diff --git a/tests/test_aec_build_contract.py b/tests/test_aec_build_contract.py new file mode 100644 index 00000000..9f256079 --- /dev/null +++ b/tests/test_aec_build_contract.py @@ -0,0 +1,271 @@ +"""The vendored build must not write back or mix SWIG generations.""" + +from __future__ import annotations + +import importlib.util +import os +from pathlib import Path +import runpy +import shutil +import sys +import tarfile +from types import SimpleNamespace +import zipfile + +import pytest +import setuptools + +from tools.verify_aec_source_build import assert_source_unchanged, check_wheel, source_snapshot + + +ROOT = Path(__file__).resolve().parents[1] +VENDOR = ROOT / "vendor" / "aec-audio-processing" + + +def load_backend(): + spec = importlib.util.spec_from_file_location("aec_build_backend", VENDOR / "build_backend.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_setup(monkeypatch): + configuration = {} + monkeypatch.setattr(setuptools, "setup", lambda **kwargs: configuration.update(kwargs)) + namespace = runpy.run_path(str(VENDOR / "setup.py")) + return namespace, configuration + + +def test_old_installation_mismatch_fails_before_any_native_build(tmp_path, monkeypatch): + # Reproduce #247's upgrade state: old builds regenerated only the wrapper + # and named proxy, leaving the public audio_processing.py at SWIG 4.3.1. + source = tmp_path / "src" + source.mkdir() + for name in ("audio_processing_wrap.cpp", "audio_processing.py", "webrtc_audio_processing.py"): + shutil.copyfile(VENDOR / "src" / name, source / name) + wrapper = source / "audio_processing_wrap.cpp" + wrapper.write_text(wrapper.read_text().replace("0x040301", "0x040501")) + proxy = source / "webrtc_audio_processing.py" + proxy.write_text(proxy.read_text().replace("# Version 4.3.1", "# Version 4.5.1")) + before = source_snapshot(source) + namespace, _ = load_setup(monkeypatch) + command_type = namespace["build_ext"] + monkeypatch.setitem(command_type.run.__globals__, "__file__", str(tmp_path / "setup.py")) + + def native_work(): + pytest.fail("mismatched bindings reached native library lookup/build") + + monkeypatch.setitem(command_type.run.__globals__, "get_webrtc_library_path", native_work) + command = command_type(setuptools.Distribution()) + with pytest.raises(RuntimeError, match="SWIG binding") as error: + command.run() + assert "4.5.1" in str(error.value) and "4.3.1" in str(error.value) + assert "git restore" in str(error.value) + assert_source_unchanged(before, source) + + +@pytest.mark.parametrize("hook", ["get_requires_for_build_wheel", "get_requires_for_build_sdist", + "prepare_metadata_for_build_wheel", "build_wheel", "build_sdist"]) +def test_backend_contains_all_build_writes_and_keeps_output_paths(tmp_path, monkeypatch, hook): + backend = load_backend() + source = tmp_path / "original" + source.mkdir() + (source / "input.txt").write_text("committed") + (source / "build").mkdir() + (source / "build" / "stale").touch() + (source / "webrtc-audio-processing" / "install").mkdir(parents=True) + monkeypatch.setattr(backend, "SOURCE", source) + monkeypatch.chdir(tmp_path) + before = source_snapshot(source) + stages = [] + + def delegate(*args): + stage = Path.cwd() + stages.append(stage) + assert stage != source + assert (stage / "input.txt").read_text() == "committed" + assert not (stage / "build").exists() + assert not (stage / "webrtc-audio-processing" / "install").exists() + (stage / "input.txt").write_text("regenerated") + (stage / "install").mkdir() + if hook in {"build_wheel", "build_sdist", "prepare_metadata_for_build_wheel"}: + assert Path(args[0]) == tmp_path / "output" + Path(args[0]).mkdir() + (Path(args[0]) / "artifact").write_text("built") + if hook == "build_wheel": + assert args[2] == str(tmp_path / "metadata") + return "result" + + monkeypatch.setattr(backend, "build_meta", SimpleNamespace(**{hook: delegate})) + arguments = ["output"] if hook in {"build_wheel", "build_sdist", "prepare_metadata_for_build_wheel"} else [] + keywords = {"metadata_directory": "metadata"} if hook == "build_wheel" else {} + assert getattr(backend, hook)(*arguments, **keywords) == "result" + assert Path.cwd() == tmp_path + assert all(not path.exists() for path in stages) + assert_source_unchanged(before, source) + if arguments: + assert (tmp_path / "output" / "artifact").read_text() == "built" + + +def test_backend_restores_cwd_and_removes_stage_after_build_failure(tmp_path, monkeypatch): + backend = load_backend() + monkeypatch.setattr(backend, "SOURCE", tmp_path) + previous = Path.cwd() + stages = [] + + def fail(*args): + stages.append(Path.cwd()) + raise RuntimeError("compiler failed") + + monkeypatch.setattr(backend, "build_meta", SimpleNamespace(build_wheel=fail)) + with pytest.raises(RuntimeError, match="compiler failed"): + backend.build_wheel(tmp_path / "output") + assert Path.cwd() == previous + assert all(not path.exists() for path in stages) + + +def test_compiler_uses_the_committed_wrapper_and_matching_public_proxies(monkeypatch): + namespace, configuration = load_setup(monkeypatch) + extension, = configuration["ext_modules"] + assert "src/audio_processing_wrap.cpp" in extension.sources + assert not any(path.endswith(".i") for path in extension.sources) + assert namespace["validate_swig_bindings"](VENDOR / "src") == (4, 3, 1) + public = (VENDOR / "src/audio_processing.py").read_text() + generated = (VENDOR / "src/webrtc_audio_processing.py").read_text() + assert public == generated + + +@pytest.mark.parametrize("versions", [ + (0x040301, "4.3.1", "4.3.1"), (0x040501, "4.5.1", "4.5.1"), + (0x040310, "4.3.10", "4.3.10"), (0x010311, "1.3.11", "1.3.11"), + (0x041001, "4.10.1", "4.10.1"), +]) +def test_binding_guard_accepts_matching_generations_with_either_line_ending(tmp_path, monkeypatch, versions): + namespace, _ = load_setup(monkeypatch) + (tmp_path / "audio_processing_wrap.cpp").write_bytes( + f"#define SWIG_VERSION 0x{versions[0]:06x}\r\n".encode()) + (tmp_path / "audio_processing.py").write_bytes(f"# Version {versions[1]}\r\n".encode()) + (tmp_path / "webrtc_audio_processing.py").write_bytes(f"# Version {versions[2]}\n".encode()) + assert namespace["validate_swig_bindings"](tmp_path) == tuple(map(int, versions[1].split("."))) + + +@pytest.mark.parametrize("name", ["audio_processing_wrap.cpp", "audio_processing.py", + "webrtc_audio_processing.py"]) +@pytest.mark.parametrize("damage", ["different_version", "missing_header", "missing_file"]) +def test_binding_guard_rejects_each_inconsistent_or_unreadable_input(tmp_path, monkeypatch, name, damage): + namespace, _ = load_setup(monkeypatch) + for filename in ("audio_processing_wrap.cpp", "audio_processing.py", "webrtc_audio_processing.py"): + shutil.copyfile(VENDOR / "src" / filename, tmp_path / filename) + damaged = tmp_path / name + if damage == "missing_file": + damaged.unlink() + elif damage == "missing_header": + damaged.write_text("not a generated binding") + else: + damaged.write_text(damaged.read_text().replace("0x040301", "0x040501") + .replace("# Version 4.3.1", "# Version 4.5.1")) + before = source_snapshot(tmp_path) + with pytest.raises(RuntimeError, match="SWIG binding") as error: + namespace["validate_swig_bindings"](tmp_path) + assert name in str(error.value) + assert "git restore --source=HEAD -- vendor/aec-audio-processing/src" in str(error.value) + assert_source_unchanged(before, tmp_path) + + +@pytest.mark.parametrize("version", [(3, 11, 9), (3, 13, 1), (3, 14, 0)]) +def test_qualification_rejects_wrong_python_before_any_work(monkeypatch, version): + from tools import verify_aec_source_build as verifier + + monkeypatch.setattr(verifier.sys, "version_info", version) + monkeypatch.setattr(verifier, "source_snapshot", lambda path: pytest.fail("work started")) + with pytest.raises(SystemExit, match="requires Python 3.12"): + verifier.main() + + +def test_qualification_removes_inherited_rust_log_only_from_child_environment(tmp_path, monkeypatch): + from tools import verify_aec_source_build as verifier + + monkeypatch.setenv("RUST_LOG", "warn") + monkeypatch.setenv("UV_CACHE_DIR", "synthetic-cache") + script = ( + "import os; assert 'RUST_LOG' not in os.environ; " + "assert os.environ['UV_CACHE_DIR'] == 'synthetic-cache'; " + f"print({verifier.ABSEIL_EVIDENCE!r})" + ) + verifier.run([sys.executable, "-c", script], cwd=tmp_path, require_abseil=True) + assert os.environ["RUST_LOG"] == "warn" + assert os.environ["UV_CACHE_DIR"] == "synthetic-cache" + + +def test_qualification_still_rejects_missing_build_evidence(tmp_path, monkeypatch): + from tools import verify_aec_source_build as verifier + + monkeypatch.setenv("RUST_LOG", "warn") + with pytest.raises(RuntimeError, match="Missing actual Meson resolution evidence"): + verifier.run([sys.executable, "-c", "print('no build occurred')"], + cwd=tmp_path, require_abseil=True) + + +def test_qualification_pins_both_builds_and_environments_to_the_running_python(tmp_path, monkeypatch): + from tools import verify_aec_source_build as verifier + + source = tmp_path / "source" + source.mkdir() + monkeypatch.setattr(verifier, "SOURCE", source) + monkeypatch.setattr(verifier, "ROOT", tmp_path) + monkeypatch.setattr(verifier.sys, "version_info", (3, 12, 10)) + monkeypatch.setattr(verifier.sys, "platform", "linux") + executable = str(tmp_path / "python3.12") + monkeypatch.setattr(verifier.sys, "executable", executable) + commands = [] + + def run(command, **kwargs): + commands.append(command) + if "--sdist" in command: + output = Path(command[command.index("--out-dir") + 1]) + with tarfile.open(output / "aec.tar.gz", "w:gz") as archive: + archive.addfile(tarfile.TarInfo("aec/build_backend.py")) + if "--wheel" in command: + Path(command[command.index("--out-dir") + 1], "aec.whl").touch() + + monkeypatch.setattr(verifier, "run", run) + monkeypatch.setattr(verifier, "check_wheel", lambda wheel, source: None) + monkeypatch.setattr(verifier.subprocess, "run", lambda command, **kwargs: None) + verifier.main() + selected = [command for command in commands if command[1] in {"sync", "build", "venv"}] + assert len(selected) == 4 + for command in selected: + assert command[command.index("--python") + 1] == executable + + +def test_cleanliness_check_detects_hidden_artifacts_and_tracked_mutation(tmp_path): + source = tmp_path / "source" + source.mkdir() + tracked = source / "wrapper.cpp" + tracked.write_bytes(b"original\r\n") + before = source_snapshot(source) + tracked.write_bytes(b"original\n") + with pytest.raises(RuntimeError, match="wrapper.cpp"): + assert_source_unchanged(before, source) + tracked.write_bytes(b"original\r\n") + (source / "build").mkdir() + with pytest.raises(RuntimeError, match="build"): + assert_source_unchanged(before, source) + + +@pytest.mark.parametrize("mismatch", [None, "audio_processing_wrap.cpp", "audio_processing.py", + "webrtc_audio_processing.py"]) +def test_wheel_qualification_rejects_a_stale_compiled_source_or_public_proxy(tmp_path, mismatch): + wheel = tmp_path / "test.whl" + with zipfile.ZipFile(wheel, "w") as archive: + for name in ("audio_processing_wrap.cpp", "audio_processing.py", "webrtc_audio_processing.py"): + content = (VENDOR / "src" / name).read_bytes() + archive.writestr(f"aec_audio_processing/{name}", b"stale" if name == mismatch else content) + archive.writestr("aec_audio_processing/_webrtc_audio_processing.cpython-312-x86_64-linux-gnu.so", b"") + archive.writestr("aec_audio_processing/files/libwebrtc-audio-processing-2.so", b"") + archive.writestr("aec_audio_processing/files/THIRD_PARTY_NOTICES.txt", b"") + if mismatch: + with pytest.raises(RuntimeError, match="compiled source/proxy pair"): + check_wheel(wheel, VENDOR) + else: + check_wheel(wheel, VENDOR) diff --git a/tests/test_release_tooling.py b/tests/test_release_tooling.py index 517da39c..00529e17 100644 --- a/tests/test_release_tooling.py +++ b/tests/test_release_tooling.py @@ -264,6 +264,7 @@ def test_source_release_keeps_the_linux_aec_path_dependency() -> None: manifest = json.loads((ROOT / "LICENSES/provenance.json").read_text(encoding="utf-8")) required = [ "vendor/aec-audio-processing/setup.py", + "vendor/aec-audio-processing/build_backend.py", "vendor/aec-audio-processing/pyproject.toml", "vendor/aec-audio-processing/LICENSE", "vendor/aec-audio-processing/src/files/THIRD_PARTY_NOTICES.txt", diff --git a/tools/verify_aec_source_build.py b/tools/verify_aec_source_build.py new file mode 100644 index 00000000..c941b8ff --- /dev/null +++ b/tools/verify_aec_source_build.py @@ -0,0 +1,122 @@ +"""Qualify a cold Linux Voice install and an independently installed AEC wheel.""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import subprocess +import sys +import tarfile +import tempfile +import zipfile + + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "vendor" / "aec-audio-processing" +ABSEIL_EVIDENCE = "AEC build: forced bundled Abseil 20240722.0" + + +def source_snapshot(source: Path) -> dict[str, str]: + """Include ignored build/install output and empty directories, but not bytecode.""" + result = {} + for path in sorted(source.rglob("*")): + if "__pycache__" in path.parts or path.suffix in {".pyc", ".pyo"}: + continue + key = path.relative_to(source).as_posix() + result[key] = hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else "" + return result + + +def assert_source_unchanged(before: dict[str, str], source: Path) -> None: + after = source_snapshot(source) + changed = sorted(key for key in before.keys() | after.keys() if before.get(key) != after.get(key)) + if changed: + raise RuntimeError("AEC build changed source paths: " + ", ".join(changed[:20])) + + +def run(command: list[str], *, cwd: Path, require_abseil: bool = False) -> None: + environment = os.environ.copy() + # An inherited Rust filter can suppress uv's verbose build evidence. + environment.pop("RUST_LOG", None) + completed = subprocess.run(command, cwd=cwd, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, encoding="utf-8", errors="replace", + env=environment) + if completed.returncode: + print(completed.stdout) + raise RuntimeError(f"{command[0]} {command[1]} failed ({completed.returncode})") + if require_abseil: + if ABSEIL_EVIDENCE not in completed.stdout: + raise RuntimeError("Missing actual Meson resolution evidence; a cold build is required") + print(ABSEIL_EVIDENCE) + + +def check_wheel(wheel: Path, source: Path) -> None: + with zipfile.ZipFile(wheel) as archive: + names = archive.namelist() + for name in ("audio_processing_wrap.cpp", "audio_processing.py", "webrtc_audio_processing.py"): + if archive.read(f"aec_audio_processing/{name}") != (source / "src" / name).read_bytes(): + raise RuntimeError(f"Wheel does not contain the compiled source/proxy pair: {name}") + if not any(name.startswith("aec_audio_processing/_webrtc_audio_processing.") + and name.endswith(".so") for name in names): + raise RuntimeError("Wheel is missing its Python native extension") + if "aec_audio_processing/files/libwebrtc-audio-processing-2.so" not in names: + raise RuntimeError("Wheel is missing the WebRTC runtime library") + if "aec_audio_processing/files/THIRD_PARTY_NOTICES.txt" not in names: + raise RuntimeError("Wheel is missing third-party notices") + + +NATIVE_SMOKE = """ +from aec_audio_processing import AudioProcessor +from aec_audio_processing.audio_processing import AudioProcessor as PublicProxy +assert AudioProcessor is PublicProxy +processor = AudioProcessor(enable_aec=True, enable_ns=False, enable_agc=False, enable_vad=False) +processor.set_stream_format(48000, 1, 48000, 1) +processor.set_reverse_stream_format(48000, 1) +processor.set_stream_delay(0) +frame = bytes(480 * 2) +for _ in range(10): + processor.process_reverse_stream(frame) + assert processor.process_stream(frame) == frame +print("Installed wheel public AudioProcessor native smoke: PASS") +""" + + +def main() -> None: + if sys.version_info[:2] != (3, 12): + raise SystemExit("AEC qualification requires Python 3.12; rerun with python3.12.") + if sys.platform != "linux": + raise SystemExit("This native source-build qualification requires Linux.") + before = source_snapshot(SOURCE) + # The CI job supplies a fresh UV_CACHE_DIR. Require build evidence so a + # cached wheel cannot turn this qualification into an install-only check. + run(["uv", "sync", "--locked", "--extra", "voice", "--extra", "dev", + "--python", sys.executable, "--verbose"], + cwd=ROOT, require_abseil=True) + assert_source_unchanged(before, SOURCE) + with tempfile.TemporaryDirectory(prefix="amadeus-aec-qualification-") as directory: + output = Path(directory) + run(["uv", "build", "--sdist", str(SOURCE), "--out-dir", str(output), + "--python", sys.executable], cwd=ROOT) + sdist, = output.glob("*.tar.gz") + with tarfile.open(sdist) as archive: + if not any(name.endswith("/build_backend.py") for name in archive.getnames()): + raise RuntimeError("Source release is missing the isolated build backend") + run(["uv", "build", "--wheel", str(sdist), "--out-dir", str(output), + "--python", sys.executable, "--verbose"], + cwd=output, require_abseil=True) + wheel, = output.glob("*.whl") + check_wheel(wheel, SOURCE) + environment = output / "venv" + run(["uv", "venv", str(environment), "--python", sys.executable], cwd=output) + python = environment / "bin" / "python" + run(["uv", "pip", "install", "--python", str(python), "--no-deps", str(wheel)], cwd=output) + # -I and an external cwd keep the checkout and user site out of imports. + subprocess.run([str(python), "-I", "-c", NATIVE_SMOKE], cwd=output, check=True, + env={key: value for key, value in os.environ.items() if key != "PYTHONPATH"}) + assert_source_unchanged(before, SOURCE) + print("Cold Voice install, sdist/wheel, native smoke and source cleanliness: PASS") + + +if __name__ == "__main__": + main() diff --git a/vendor/aec-audio-processing.PROVENANCE.md b/vendor/aec-audio-processing.PROVENANCE.md index dc378d4b..1c42672d 100644 --- a/vendor/aec-audio-processing.PROVENANCE.md +++ b/vendor/aec-audio-processing.PROVENANCE.md @@ -3,14 +3,29 @@ - Source of truth: [official PyPI release metadata](https://pypi.org/pypi/aec-audio-processing/1.0.1/json). - Artifact: `aec_audio_processing-1.0.1.tar.gz` (5,897,164 bytes). - SHA256: `f03a3ce3f45bf15a04a8504d0f692a651180ca2d3171bd9f3de0516e5f2b8360`. -- Upstream version: `1.0.1`; backend: `setuptools.build_meta`. +- Upstream version: `1.0.1`; upstream backend: `setuptools.build_meta`. - Baseline includes WebRTC Audio Processing 2.1 and Abseil 20240722.0. -The only upstream source change is [aec-audio-processing.patch](aec-audio-processing.patch): -add `--force-fallback-for=abseil-cpp` to `setup.py`'s Meson setup arguments. -It keeps incompatible system Abseil out of this dependency's source build; -no WebRTC/Abseil source or audio runtime code is changed. The Linux-only -`tool.uv.sources` marker preserves the Windows/macOS PyPI paths. +The complete build-only delta is [aec-audio-processing.patch](aec-audio-processing.patch): + +- Keep `--force-fallback-for=abseil-cpp` and verify Meson's actual selected + subproject version before compilation. +- Delegate PEP 517 metadata, wheel and sdist hooks to setuptools inside a + disposable source copy. Meson build/install output and setuptools artifacts + stay in that copy; stale source-tree build/install output is excluded. +- Compile the committed SWIG 4.3.1 C++ wrapper alongside its committed Python + proxies, including the public `audio_processing.py` entry point. Installation + no longer regenerates bindings or requires SWIG. Regeneration is an explicit + source-maintenance task that must update and validate the wrapper and both + proxies together. +- Check wrapper and both proxy version headers before native work. Mismatched + or unreadable generations (including old-install leftovers preserved by Git) + fail with explicit restore/fresh-archive instructions; no user files are + repaired automatically. +- Include the small staging backend in source distributions. + +No WebRTC/Abseil source, generated binding, or audio runtime code is changed. +The Linux-only `tool.uv.sources` marker preserves the Windows/macOS PyPI paths. All upstream license/attribution files and source notices are retained. `src/files/THIRD_PARTY_NOTICES.txt` is an additional, verbatim concatenation of @@ -33,3 +48,18 @@ exclusion and AEC provenance/notice entries after a replacement official artifac modern system Abseil, locked Voice installation, verifier and AEC smoke; recheck Windows/macOS artifact selection and regenerate `uv.lock`. Remove the source archive's `vendor` include root if no other vendored dependency needs it. + +Linux qualification uses `python3.12 tools/verify_aec_source_build.py` with a fresh +`UV_CACHE_DIR`. It verifies the locked Voice source build, actual bundled Abseil +resolution, byte-preserving source cleanliness (including ignored build output), +sdist-to-wheel packaging, unchanged wrapper/proxy bytes in the wheel, and the +public native API in a fresh environment outside the checkout. Python bytecode +caches are excluded from the cleanliness comparison. The temporary Meson tree +is removed after each hook; CI no longer expects introspection under the vendor +source. This remains build/binding qualification, not acoustic-quality testing. +The qualifier requires Python 3.12 and selects that same interpreter for the +locked install, both distribution builds, and the isolated wheel environment. +It removes inherited `RUST_LOG` from its child-process environment so uv's +verbose build evidence remains visible; the caller's environment is unchanged. +Existing checkouts should follow the +[Linux upgrade instructions](../docs/install_profiles.md#upgrading-an-existing-linux-voice-checkout). diff --git a/vendor/aec-audio-processing.patch b/vendor/aec-audio-processing.patch index 31b37573..8bfa3375 100644 --- a/vendor/aec-audio-processing.patch +++ b/vendor/aec-audio-processing.patch @@ -1,10 +1,169 @@ --- a/setup.py +++ b/setup.py -@@ -65,6 +65,7 @@ - 'meson', 'setup', - '--buildtype=release', - '--default-library=shared', +@@ -11,0 +12,3 @@ ++import json ++import re ++from pathlib import Path +@@ -13,0 +17,36 @@ ++ ++ ++def validate_swig_bindings(source_dir): ++ """Reject bindings left at different generations by an earlier install.""" ++ source_dir = Path(source_dir) ++ patterns = { ++ 'audio_processing_wrap.cpp': r'^\s*#define\s+SWIG_VERSION\s+0x([0-9]{6})\b', ++ 'audio_processing.py': r'^# Version (\d+)\.(\d+)\.(\d+)\s*$', ++ 'webrtc_audio_processing.py': r'^# Version (\d+)\.(\d+)\.(\d+)\s*$', ++ } ++ versions = {} ++ for name, pattern in patterns.items(): ++ try: ++ text = (source_dir / name).read_text(encoding='utf-8') ++ except (OSError, UnicodeError): ++ text = '' ++ match = re.search(pattern, text, re.MULTILINE) ++ if match is None: ++ versions[name] = None ++ elif name.endswith('.cpp'): ++ # SWIG uses decimal digit pairs: 0x010311 represents 1.3.11. ++ versions[name] = tuple(int(match[1][index:index + 2], 10) for index in (0, 2, 4)) ++ else: ++ versions[name] = tuple(map(int, match.groups())) ++ if None in versions.values() or len(set(versions.values())) != 1: ++ details = ', '.join( ++ f'{name}={".".join(map(str, version)) if version else "unreadable"}' ++ for name, version in versions.items() ++ ) ++ raise RuntimeError( ++ f'AEC SWIG binding versions are inconsistent or unreadable: {details}. ' ++ 'Back up any intentional edits, then from the updated repository root run ' ++ '`git restore --source=HEAD -- vendor/aec-audio-processing/src` and rebuild. ' ++ 'For a source archive, unpack a fresh copy instead.' ++ ) ++ return versions['audio_processing_wrap.cpp'] +@@ -67,0 +107 @@ + '--force-fallback-for=abseil-cpp', - f'--prefix={install_dir}', - build_dir - ] +@@ -71,0 +112,11 @@ ++ ++ # Verify the actual resolution, not just the requested Meson option. ++ info_dir = os.path.join(build_dir, 'meson-info') ++ with open(os.path.join(info_dir, 'intro-buildoptions.json'), encoding='utf-8') as handle: ++ options = {item['name']: item['value'] for item in json.load(handle)} ++ with open(os.path.join(info_dir, 'intro-projectinfo.json'), encoding='utf-8') as handle: ++ subprojects = {item['name']: item['version'] for item in json.load(handle)['subprojects']} ++ if ('abseil-cpp' not in options['force_fallback_for'] ++ or subprojects.get('abseil-cpp') != '20240722.0'): ++ raise RuntimeError('AEC build did not select bundled Abseil 20240722.0') ++ print('AEC build: forced bundled Abseil 20240722.0', flush=True) +@@ -92,0 +144 @@ ++ validate_swig_bindings(Path(__file__).resolve().parent / 'src') +@@ -157,9 +208,0 @@ +-] +- +-swig_opts = [ +- '-c++', +- '-Isrc', +- f'-I{webrtc_dir}', +- f'-I{os.path.join(webrtc_dir, "webrtc")}', +- f'-I{abseil_dir}', +- f'-I{install_include_dir}', +@@ -203 +246,3 @@ +- sources=['src/audio_processing.i', 'src/audio_processing_module.cpp'], ++ # Keep the committed SWIG 4.3.1 wrapper and Python proxies together. ++ # Regeneration is an explicit source-maintenance task, not installation. ++ sources=['src/audio_processing_wrap.cpp', 'src/audio_processing_module.cpp'], +@@ -205 +249,0 @@ +- swig_opts=swig_opts, +@@ -218 +262 @@ +-) +\ No newline at end of file ++) +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -5 +4,0 @@ +- "swig>=4.0.0", +@@ -10 +9,2 @@ +-build-backend = "setuptools.build_meta" ++build-backend = "build_backend" ++backend-path = ["."] +--- a/MANIFEST.in ++++ b/MANIFEST.in +@@ -5,0 +6 @@ ++include build_backend.py +@@ -25 +26 @@ +-global-exclude .git* +\ No newline at end of file ++global-exclude .git* +--- /dev/null ++++ b/build_backend.py +@@ -0,0 +1,70 @@ ++"""Run the vendored setuptools build in a disposable source copy. ++ ++Meson and setuptools both write beside their inputs. A PEP 517 path dependency ++must not let those writes mutate Amadeus's checked-in third-party sources. ++""" ++ ++from contextlib import contextmanager ++import os ++from pathlib import Path ++import shutil ++import tempfile ++ ++from setuptools import build_meta ++ ++ ++SOURCE = Path(__file__).resolve().parent ++ ++ ++def _ignore_build_outputs(directory, names): ++ ignored = {"__pycache__", ".git", ".pytest_cache"} ++ directory = Path(directory) ++ if directory == SOURCE: ++ ignored.update({"build", "dist", ".eggs"}) ++ ignored.update(name for name in names if name.endswith(".egg-info")) ++ if directory == SOURCE / "webrtc-audio-processing": ++ ignored.update({"build", "install"}) ++ return ignored.intersection(names) ++ ++ ++@contextmanager ++def _staged_source(): ++ previous = Path.cwd() ++ with tempfile.TemporaryDirectory(prefix="amadeus-aec-build-") as directory: ++ staging = Path(directory) / "source" ++ shutil.copytree(SOURCE, staging, ignore=_ignore_build_outputs) ++ try: ++ os.chdir(staging) ++ yield ++ finally: ++ os.chdir(previous) ++ ++ ++def get_requires_for_build_wheel(config_settings=None): ++ with _staged_source(): ++ return build_meta.get_requires_for_build_wheel(config_settings) ++ ++ ++def get_requires_for_build_sdist(config_settings=None): ++ with _staged_source(): ++ return build_meta.get_requires_for_build_sdist(config_settings) ++ ++ ++def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): ++ metadata_directory = str(Path(metadata_directory).resolve()) ++ with _staged_source(): ++ return build_meta.prepare_metadata_for_build_wheel(metadata_directory, config_settings) ++ ++ ++def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): ++ wheel_directory = str(Path(wheel_directory).resolve()) ++ if metadata_directory is not None: ++ metadata_directory = str(Path(metadata_directory).resolve()) ++ with _staged_source(): ++ return build_meta.build_wheel(wheel_directory, config_settings, metadata_directory) ++ ++ ++def build_sdist(sdist_directory, config_settings=None): ++ sdist_directory = str(Path(sdist_directory).resolve()) ++ with _staged_source(): ++ return build_meta.build_sdist(sdist_directory, config_settings) diff --git a/vendor/aec-audio-processing/MANIFEST.in b/vendor/aec-audio-processing/MANIFEST.in index ff5b5204..257003cd 100644 --- a/vendor/aec-audio-processing/MANIFEST.in +++ b/vendor/aec-audio-processing/MANIFEST.in @@ -3,6 +3,7 @@ include LICENSE include setup.py include setup.cfg include pyproject.toml +include build_backend.py # Include all source files recursive-include src *.py *.cpp *.h *.i @@ -22,4 +23,4 @@ global-exclude .DS_Store global-exclude *.pyc global-exclude __pycache__ global-exclude *.so -global-exclude .git* \ No newline at end of file +global-exclude .git* diff --git a/vendor/aec-audio-processing/build_backend.py b/vendor/aec-audio-processing/build_backend.py new file mode 100644 index 00000000..f15c5e99 --- /dev/null +++ b/vendor/aec-audio-processing/build_backend.py @@ -0,0 +1,70 @@ +"""Run the vendored setuptools build in a disposable source copy. + +Meson and setuptools both write beside their inputs. A PEP 517 path dependency +must not let those writes mutate Amadeus's checked-in third-party sources. +""" + +from contextlib import contextmanager +import os +from pathlib import Path +import shutil +import tempfile + +from setuptools import build_meta + + +SOURCE = Path(__file__).resolve().parent + + +def _ignore_build_outputs(directory, names): + ignored = {"__pycache__", ".git", ".pytest_cache"} + directory = Path(directory) + if directory == SOURCE: + ignored.update({"build", "dist", ".eggs"}) + ignored.update(name for name in names if name.endswith(".egg-info")) + if directory == SOURCE / "webrtc-audio-processing": + ignored.update({"build", "install"}) + return ignored.intersection(names) + + +@contextmanager +def _staged_source(): + previous = Path.cwd() + with tempfile.TemporaryDirectory(prefix="amadeus-aec-build-") as directory: + staging = Path(directory) / "source" + shutil.copytree(SOURCE, staging, ignore=_ignore_build_outputs) + try: + os.chdir(staging) + yield + finally: + os.chdir(previous) + + +def get_requires_for_build_wheel(config_settings=None): + with _staged_source(): + return build_meta.get_requires_for_build_wheel(config_settings) + + +def get_requires_for_build_sdist(config_settings=None): + with _staged_source(): + return build_meta.get_requires_for_build_sdist(config_settings) + + +def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): + metadata_directory = str(Path(metadata_directory).resolve()) + with _staged_source(): + return build_meta.prepare_metadata_for_build_wheel(metadata_directory, config_settings) + + +def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + wheel_directory = str(Path(wheel_directory).resolve()) + if metadata_directory is not None: + metadata_directory = str(Path(metadata_directory).resolve()) + with _staged_source(): + return build_meta.build_wheel(wheel_directory, config_settings, metadata_directory) + + +def build_sdist(sdist_directory, config_settings=None): + sdist_directory = str(Path(sdist_directory).resolve()) + with _staged_source(): + return build_meta.build_sdist(sdist_directory, config_settings) diff --git a/vendor/aec-audio-processing/pyproject.toml b/vendor/aec-audio-processing/pyproject.toml index 4dea8715..9c886459 100644 --- a/vendor/aec-audio-processing/pyproject.toml +++ b/vendor/aec-audio-processing/pyproject.toml @@ -2,12 +2,12 @@ requires = [ "setuptools>=42", "wheel", - "swig>=4.0.0", "meson>=0.63.0", "ninja", "build", ] -build-backend = "setuptools.build_meta" +build-backend = "build_backend" +backend-path = ["."] [project] name = "aec-audio-processing" diff --git a/vendor/aec-audio-processing/setup.py b/vendor/aec-audio-processing/setup.py index ed52348a..9f2b8435 100644 --- a/vendor/aec-audio-processing/setup.py +++ b/vendor/aec-audio-processing/setup.py @@ -9,10 +9,49 @@ import subprocess import sys import glob +import json +import re +from pathlib import Path from setuptools import setup, Extension, find_packages from setuptools.command.build_ext import build_ext as _build_ext +def validate_swig_bindings(source_dir): + """Reject bindings left at different generations by an earlier install.""" + source_dir = Path(source_dir) + patterns = { + 'audio_processing_wrap.cpp': r'^\s*#define\s+SWIG_VERSION\s+0x([0-9]{6})\b', + 'audio_processing.py': r'^# Version (\d+)\.(\d+)\.(\d+)\s*$', + 'webrtc_audio_processing.py': r'^# Version (\d+)\.(\d+)\.(\d+)\s*$', + } + versions = {} + for name, pattern in patterns.items(): + try: + text = (source_dir / name).read_text(encoding='utf-8') + except (OSError, UnicodeError): + text = '' + match = re.search(pattern, text, re.MULTILINE) + if match is None: + versions[name] = None + elif name.endswith('.cpp'): + # SWIG uses decimal digit pairs: 0x010311 represents 1.3.11. + versions[name] = tuple(int(match[1][index:index + 2], 10) for index in (0, 2, 4)) + else: + versions[name] = tuple(map(int, match.groups())) + if None in versions.values() or len(set(versions.values())) != 1: + details = ', '.join( + f'{name}={".".join(map(str, version)) if version else "unreadable"}' + for name, version in versions.items() + ) + raise RuntimeError( + f'AEC SWIG binding versions are inconsistent or unreadable: {details}. ' + 'Back up any intentional edits, then from the updated repository root run ' + '`git restore --source=HEAD -- vendor/aec-audio-processing/src` and rebuild. ' + 'For a source archive, unpack a fresh copy instead.' + ) + return versions['audio_processing_wrap.cpp'] + + def get_webrtc_library_path(): """Returns the path to the pre-compiled WebRTC library.""" webrtc_dir = os.path.join(os.path.dirname(__file__), 'webrtc-audio-processing') @@ -71,6 +110,17 @@ def build_webrtc(): ] subprocess.check_call(meson_cmd, cwd=webrtc_dir) + # Verify the actual resolution, not just the requested Meson option. + info_dir = os.path.join(build_dir, 'meson-info') + with open(os.path.join(info_dir, 'intro-buildoptions.json'), encoding='utf-8') as handle: + options = {item['name']: item['value'] for item in json.load(handle)} + with open(os.path.join(info_dir, 'intro-projectinfo.json'), encoding='utf-8') as handle: + subprojects = {item['name']: item['version'] for item in json.load(handle)['subprojects']} + if ('abseil-cpp' not in options['force_fallback_for'] + or subprojects.get('abseil-cpp') != '20240722.0'): + raise RuntimeError('AEC build did not select bundled Abseil 20240722.0') + print('AEC build: forced bundled Abseil 20240722.0', flush=True) + print("Building with ninja...") subprocess.check_call(['ninja', '-C', build_dir], cwd=webrtc_dir) @@ -91,6 +141,7 @@ def build_webrtc(): class build_ext(_build_ext): def run(self): + validate_swig_bindings(Path(__file__).resolve().parent / 'src') # Build WebRTC library if not found lib_path = get_webrtc_library_path() if not os.path.exists(lib_path): @@ -157,15 +208,6 @@ def run(self): install_include_dir, ] -swig_opts = [ - '-c++', - '-Isrc', - f'-I{webrtc_dir}', - f'-I{os.path.join(webrtc_dir, "webrtc")}', - f'-I{abseil_dir}', - f'-I{install_include_dir}', -] - # EDIT 2: Use the correct C++ standard flag for the MSVC compiler if sys.platform == 'win32': cxx_flags = ['/std:c++20'] @@ -201,9 +243,10 @@ def run(self): ext_modules = [ Extension( name='aec_audio_processing._webrtc_audio_processing', - sources=['src/audio_processing.i', 'src/audio_processing_module.cpp'], + # Keep the committed SWIG 4.3.1 wrapper and Python proxies together. + # Regeneration is an explicit source-maintenance task, not installation. + sources=['src/audio_processing_wrap.cpp', 'src/audio_processing_module.cpp'], include_dirs=include_dirs, - swig_opts=swig_opts, extra_compile_args=cxx_flags, extra_link_args=[], # Populated by build_ext extra_objects=[], # Populated by build_ext @@ -216,4 +259,4 @@ def run(self): cmdclass={ 'build_ext': build_ext, }, -) \ No newline at end of file +)