diff --git a/.ci/scripts/wheel/pre_build_script.sh b/.ci/scripts/wheel/pre_build_script.sh index 367d398bac8..c617217009b 100755 --- a/.ci/scripts/wheel/pre_build_script.sh +++ b/.ci/scripts/wheel/pre_build_script.sh @@ -44,6 +44,80 @@ if [[ "$(uname -m)" == "aarch64" ]]; then echo "the file $file has been modified for atomic to use full path" fi +# A CPU row must say so, rather than relying on the builder having no CUDA toolkit installed. The build +# turns CUDA on when it detects one, so a builder that gains a toolkit would silently start producing a +# CPU wheel carrying the CUDA delegate. That already happened on Windows, where the image ships a toolkit +# on PATH and the resulting wheel failed to load its own extension. +# +# Stated as the inverse rule: anything that does not name a CUDA train this project supports is a CPU row. +# An allowlist of spellings was tried first and left a gap for every spelling nobody thought of, which is +# the same defect twice: testing only for empty let a row spelled "cpu" through, and listing "cpu" still +# leaves "cpu-aarch64", "rocm" and anything else the matrix generator emits. +# +# The supported trains come from install_utils.py, so both classifiers see the same list. +# A row that names CUDA and is not a supported train fails here rather than being +# rebadged as CPU. The wheel matrix comes from a different repository than this list, so +# when they drift the alternative is publishing a CPU wheel under a CUDA-named index with +# no error anywhere: setup.py cannot catch it, because the CPU option this script writes +# is read first and returns early. +CUDA_ROW=0 +if [[ -n "${CU_VERSION:-${DESIRED_CUDA:-}}" ]]; then + ROW_VALUE="${CU_VERSION:-${DESIRED_CUDA:-}}" + # Windows builders run MSYS bash in a conda environment that has python.exe + # and no python3, so either name alone breaks a platform. Mirrors + # run_python_script.sh. Resolving up front instead of testing the exit status + # keeps a missing interpreter a hard failure rather than a silent CPU build. + PYTHON_BIN=$(command -v python3 || command -v python) + row_classification=$("${PYTHON_BIN}" - "${ROW_VALUE}" <<'PY' +import re, sys +sys.path.insert(0, '.') +from install_utils import SUPPORTED_CUDA_VERSIONS +raw = sys.argv[1].strip().lower() +trains = {f'{major}{minor}' for major, minor in SUPPORTED_CUDA_VERSIONS} +# Decide by what the row NAMES, not by the digits it happens to contain. Reducing the whole +# value to digits classified rocm13.2 as CUDA and rebadged a row named plainly "cuda" as CPU. +match = re.fullmatch(r'cu(?:da)?[-_]?(.*)', raw) +if match is None: + print('cpu') +else: + digits = re.sub(r'[^0-9]', '', match.group(1)) + print('cuda' if digits in trains else 'unsupported') +PY +) + if [[ "${row_classification}" == "cuda" ]]; then + CUDA_ROW=1 + elif [[ "${row_classification}" == "unsupported" ]]; then + echo "row '${ROW_VALUE}' names a CUDA train this project does not support." >&2 + echo "Add it to SUPPORTED_CUDA_VERSIONS in install_utils.py, and to" >&2 + echo "_CUDA_RUNTIME_PACKAGES and _CUDA_LIBRARY_DIRECTORIES in setup.py, or" >&2 + echo "stop building this row. Building it as CPU would publish a CPU wheel" >&2 + echo "under a CUDA-named index." >&2 + exit 1 + fi +fi + +if [[ ${CUDA_ROW} -eq 0 ]]; then + export CMAKE_ARGS="${CMAKE_ARGS:-} -DEXECUTORCH_BUILD_CUDA=OFF" + echo "CMAKE_ARGS=${CMAKE_ARGS}" >> "${GITHUB_ENV}" + echo "row '${CU_VERSION:-${DESIRED_CUDA:-}}' names no supported CUDA train, building CPU-only" +else + # A CUDA row must produce the CUDA libraries. Left at the default, the build only + # turns CUDA on if it happens to detect a toolkit, so a builder without one produced + # a wheel tagged for CUDA, carrying no CUDA library, while still declaring the CUDA + # runtime packages. That installs cleanly and then reports the backend as + # unregistered when a model runs. Asking for it explicitly stops the decision from + # depending on detection. + # + # It does not turn a missing toolkit into a configure failure. The CUDA directory + # requires the toolkit but gates its sources on a working compiler, so a builder that + # has toolkit files and no usable nvcc still configures and simply compiles none of + # them. That leniency is deliberate, for packaging jobs that cannot complete compiler + # identification, so the row itself has to verify the libraries it expected are present. + export CMAKE_ARGS="${CMAKE_ARGS:-} -DEXECUTORCH_BUILD_CUDA=ON" + echo "CMAKE_ARGS=${CMAKE_ARGS}" >> "${GITHUB_ENV}" + echo "row '${CU_VERSION:-${DESIRED_CUDA:-}}' is a CUDA row, requiring the CUDA build" +fi + # On Windows, enable symlinks and re-checkout the current revision to create # the symlinked src/ directory. This is needed to build the wheel. if [[ $UNAME_S == *"MINGW"* || $UNAME_S == *"MSYS"* ]]; then diff --git a/.ci/scripts/wheel/test_cpp_sdk.py b/.ci/scripts/wheel/test_cpp_sdk.py index e346d38f6f4..8165325154f 100644 --- a/.ci/scripts/wheel/test_cpp_sdk.py +++ b/.ci/scripts/wheel/test_cpp_sdk.py @@ -863,16 +863,10 @@ def test_every_shipped_header_compiles(work_dir: Path) -> None: # tests a configuration no consumer of this package is ever in. "-DC10_USING_CUSTOM_GENERATED_MACROS", ] - # A CUDA wheel's headers reference the CUDA runtime, which the wheel does not publish headers for and - # states as a requirement instead. A consumer of that component supplies a toolkit of its own, so this - # check does the same rather than treating the header as unbuildable. Taken from the toolkit itself, so - # it follows whichever toolkit the build used instead of a list of prefixes that goes stale. - nvcc = shutil.which("nvcc") - cuda_root = os.environ.get("CUDA_HOME") or ( - str(Path(nvcc).parent.parent) if nvcc else "" - ) - if cuda_root and (Path(cuda_root) / "include" / "cuda_runtime.h").is_file(): - includes.append(f"-I{Path(cuda_root) / 'include'}") + # Deliberately no CUDA toolkit include directory. A CUDA wheel's own headers have to compile + # against nothing but the wheel, the same as every other header here. Adding the builder's toolkit + # would measure the build machine rather than the consumer, and a header that only compiles that way + # fails in the consumer's project instead of here. # Headers a wheel-only consumer cannot compile and is not expected to. Each needs something outside the # package: a platform that is not the one being built for, or a third-party library the wheel does not # carry. They ship because a source build includes them, and holding them to this rule would report a diff --git a/.ci/scripts/wheel/test_shared_libraries.py b/.ci/scripts/wheel/test_shared_libraries.py index e3e8ec42f70..f2ef611c5ee 100644 --- a/.ci/scripts/wheel/test_shared_libraries.py +++ b/.ci/scripts/wheel/test_shared_libraries.py @@ -55,6 +55,34 @@ # The quantized kernels, whose own library the wheel ships when they are built. # A separate group because they have a separate owner, and because a wheel built # without them ships neither the library nor these symbols. +# The CUDA delegate and its stream helper, for a wheel built from a CUDA index. The +# stream helper matters most: two copies means two notions of the caller's stream, so +# work queued through one is invisible to the other. +# A strongly defined symbol, chosen by reading the built library rather than guessed. The +# CudaBackend methods are emitted weak, and a weak definition can be replaced at load time by a +# strong one elsewhere, so naming one of those would count a definition that may not be the one +# the process uses. +_CUDA_BACKEND_SYMBOLS = ("executorch::backends::cuda::load_library",) +_CUDA_STREAM_SYMBOLS = ("executorch::extension::cuda::getCallerStream",) + +# The AOTI shim layer and the stream-guard state that lives with it. This is the state that was +# genuinely duplicated: extracting the shims with a PUBLIC whole-archive replayed the extraction at +# every consumer's link, so the guard's thread_local and these shims landed in three shipped binaries +# at once, and a stream selected through one copy was invisible to the other two. The row above cannot +# catch that, because its symbol only ever existed in one unconditionally shared library. +_AOTI_SHIM_SYMBOLS = ( + "aoti_torch_empty_strided", + "aoti_torch_delete_tensor_object", + "executorch::backends::cuda::CUDAStreamGuard::create", + # From the CUDA sources rather than the C++ ones. The build drops every .cu file when no working + # compiler is found, and the library is still produced from its .cpp sources, so a check that + # names only C++ symbols passes on a library missing every kernel it was gated for. + "aoti_torch_cuda__weight_int4pack_mm", + "aoti_torch_cuda_sort_stable", + "aoti_torch_cuda_rand", + "aoti_torch_cuda_randint_low_out", +) + _QUANTIZED_KERNEL_SYMBOLS = ( "torch::executor::native::quantize_per_tensor_out", "torch::executor::native::dequantize_per_tensor_out", @@ -300,24 +328,29 @@ def _is_export_only(library: Path) -> bool: the kernels, because the copy a C++ application links is registered into a table those libraries never read. - Named by the caller per component rather than excluded everywhere. Counting them for - the component they duplicate would report a duplicate that is not one, and excusing - them for every component would stop this catching a second registry hiding inside - one of them. + Excluded from the single-owner check for the one component that genuinely has two + copies. Counting them there would report a duplicate for something that is not one, + and the alternative, making them resolve the kernels from the shipped library, would + mean an export-time library depending on a runtime layout it never uses. - Matched on both the torch dependency and the name marker, because either alone - misfires: several shipped libraries link torch without being export-side, and a - name check alone would accept a runtime library that adopted the suffix. + Recognised by linking torch, which is the property that makes a library export-side. + Python extensions link torch too and are not export-side operator libraries, so they + are excluded by their interpreter suffix rather than by an operator-library name, + which keeps the test's verdict the same whether or not readelf is installed. """ + if ".cpython-" in library.name or library.name.endswith(".pyd"): + return False + if library.name.endswith("_aot_lib.so"): + return True if _tool("readelf") is None: - return library.name.endswith("_aot_lib.so") + return False dynamic = subprocess.run( [_tool("readelf"), "-d", str(library)], capture_output=True, text=True, check=False, ).stdout - return "libtorch.so" in dynamic and "_aot_lib" in library.name + return "libtorch.so" in dynamic def _assert_single_definer( @@ -398,6 +431,22 @@ def _assert_single_definer( print(f"✓ single {what}{where} across {len(libraries)} shipped libraries") +def _wheel_cuda_train() -> str: + """The CUDA train the installed wheel was built for, or "" for a CPU wheel. + + Read from the local version segment, which is the only place the wheel states what + it was built for. `1.5.0+cu126` gives "126". + """ + local = importlib.metadata.version("executorch").partition("+")[2] + return local[2:] if local.startswith("cu") else "" + + +# Marker for a row whose owner is required only when the wheel is a CUDA wheel. A +# sentinel rather than a boolean, because the answer is not known until the installed +# wheel is inspected, and a row cannot call that at import time. +_REQUIRED_ON_A_CUDA_WHEEL = "cuda-wheel-only" + + # Each component the wheel ships as its own library, the symbols that identify it, # and the library that must own them. `required` says whether the owner has to be # present: the optimized kernels are optional, because a wheel built without them @@ -430,6 +479,29 @@ def _assert_single_definer( "libexecutorch_kernels_quantized.so", True, ), + # The CUDA components. Required exactly when the wheel says it is a CUDA wheel, + # which is decided at check time rather than here: a fixed False meant a wheel + # tagged +cu126 carrying no CUDA library at all passed every check in this file, + # while a fixed True would fail every CPU wheel. The marker is the string these + # rows are keyed on below. + ( + "CUDA delegate", + _CUDA_BACKEND_SYMBOLS, + "libexecutorch_backend_cuda.so", + _REQUIRED_ON_A_CUDA_WHEEL, + ), + ( + "CUDA stream helper", + _CUDA_STREAM_SYMBOLS, + "libexecutorch_extension_cuda.so", + _REQUIRED_ON_A_CUDA_WHEEL, + ), + ( + "AOTI shim layer", + _AOTI_SHIM_SYMBOLS, + "libaoti_cuda_shims.so", + _REQUIRED_ON_A_CUDA_WHEEL, + ), # The third-party code these libraries bundle, checked separately from the # wrappers above. A wrapper can have a single owner while the implementation # underneath it is bundled into two of these, which is two real thread pools or @@ -455,7 +527,6 @@ def _assert_single_definer( ), ) - # The one component that legitimately exists twice. The quantized kernels are compiled into the runtime # library and again into the library torch loads at export time, because each side registers into a # table the other never reads, so a second definer there is expected rather than a fault. A process @@ -473,10 +544,17 @@ def test_each_component_has_one_owner() -> None: registers into a table nothing else reads shows up as an operator missing at run time rather than as a link error. """ - shipped = { - path.name for path in _shipped_runtime_libraries(_installed_package_dir()) - } + # Every shipped shared object, not only the ones under lib/. One owner, libaoti_cuda_shims.so, + # ships under backends/cuda/, and scanning lib/ alone reported it as absent, which each row + # treats as an acceptable state and so would have skipped the check entirely. + shipped = {path.name for path in _shipped_shared_objects(_installed_package_dir())} + on_a_cuda_wheel = bool(_wheel_cuda_train()) for what, symbols, owner, required in _OWNED_COMPONENTS: + if required == _REQUIRED_ON_A_CUDA_WHEEL: + # Resolved here rather than in the table, because it depends on the installed + # wheel. A fixed False let a wheel tagged +cu126 ship with no CUDA library at + # all and still pass, which is the whole point of these three rows. + required = on_a_cuda_wheel present = any(name.startswith(owner) for name in shipped) assert present or not required, ( f"the wheel ships no {owner}, which owns the {what}. Either packaging " @@ -1173,7 +1251,8 @@ def names_a_build_directory(entry: str) -> bool: ) offenders = {} - checked = 0 + inspected = 0 + with_a_runtime_path = 0 for library in sorted(package_dir.rglob("*.so*")): if not library.is_file() or library.is_symlink(): continue @@ -1185,6 +1264,7 @@ def names_a_build_directory(entry: str) -> bool: ) if result.returncode != 0: continue + inspected += 1 # An absent RPATH and one containing a single empty entry both print as an # empty string, so treat empty output as "no runtime path" rather than as an # empty entry. A library with nothing to search is fine; the defect is @@ -1192,7 +1272,7 @@ def names_a_build_directory(entry: str) -> bool: raw = result.stdout.strip() if not raw: continue - checked += 1 + with_a_runtime_path += 1 bad = [] for entry in raw.split(":"): if not entry: @@ -1218,13 +1298,15 @@ def names_a_build_directory(entry: str) -> bool: "the build tree that produced the wheel or, for an empty entry, the process " f"working directory: {offenders}" ) - assert checked, ( - f"no shipped library under {package_dir} carries a runtime search path, so this check examined " - "nothing and would pass on a wheel that shipped no libraries at all" + # Counted separately, because a wheel whose libraries all had their runtime paths removed + # entirely would satisfy a readable-file count while this check examined no path at all. + assert with_a_runtime_path, ( + f"none of the {inspected} shipped libraries under {package_dir} carries a runtime search path, " + "so this check examined nothing. The shipped libraries need a relative path to reach each other." ) print( - f"✓ none of the {checked} shipped libraries searches a build-tree or empty " - "runtime path" + f"✓ none of the {with_a_runtime_path} shipped libraries with a runtime path searches a " + f"build-tree or empty directory ({inspected} inspected)" ) @@ -1261,12 +1343,17 @@ def test_extension_contains_no_component() -> None: # # The bundled third-party groups are left out on purpose. That code is also linked by torch, and the # extension links torch, so seeing those symbols there says nothing about this split. - # Only components whose owning library is actually in this wheel. One of them is optional, so a build - # with it turned off ships no owner, and asserting the extension does not define its symbols would - # reject a configuration the table itself marks as supported. - shipped = { - path.name for path in _shipped_runtime_libraries(_installed_package_dir()) - } + # Only components whose owning library is actually in this wheel. A build with an optional component + # turned off ships no owner, and asserting the extension does not define its symbols would reject a + # configuration the table itself marks as supported. Required rows are kept regardless, because their + # owner missing is a packaging fault the owner check reports. + shipped = {path.name for path in _shipped_runtime_libraries(package_dir)} + # Guarded here rather than further down, so it protects the filter that uses it: a wheel that + # installed no runtime libraries would otherwise compare the extension against an empty set and pass. + assert shipped, ( + f"the wheel installed no runtime libraries under {package_dir / 'lib'}, so this check would " + "compare the extension against nothing and pass" + ) owned = tuple( symbol for _, symbols, owner, required in _OWNED_COMPONENTS @@ -1297,23 +1384,20 @@ def test_extension_contains_no_component() -> None: # this split moved out of it, so the extension must now resolve them from outside or # a retention option silently failed. # - # Not every shipped library serves Python. The quantized kernels and the CUDA - # delegate exist for a C++ application: Python registers quantized operators through - # the torch-linked ahead-of-time library at export time, and never loads the CUDA - # delegate from this extension at all. Requiring a dependency on those would demand - # the extension link code it has no use for. - shipped = {path.name for path in _shipped_runtime_libraries(package_dir)} - assert shipped, ( - f"the wheel installed no runtime libraries under {package_dir / 'lib'}, so this check would " - "compare the extension against nothing and pass" - ) + # Not every shipped library serves Python. The quantized kernels exist for a C++ + # application, since Python registers those operators through the torch-linked + # ahead-of-time library at export time, and requiring a dependency would demand the + # extension link code it has no use for. + # + # The CUDA delegate is NOT in that category. The build deliberately links it into the + # extension with a retention option, so it does carry a dependency, and excluding it + # switched off the one check that would notice if that retention stopped working. The + # stream helper stays excluded because the extension reaches it only through the + # delegate's public link, with no retention of its own to protect. expected = { name for name in shipped - if not any( - marker in name - for marker in ("kernels_quantized", "backend_cuda", "extension_cuda") - ) + if not any(marker in name for marker in ("kernels_quantized", "extension_cuda")) } unused = sorted(expected - needed) assert not unused, ( @@ -1375,11 +1459,13 @@ def test_shipped_library_names_are_expected() -> None: and still passed every symbol check, because those checks only ask how many definers a symbol has, never whether a file belongs in the wheel at all. - Two properties catch it. A library's recorded soname matches its file name, or a - consumer records a dependency the wheel does not contain. And its name is one - packaging knows how to produce, which is what a leftover from an older layout - fails. A wheel ships unversioned names on purpose, so the name itself carries no - version to check. + Two properties catch it. Its name is one packaging knows how to produce, which is + what a leftover from an older layout fails. And its recorded soname matches its + file name, or a consumer records a dependency the wheel does not contain. + + The names are unversioned, because these libraries ship one file each with no + symlink chain, and a versioned name without the usual symlinks is harder to load + rather than safer. """ package_dir = _installed_package_dir() lib_dir = package_dir / "lib" @@ -1407,6 +1493,13 @@ def test_shipped_library_names_are_expected() -> None: "libexecutorch", "libexecutorch_kernels_optimized", "libexecutorch_kernels_quantized", + "libexecutorch_backend_cuda", + "libexecutorch_extension_cuda", + # The same library under the name a non-shared build gives it. The shared + # build renames it to match the other shipped components; every other build + # leaves this spelling, and the shim layer records whichever one exists as a + # dependency, so both have to ship and both are expected here. + "libextension_cuda", "libexecutorch_backend_xnnpack", "libexecutorch_threadpool", "libexecutorch_etdump", diff --git a/backends/cuda/CMakeLists.txt b/backends/cuda/CMakeLists.txt index baab8ed559e..7c380def535 100644 --- a/backends/cuda/CMakeLists.txt +++ b/backends/cuda/CMakeLists.txt @@ -167,11 +167,17 @@ if(_cuda_is_msvc_toolchain) # avoiding duplicate static/object inclusion and interface leakage. target_link_libraries(aoti_cuda_shims PRIVATE aoti_common_shims_slim_obj) else() + # The whole-archive pair is PRIVATE so it applies to this library's own link + # and does not replay at the link of anything that links this one. In PUBLIC + # scope the archive was extracted again by each consumer, so the translation + # unit holding SlimTensor's per-device current stream thread_local landed in + # three shipped binaries, and one copy per process is what makes that state + # agree. The MSVC branch above uses PRIVATE for the same reason. target_link_libraries( aoti_cuda_shims - PRIVATE cuda_platform - PUBLIC -Wl,--whole-archive aoti_common_shims_slim -Wl,--no-whole-archive - CUDA::cudart extension_cuda ${CMAKE_DL_LIBS} + PRIVATE cuda_platform -Wl,--whole-archive aoti_common_shims_slim + -Wl,--no-whole-archive + PUBLIC CUDA::cudart extension_cuda ${CMAKE_DL_LIBS} ) endif() @@ -179,6 +185,13 @@ if(NOT _cuda_is_msvc_toolchain) executorch_target_link_options_shared_lib(aoti_cuda_shims) endif() +# This library links the CUDA runtime directly and the wheel ships it, so it +# needs a search path for the same reason the delegate does. Without one it can +# ship with no runtime path at all, on a builder where the runtime resolves from +# an implicit link directory, and the wheel then adds no relative hop either, +# because the step that adds one has nothing to rewrite. +executorch_target_shipped_runtime_path(aoti_cuda_shims) + install( TARGETS aoti_cuda_shims EXPORT ExecuTorchTargets @@ -195,7 +208,17 @@ if(_cuda_is_msvc_toolchain) list(APPEND _aoti_cuda_backend_sources runtime/cuda_allocator.cpp) endif() -add_library(aoti_cuda_backend STATIC ${_aoti_cuda_backend_sources}) +# Build the delegate as a shared library for the wheel so a process has one copy +# of it, and keep it static everywhere else so no other build changes. +if(EXECUTORCH_BUILD_SHARED) + set(_aoti_cuda_backend_library_type SHARED) +else() + set(_aoti_cuda_backend_library_type STATIC) +endif() +add_library( + aoti_cuda_backend ${_aoti_cuda_backend_library_type} + ${_aoti_cuda_backend_sources} +) target_include_directories( aoti_cuda_backend @@ -235,6 +258,22 @@ endif() executorch_target_link_options_shared_lib(aoti_cuda_backend) +if(EXECUTORCH_BUILD_SHARED) + # Named after what the library provides rather than after the target that + # produces it, matching the other shipped delegates, so the file reads as + # libexecutorch_backend_cuda.so. The target name stays as it is because the + # rest of the build already refers to it. + set_target_properties( + aoti_cuda_backend PROPERTIES OUTPUT_NAME executorch_backend_cuda + ) + executorch_target_soname_policy(aoti_cuda_backend) + # Resolve the runtime from the shared library rather than from the static + # core, so the delegate registers into the one registry the process has. + target_link_libraries(aoti_cuda_backend PUBLIC executorch_shared) + # Ships beside the runtime in the wheel's lib/ directory. + executorch_target_shipped_runtime_path(aoti_cuda_backend) +endif() + install( TARGETS aoti_cuda_backend EXPORT ExecuTorchTargets diff --git a/docs/source/using-executorch-cpp.md b/docs/source/using-executorch-cpp.md index 3c0756a753e..bdcfa7bc60f 100644 --- a/docs/source/using-executorch-cpp.md +++ b/docs/source/using-executorch-cpp.md @@ -110,6 +110,8 @@ reported while CMake configures, rather than failing later at link time. | `executorch::kernels_optimized` | CPU operator kernels. Needed for any operator a delegate does not claim. | | `executorch::kernels_quantized` | quantized operator kernels, for a quantized model. Link it only when you need it: see the note below. | | `executorch::backend_xnnpack` | the XNNPACK delegate. | +| `executorch::backend_cuda` | the CUDA delegate, in a CUDA wheel. | +| `executorch::extension_cuda` | the CUDA stream helper, in a CUDA wheel. Lets you pick the CUDA stream a model runs on. | | `executorch::threadpool` | the shared thread pool. | | `executorch::etdump` | the profiler. | diff --git a/extension/cuda/CMakeLists.txt b/extension/cuda/CMakeLists.txt index 0003691ac8b..1334a7140f4 100644 --- a/extension/cuda/CMakeLists.txt +++ b/extension/cuda/CMakeLists.txt @@ -16,14 +16,24 @@ if(NOT EXECUTORCH_ROOT) set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..) endif() -find_package(CUDAToolkit REQUIRED) - # SHARED on purpose: the caller-stream thread-local must have a single # definition across every shared object in the process (see export.h). A static # copy linked into multiple shared libraries would create multiple thread-locals # and silently break the caller-stream handshake. add_library(extension_cuda SHARED caller_stream.cpp) -target_link_libraries(extension_cuda PUBLIC CUDA::cudart) +if(EXECUTORCH_BUILD_SHARED) + # Named after what it provides rather than after the target, matching the + # other libraries the wheel ships. The target name stays as it is because the + # rest of the build already refers to it. + set_target_properties( + extension_cuda PROPERTIES OUTPUT_NAME executorch_extension_cuda + ) + executorch_target_soname_policy(extension_cuda) +endif() +# No CUDA headers or libraries: caller_stream.cpp uses cudaStream_t as an opaque +# handle and calls no CUDA function, so it compiles without cuda_runtime.h and +# needs no libcudart. Anything else that links this library and does call CUDA +# links the runtime itself. target_include_directories(extension_cuda PUBLIC ${_common_include_directories}) target_compile_options( extension_cuda PUBLIC "$<$:${_common_compile_options}>" diff --git a/extension/cuda/caller_stream.h b/extension/cuda/caller_stream.h index a13b7a9b396..825afa14345 100644 --- a/extension/cuda/caller_stream.h +++ b/extension/cuda/caller_stream.h @@ -8,12 +8,19 @@ #pragma once -#include #include #include #include +// Declared here rather than including , so this header compiles +// against a distribution that ships it without a CUDA toolkit. A stream is only +// ever stored and handed back below, never dereferenced, so the handle is all +// this interface needs. Repeating the declaration CUDA itself makes is +// well-formed, so a consumer that also includes is unaffected, +// in either include order. +typedef struct CUstream_st* cudaStream_t; + namespace executorch::extension::cuda { /** diff --git a/install_utils.py b/install_utils.py index 5bbd3eeac73..675fdb7e5b0 100644 --- a/install_utils.py +++ b/install_utils.py @@ -9,6 +9,8 @@ import os import platform import re +import shlex +import shutil import subprocess import sys from typing import List, Optional @@ -105,8 +107,10 @@ def _get_cuda_version(): """ try: # Get CUDA version from nvcc (CUDA compiler) + # Same selection rule as _detected_cuda_major, so the two cannot disagree about which + # toolkit this build uses. nvcc_result = subprocess.run( - ["nvcc", "--version"], capture_output=True, text=True, check=True + _selected_nvcc(), capture_output=True, text=True, check=True ) # Parse nvcc output for CUDA version # Output contains line like "Cuda compilation tools, release 12.6, V12.6.68" @@ -141,6 +145,84 @@ def _get_cuda_version(): ) +def _selected_nvcc() -> List[str]: + """The nvcc command line to ask for a version, matching what the build will use. + + Reading the bare command described whichever toolkit was on PATH, while the build also honours + these variables. Packaging then declared the runtime for one toolkit while compiling against + another, or declared nothing at all when the selected compiler was not on PATH. + """ + # CMake reads -DCMAKE_CUDA_COMPILER from the command line and CUDACXX from the environment. It does + # NOT read an environment variable named CMAKE_CUDA_COMPILER, measured with cmake 3.31.8, so asking + # the environment for that name first described a compiler the build would never use. + explicit = _extract_cmake_define(_cmake_args_from_env(), "CMAKE_CUDA_COMPILER") + if not explicit: + explicit = os.environ.get("CUDACXX") + if explicit: + # Only the program, because CMake accepts trailing options here and splits them off: + # measured, CUDACXX="/path/nvcc -allow-unsupported-compiler" still compiles with + # /path/nvcc. Passing the whole value as one program name found nothing, so a build + # that compiles fine reported no toolkit at all. That option is a common workaround + # on a newer host compiler, so the value is not unusual. + # + # Splitting also matches CMake on an unquoted path containing a space, which CMake + # reports as NOTFOUND rather than treating as one name, so this reads nothing exactly + # where the build would also refuse to compile. + program = shlex.split(explicit) or [explicit] + return [program[0], "--version"] + # Follow CMake's COMPILER search, since that is what decides which nvcc compiles the sources: + # CUDACXX above, then PATH, then CUDA_PATH, then the conventional symlink. PATH has to come + # before the others. Measured with only PATH pointing at 13.0 on a box whose /usr/local/cuda is + # 12.8, CMake compiles with 13.0, so consulting the symlink first reported 12.8 and produced + # metadata for a train the binaries were not built with. + # + # CUDAToolkit_ROOT is deliberately absent: it steers find_package(CUDAToolkit) but CMake's + # compiler search ignores it, so reading it here names a compiler that will not be used. + on_path = shutil.which("nvcc") + if on_path: + return [on_path, "--version"] + for root in (os.environ.get("CUDA_PATH"), "/usr/local/cuda"): + if not root: + continue + candidate = os.path.join(root, "bin", "nvcc") + if os.path.exists(candidate): + return [candidate, "--version"] + return ["nvcc", "--version"] + + +def _cmake_args_from_env() -> List[str]: + """CMAKE_ARGS split into arguments, tolerating an unbalanced quote. + + shlex is the right parser for a value naming a shell argument list, but it raises on an unbalanced + quote, and a path containing an apostrophe is enough to trigger it. + """ + raw = os.environ.get("CMAKE_ARGS", "") + try: + return shlex.split(raw) + except ValueError: + return raw.split() + + +@functools.lru_cache(maxsize=1) +def _detected_cuda_major() -> Optional[int]: + """The CUDA major version of the installed toolkit, or None if none is installed. + + Kept separate from `_get_cuda_version` because the mismatch guard in the wheel build + needs the major regardless of whether the exact (major, minor) is listed in + SUPPORTED_CUDA_VERSIONS. Reading through the validator caused the guard to see an + empty detection for any unlisted minor (say 12.8), so a cu130 row built on a CUDA 12 + toolkit produced a wheel with no error. + """ + try: + result = subprocess.run( + _selected_nvcc(), capture_output=True, text=True, check=True + ) + except (FileNotFoundError, subprocess.CalledProcessError, OSError): + return None + match = re.search(r"release (\d+)\.\d+", result.stdout) + return int(match.group(1)) if match else None + + def _extract_cmake_define(args: List[str], name: str) -> Optional[str]: """The value CMake would use for -D, which is the last one given. diff --git a/setup.py b/setup.py index bf08c5c84fd..5c72fc49435 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ import logging import os import re +import shlex import shutil import site import stat @@ -62,7 +63,7 @@ import sys from distutils import log # type: ignore[import-not-found] from distutils.sysconfig import get_python_lib # type: ignore[import-not-found] -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import List, Optional # Clean dynamic import using importlib @@ -206,6 +207,327 @@ def _minimal_packages() -> List[str]: ) +# The published project names for the CUDA runtime components a CUDA wheel links but +# does not bundle, keyed by CUDA major version. Not derivable from a suffix rule: the +# CUDA 12 wheels carry a "-cu12" suffix while the CUDA 13 ones are published under +# unsuffixed names. A train with no entry here declares nothing rather than guessing a +# name that may not exist. +# +# Only what a shipped library actually loads. Measured on a built wheel, the CUDA libraries need +# the CUDA runtime and cuRAND and nothing else, and the generated model library embeds its kernels +# rather than compiling them at run time, so there is no runtime compiler to satisfy either. +_CUDA_RUNTIME_PACKAGES = { + "12": ("nvidia-cuda-runtime-cu12",), + "13": ("nvidia-cuda-runtime",), +} + +# Where each train installs its libraries under site-packages. CUDA 13 collects them in +# one directory while CUDA 12 gives each component its own, so the search path differs by +# train and cannot be a single literal. +# +# Every declared package needs its directory here, and nothing else belongs. The loader only +# searches what is recorded here, so a missing directory leaves a shipped library unable to find +# a package that is installed, and an extra one implies a dependency the wheel does not have. +_CUDA_LIBRARY_DIRECTORIES = { + "12": ("nvidia/cuda_runtime/lib",), + "13": ("nvidia/cu13/lib",), +} + + +def _cmake_args() -> List[str]: + """CMAKE_ARGS split into arguments, tolerating an unbalanced quote. + + shlex is the correct parser for a value that names a shell argument list, but it raises on an + unbalanced quote, and a path containing an apostrophe is enough to trigger it. Both callers run at + module scope, so the exception surfaced as a traceback during the build rather than a diagnosable + error. Falling back to whitespace splitting keeps the build working for the case that caused it. + """ + raw = os.environ.get("CMAKE_ARGS", "") + try: + return shlex.split(raw) + except ValueError: + return raw.split() + + +def _row_is_cpu_only() -> bool: + """Whether the release row this build belongs to names itself a CPU row. + + The metadata side already reads the row to decide which NVIDIA packages to declare, so the build + has to read the same input or the two disagree and the wheel ships a delegate it cannot load. + Absent means unknown rather than CPU, which keeps a plain local build behaving as before. + """ + raw = ( + (os.environ.get("CU_VERSION") or os.environ.get("DESIRED_CUDA") or "") + .strip() + .lower() + ) + return raw == "cpu" + + +def _cuda_train() -> str: + """The CUDA major version this wheel is being built for, or "" for a CPU wheel. + + The release row's own field wins when it is set, because a row states the train it + targets and that is more authoritative than whichever toolkit happens to sit on the + builder. The wheel build exports CU_VERSION; DESIRED_CUDA is the matrix field name. + + Falling back to the installed toolkit matters for every build that is not a release + job. The build turns CUDA on by detecting a toolkit, so keying only off the release + field produced a wheel that carried the CUDA libraries with no dependency declarations + and no way to find the CUDA runtime. + + Returns "" when the build did not enable CUDA, so a CPU wheel declares nothing even on + a machine that has a toolkit installed. + + Raises when a release row names a train the installed toolkit does not provide. The + declared packages and the loader paths both come from this value, so disagreeing with + the toolkit that compiled the libraries produces a wheel that installs cleanly and then + cannot load: a cu126 row built against a 13.0 toolkit declares the CUDA 12 runtime for + binaries that need libcudart.so.13. + """ + # An explicit OFF first, ahead of the release field. A CPU row on a builder that has a + # toolkit installed sets both, so reading the row field first would declare a runtime + # the wheel never loads. + if not install_utils.is_cmake_option_on( + _cmake_args(), + "EXECUTORCH_BUILD_CUDA", + default=True, + ): + return "" + + raw = os.environ.get("CU_VERSION") or os.environ.get("DESIRED_CUDA") or "" + # A row spelled "cpu" is a CPU row regardless of CMAKE_ARGS. Recognised here so that a + # local build named that way with no CMAKE_ARGS set does not fall through and raise on + # the unsupported-train branch below. + if raw.lower() in ("cpu", "cpu-aarch64"): + return "" + # Reduce to digits and match against the same (major, minor) trains the shell classifier + # uses. Previously this took the first two digits and matched against major only, so a + # row spelled with an unsupported minor (say cu125) was classified CPU by the shell and + # CUDA 12 here, and the wheel then declared CUDA runtime packages for a CPU build. + digits = re.sub(r"[^0-9]", "", raw) + trains = { + f"{major}{minor}": str(major) + for major, minor in install_utils.SUPPORTED_CUDA_VERSIONS + } + requested = trains.get(digits, "") + + # Read the toolkit major directly, without the (major, minor) validator, so the guard + # below fires on any mismatch rather than only on the three listed pairs. + detected_major = install_utils._detected_cuda_major() + detected = ( + str(detected_major) + if detected_major is not None and str(detected_major) in _CUDA_RUNTIME_PACKAGES + else "" + ) + + if requested: + # A row that names a train has to be buildable for that train. Reported here + # rather than left to produce a mismatched wheel, because nothing downstream + # compares the two: the metadata comes from the row and the binaries come from + # the toolkit. + if detected and detected != requested: + raise RuntimeError( + f"this build targets CUDA {requested} (from " + f"{'CU_VERSION' if os.environ.get('CU_VERSION') else 'DESIRED_CUDA'}=" + f"{raw!r}) but the installed toolkit is CUDA {detected}. The declared " + "runtime packages and the loader search paths come from the requested " + "train while the libraries are compiled by the installed one, so the " + "wheel would install and then fail to load. Install a matching toolkit " + "or build the row that matches this one." + ) + return requested + + if raw and not requested: + # A row named something this packaging does not recognise. Silently reporting the + # builder's toolkit instead contradicts "the row's field wins" and produced a + # wheel tagged for one train carrying another. + supported = ", ".join( + f"cu{major}{minor}" + for major, minor in install_utils.SUPPORTED_CUDA_VERSIONS + ) + raise RuntimeError( + f"the release row requests CUDA {raw!r}, which is not a train this project " + f"supports ({supported}). Add it to SUPPORTED_CUDA_VERSIONS in install_utils " + "and to _CUDA_RUNTIME_PACKAGES and _CUDA_LIBRARY_DIRECTORIES here, or build " + "a supported row. Falling back to whatever toolkit this builder has would tag " + "the wheel for one train and fill it with another." + ) + + # Fall back to the installed toolkit, because keying this off a release variable alone produced a wheel + # that carried the CUDA libraries while declaring no CUDA runtime and recording no way to reach one. + # + # Two ways CUDA gets built, and both have to agree with what is declared here. The build gate turns it + # on when a SUPPORTED train is installed, so a toolkit whose minor is unlisted builds CPU-only and + # declaring runtime packages for it would make a CPU wheel demand four CUDA wheels. An explicit ON + # bypasses that gate and reaches CMake directly, where find_package(CUDAToolkit) accepts a toolkit + # this packaging does not list, so the libraries ship and the runtime has to be declared for them. + # Asking only whether the train is supported got the first case right and the second wrong. + explicit_on = install_utils.is_cmake_option_on( + _cmake_args(), + "EXECUTORCH_BUILD_CUDA", + default=False, + ) + if not install_utils.is_cuda_available() and not explicit_on: + return "" + return detected + + +def _cuda_libraries_built(cmake_cache_dir: Optional[str]) -> bool: + """Whether this build produced the CUDA libraries, read from the CMake cache. + + The build turns CUDA on from the cache, so the cache is the fact that decides what ships. The + release row's CUDA version is a different question: a build on a toolkit whose train this packaging + does not recognise still produces the libraries while declaring no train, and gating anything else on + the train left that wheel carrying libraries with no matching header. + + Falls back to the train when no cache is readable, which is the case for a source distribution where + nothing was built here anyway. + """ + cache_path = os.path.join(cmake_cache_dir or "", "CMakeCache.txt") + if os.path.exists(cache_path): + return CMakeCache(cache_path=cache_path).is_enabled("EXECUTORCH_BUILD_CUDA") + return bool(_cuda_train()) + + +def _verify_cuda_runtime_matches_train(cmake_cache_dir: Optional[str]) -> None: + """Fail the build when the linked CUDA runtime is not the train being declared. + + The declared packages come from the compiler version, while the library that actually gets + linked comes from find_package(CUDAToolkit). Those are normally the same toolkit, but + CUDAToolkit_ROOT steers the second and not the first, so they can split inside a single + find_package call: measured with the compiler at 13.0 and that variable at 12.8, + CUDAToolkit_VERSION reported 13.0.88 while the binary needed libcudart.so.12. Packaging + would then declare the CUDA 13 runtime for a wheel that cannot load without CUDA 12. + + Read from the CMake cache rather than from the environment, because the cache records what + the build resolved rather than what was requested. + """ + train = _cuda_train() + if not train: + return + cache_path = os.path.join(cmake_cache_dir or "", "CMakeCache.txt") + if not os.path.exists(cache_path): + return + cache = CMakeCache(cache_path=cache_path) + if not cache.is_enabled("EXECUTORCH_BUILD_CUDA"): + return + linked = cache.get("CUDA_cudart_LIBRARY") + if linked is None or not linked.value: + return + # The major is the part that decides the soname, and it is the only part the declared + # package names, so compare that rather than the full version. + found = re.search(r"cuda-(\d+)\.", linked.value) + if found is None or found.group(1) == train: + return + raise RuntimeError( + f"this build declares the CUDA {train} runtime but linked the CUDA " + f"{found.group(1)} one from {linked.value!r}, so the wheel would install and then " + "fail to load. The declared train follows the CUDA compiler while the linked " + "libraries follow find_package(CUDAToolkit), so point CUDACXX and CUDAToolkit_ROOT " + "at the same toolkit." + ) + + +def _cuda_dependencies() -> List[str]: + """Runtime libraries a CUDA wheel needs but does not bundle. + + Declared rather than vendored, the way the PyTorch CUDA wheels do it, so one copy is + shared with torch instead of shipping a second one. + """ + train = _cuda_train() + # Marked for Linux, because a CUDA wheel is only built there and these nvidia wheels publish no + # distribution for the other platforms, so an unmarked requirement would make a source install + # elsewhere fail on a dependency it cannot satisfy and does not need. + return [ + f"{name}; platform_system == 'Linux'" + for name in _CUDA_RUNTIME_PACKAGES.get(train, ()) + ] + + +# Directories inside the wheel that hold libraries a shipped library links, relative to the package +# root rather than to the linking library, because the wheel ships libraries at more than one depth. +# +# The CUDA libraries are split across two directories and reference each other in both directions: +# the delegate in lib/ links the shims library in backends/cuda/, and the shims library links the +# stream helper back in lib/. So both hops are needed. +# +# Applied to every shipped library rather than mapping each library to the directories it happens to +# need. An unused hop costs nothing at load time, while a missing one produces a wheel that installs +# and then fails to load, and a per-library mapping would have to be revisited every time a library +# moves. +_SIBLING_LIBRARY_DIRECTORIES = ("backends/cuda", "lib") + + +def _sibling_library_search_paths(depth: int = 1) -> List[str]: + """Loader paths that reach another directory inside this same package. + + `depth` is how many directories separate the linking library from the package root, and it has to + be honoured for the same reason the CUDA hops honour it: the wheel ships libraries at depth one + (lib/) and depth two (backends/cuda/, extension/pybindings/ and others). Measured with a fixed + pair sized for one depth, six of twelve hops landed somewhere that does not exist, and the hop + from lib/ escaped the package entirely into a sibling of it, where an unrelated library with a + matching SONAME could satisfy the dependency first. + """ + up = "/".join([".."] * depth) + return [f"$ORIGIN/{up}/{directory}" for directory in _SIBLING_LIBRARY_DIRECTORIES] + + +def _cuda_runtime_search_paths(depth: int = 1) -> List[str]: + """Loader paths that reach the CUDA wheels installed beside this one. + + Those wheels install as siblings of this package, so the hop has to climb out of the package first. + `depth` is how many directories separate the library from the package root, and the wheel ships + libraries at more than one depth: a hop sized for one of them lands inside this package from the + other, where nothing is found. + """ + train = _cuda_train() + out = "/".join([".."] * (depth + 1)) + return [ + f"$ORIGIN/{out}/{directory}" + for directory in _CUDA_LIBRARY_DIRECTORIES.get(train, ()) + ] + + +def _is_cuda_toolkit_directory(entry: str) -> bool: + """Whether a runtime search path entry names a library directory inside a CUDA toolkit. + + Matched on the two layouts a toolkit actually installs rather than on the word "cuda" appearing + somewhere above the directory. Scanning a window of components dropped a torch directory whose build + root happened to be named after a CUDA version, and torch's directory is the one absolute path a + shipped library has to keep. + + Position alone cannot separate the two, because a real targets layout puts the cuda-named component at + the same depth a build root does, so each layout is spelled out instead. + """ + parts = [part.lower() for part in PurePosixPath(entry).parts] + if not parts or parts[-1] not in ("lib", "lib64"): + return False + + def cuda_named(part: str) -> bool: + return bool(re.fullmatch(r"cuda(?:-\d+(?:\.\d+)*|[-_]?toolkit)?", part)) + + # /lib64 + if len(parts) >= 2 and cuda_named(parts[-2]): + return True + # /targets//lib + return len(parts) >= 4 and parts[-3] == "targets" and cuda_named(parts[-4]) + + +def _package_relative_depth(library: Path) -> int: + """How many directories separate a shipped library from the installed package root. + + Searched from the END of the path. At build time the path is absolute and a source checkout is + often named after the package too, so taking the first match found the checkout instead of the + package inside the build output and produced a hop that climbs out of the install directory. + """ + parts = list(Path(library).parts) + if "executorch" not in parts: + return 1 + index = len(parts) - 1 - parts[::-1].index("executorch") + return max(len(parts) - index - 2, 0) + + def _base_dependencies() -> List[str]: """Runtime dependencies for the full wheel. @@ -368,6 +690,20 @@ def get_dynamic_lib_name(name: str) -> str: return f"lib{name}.so" +def _dynamic_lib_suffix() -> str: + """The loadable-library suffix on this platform, including the dot. + + Separate from get_dynamic_lib_name because a file whose prefix is not known + ahead of time still needs the suffix named: globbing the suffix as well would + also match an import library, an exports file, or a soname's versioned links. + """ + if _is_windows(): + return ".dll" + if _is_macos(): + return ".dylib" + return ".so" + + def get_executable_name(name: str) -> str: if _is_windows(): return name + ".exe" @@ -731,10 +1067,85 @@ def build_extension(self, ext: _BaseExtension) -> None: if not os.access(dst_file, os.W_OK): os.chmod(dst_file, os.stat(dst_file).st_mode | stat.S_IWUSR) - _strip_absolute_runtime_paths(dst_file) + cmake_cache_dir = getattr( + self.get_finalized_command("build"), "cmake_cache_dir", None + ) + _strip_absolute_runtime_paths(dst_file, _cuda_libraries_built(cmake_cache_dir)) + + +def _append_relative_search_paths(entries: List[str], depth: int = 1) -> None: + """Add the relative hops a shipped library needs, skipping any already present. + + Two kinds, both relative so the wheel works wherever the environment lives: + the CUDA runtime, which arrives in its own wheel installed beside this one, and a sibling + ExecuTorch library that the wheel installs in a different directory from the library linking it. + + `depth` sizes the hop out of this package, since the wheel ships libraries at more than one depth. + """ + for search_path in ( + *_cuda_runtime_search_paths(depth), + *_sibling_library_search_paths(depth), + ): + if search_path not in entries: + entries.append(search_path) + + +def _is_usable_runtime_path( + entry: str, + safe_to_drop_toolkit_paths: bool, + has_relative_torch_route: bool, +) -> bool: + if not entry: + # The loader reads an empty entry as the process working directory. + return False + if not entry.startswith("/"): + return True + # Absolute, so decide by what it points at. + # + # A directory inside this build cannot exist for a user. + # + # A CUDA toolkit directory is dropped for a different reason: the wheel declares the CUDA runtime + # as a dependency and reaches it through a relative hop, so an absolute toolkit path is both + # unnecessary and harmful. It sits ahead of the hop, so a user who happens to have a toolkit at + # that prefix resolves the runtime from there instead of from the declared dependency. + # + # Whether that is safe is decided above, because the one case it is not is a CUDA build on an + # unrecognised train, which has no hop to fall back on. + # + # A torch lib directory is dropped when a relative route to torch is already recorded, since + # that route is what resolves torch on an installed wheel while the absolute one only names a + # directory from the machine that built it. It is kept when no relative route exists, because + # then it is the only way this library finds torch. + # + # Anything else absolute stays, because it is a dependency the environment provides and the wheel + # has no relative answer for. + # + # The build directories are matched as whole path components rather than as substrings. A bare + # "/cmake-out" also matches "/home/user/cmake-outputs/torchlibs", which is an unrelated directory + # a user could really have, and stripping it breaks a dependency the library resolves there. + # The setuptools staging directory is spelled build/lib.-, for example + # lib.linux-x86_64-cpython-312, so match the whole shape rather than any part starting "lib.". + if any( + part in ("pip-out", "cmake-out") + or re.fullmatch(r"lib\.[^/]+-(cpython-\d+|\d+(?:\.\d+)*)", part) + for part in entry.split("/") + ): + return False + # Matched on the layout a CUDA toolkit actually installs, not on the word "cuda" anywhere in the + # path. A substring test dropped a torch directory that merely sat under a directory named after a + # CUDA version, which is the one absolute path that has to survive. + if safe_to_drop_toolkit_paths and _is_cuda_toolkit_directory(entry): + return False + if entry.rstrip("/").endswith("/torch/lib") and has_relative_torch_route: + return False + return True -def _strip_absolute_runtime_paths(library: Path) -> None: +# Whether the library can still reach torch without the absolute entry. Read inside keep, +# which closes over this scope. + + +def _strip_absolute_runtime_paths(library: Path, ships_cuda: bool) -> None: """Remove unusable runtime search paths from a library the wheel ships. These libraries are copied out of the build tree rather than installed, so they @@ -783,48 +1194,33 @@ def _strip_absolute_runtime_paths(library: Path) -> None: # entry, so there is nothing to distinguish here and nothing to do either way. return - def keep(entry: str) -> bool: - if not entry: - # The loader reads an empty entry as the process working directory. - return False - if not entry.startswith("/"): - return True - # Absolute, so decide by what it points at. A directory inside this build - # cannot exist for a user. - # - # A torch lib directory is dropped when a relative route to torch is already - # recorded, since that route is what resolves torch on an installed wheel - # while the absolute one only names a directory from the machine that built - # it. It is kept when no relative route exists, because then it is the only - # way this library finds torch. - # - # Anything else absolute is a dependency the environment provides and the - # wheel has no relative answer for. - # - # Matched as whole path components rather than as substrings. A bare - # "/cmake-out" also matches "/home/user/cmake-outputs/torchlibs", which is - # an unrelated directory a user could really have, and stripping it breaks - # a dependency the library legitimately resolves there. - if entry.rstrip("/").endswith("/torch/lib") and has_relative_torch_route: - return False - parts = entry.split("/") - # The setuptools staging directory is spelled build/lib.-, - # for example lib.linux-x86_64-cpython-312. A bare startswith("lib.") also - # stripped a real user path like /opt/acme/lib.v2, so match the whole shape. - return not any( - part == "pip-out" - or part == "cmake-out" - or re.fullmatch(r"lib\.[^/]+-(cpython-\d+|\d+(?:\.\d+)*)", part) - for part in parts - ) + # Whether dropping an absolute CUDA toolkit path is safe. It is a cleanup when a relative hop replaces + # it, and also when this wheel carries no CUDA at all, because then nothing in it loads from that + # directory and the path only names the build machine. It is a regression only for a CUDA build whose + # train this packaging does not recognise, which declares no dependency and adds no hop, so dropping + # the path there would leave the delegate with no route to libcudart. + safe_to_drop_toolkit_paths = not ships_cuda or bool( + _cuda_runtime_search_paths(_package_relative_depth(library)) + ) - # Whether the library can still reach torch without the absolute entry. Read - # inside keep, which closes over this scope. has_relative_torch_route = any( not entry.startswith("/") and entry.rstrip("/").endswith("/torch/lib") for entry in original.split(":") ) - rewritten = ":".join(entry for entry in original.split(":") if keep(entry)) + entries = [ + entry + for entry in original.split(":") + if _is_usable_runtime_path( + entry, safe_to_drop_toolkit_paths, has_relative_torch_route + ) + ] + # A CUDA wheel links the CUDA runtime from a separate wheel installed beside this + # one, so the loader needs a relative hop to reach it. Without this the library + # resolves the runtime only through the absolute toolkit path the linker recorded, + # which names the build machine and will not exist for a user who installed from an + # index. Appended, so a path already present keeps its position. + _append_relative_search_paths(entries, _package_relative_depth(library)) + rewritten = ":".join(entries) if rewritten == original: return subprocess.run( @@ -890,6 +1286,9 @@ def run(self): ("schema/program.fbs", "exir/_serialize/program.fbs"), ] if not _is_minimal_build(): + cmake_cache_dir = getattr( + self.get_finalized_command("build"), "cmake_cache_dir", None + ) src_to_dst += [ ( "devtools/bundled_program/schema/bundled_program_schema.fbs", @@ -960,7 +1359,19 @@ def run(self): "devtools/etdump/emitter.h", "devtools/etdump/utils.h", "devtools/etdump/data_sinks/", - ]: + ] + ( + # The CUDA stream helper's public header, and the export macros it includes. Its library is + # shared so the process has one copy of the caller-stream state, and that is a handshake the + # caller takes part in, so a consumer needs the declarations to take part at all. + # + # Only when this wheel carries the CUDA delegate, and decided from the same CMake cache the + # libraries ship on. Keying it off the release row's CUDA version instead meant a build on + # an unrecognised toolkit shipped both CUDA libraries and both CMake components with no + # header, so a consumer got a component it could link and not include. + ["extension/cuda/caller_stream.h", "extension/cuda/export.h"] + if _cuda_libraries_built(cmake_cache_dir) + else [] + ): # A directory entry publishes everything under it, and a file entry publishes # just that file. Some directories hold headers a consumer cannot compile # against, so those are named individually rather than swept in. @@ -1167,11 +1578,36 @@ def run(self): # noqa C901 if minimal_build: cmake_configuration_args += _minimal_cmake_flags() - # Check if CUDA is available, and if so, enable building the CUDA - # backend by default. + # A row that names a CUDA train has already declared the NVIDIA runtime packages in + # install_requires, which is set before any build runs and therefore cannot consult the + # build. If no toolkit is reachable here, the wheel would ship no CUDA library while + # still making pip fetch the CUDA runtime, so stop rather than publish that mismatch. + # + # Tested against the same detection that produced the train, not against the stricter + # (major, minor) validator. The validator rejects a toolkit whose minor is unlisted, and + # an explicit EXECUTORCH_BUILD_CUDA=ON deliberately builds on such a toolkit, so asking + # the validator here reported "no toolkit" on a machine whose toolkit had just been used + # to derive the train, and refused the one case the fallback above exists to support. + if ( + not minimal_build + and _cuda_train() + and install_utils._detected_cuda_major() is None + ): + raise RuntimeError( + "this row names a CUDA train but no CUDA compiler was found, so the wheel " + "would declare the CUDA runtime and ship no CUDA library. Put nvcc on PATH or " + "point CUDACXX at it, or build the CPU row instead." + ) + + # Enable the CUDA delegate when a toolkit is present, unless the release row says this is a + # CPU wheel. Without that second condition a CPU row built on a machine that happens to have + # a toolkit produced a wheel carrying the delegate while its metadata declared no NVIDIA + # package and no way to reach one, so the delegate could not load. An explicit option still + # wins, so a caller can override the row on purpose. if ( not minimal_build and install_utils.is_cuda_available() + and not _row_is_cpu_only() and install_utils.is_cmake_option_on( cmake_configuration_args, "EXECUTORCH_BUILD_CUDA", default=True ) @@ -1255,6 +1691,9 @@ def run(self): # noqa C901 cmake_cache = CMakeCache( cache_path=os.path.join(cmake_cache_dir, "CMakeCache.txt") ) + # Checked here rather than after the build, because the cache already records which + # toolkit was resolved and reporting a mismatch is cheaper than compiling first. + _verify_cuda_runtime_matches_train(cmake_cache_dir) cmake_build_args = [ # Default build parallelism based on number of cores, but allow # overriding through the environment. @@ -1304,6 +1743,10 @@ def run(self): # noqa C901 if cmake_cache.is_enabled("EXECUTORCH_BUILD_CUDA"): cmake_build_args += ["--target", "aoti_cuda_backend"] cmake_build_args += ["--target", "aoti_common_shims_slim"] + if cmake_cache.is_enabled("EXECUTORCH_BUILD_SHARED"): + # The stream helper ships as its own library so a process has one + # of it. Named because nothing else in a wheel build links it. + cmake_build_args += ["--target", "extension_cuda"] if cmake_cache.is_enabled("EXECUTORCH_BUILD_EXTENSION_MODULE"): cmake_build_args += ["--target", "extension_module"] @@ -1355,7 +1798,9 @@ def run(self): # noqa C901 setup_kwargs["packages"] = _minimal_packages() setup_kwargs["install_requires"] = _minimal_dependencies() else: - setup_kwargs["install_requires"] = _base_dependencies() + # A CUDA wheel links the CUDA runtime but does not bundle it, so the wheels that + # carry it are declared here. A CPU wheel adds nothing. + setup_kwargs["install_requires"] = _base_dependencies() + _cuda_dependencies() setup( @@ -1444,6 +1889,39 @@ def run(self): # noqa C901 "EXECUTORCH_BUILD_KERNELS_OPTIMIZED", ], ), + # The CUDA delegate and the process-wide CUDA stream helper, for a + # wheel built from a CUDA index. Only present when the build asks for + # CUDA, so packaging requires that rather than looking for files a + # CPU-only build never produced. + BuiltFile( + src_dir="%CMAKE_CACHE_DIR%/backends/cuda/", + src_name="libexecutorch_backend_cuda.so", + dst="executorch/lib/libexecutorch_backend_cuda.so", + dependent_cmake_flags=[ + "EXECUTORCH_BUILD_SHARED", + "EXECUTORCH_BUILD_CUDA", + ], + ), + # The stream helper the delegate and the shim layer both record as a + # dependency. Globbed rather than named, because the file name depends on + # the build: a shared build renames it to libexecutorch_extension_cuda.so + # to match the other shipped components, and any other build leaves it as + # libextension_cuda.so. The shim ships whenever CUDA is on, so naming only + # the shared spelling left the non-shared build shipping a shim whose + # DT_NEEDED resolved to nothing. Two names means is_dynamic_lib cannot be + # used, since it builds one name and prepends a prefix the shared spelling + # does not have, so the prefix is globbed and the suffix is named. The + # build type is in the directory the way the sibling entries have it. + # Naming the suffix matters: this entry accepts exactly one file, and a + # bare wildcard also matches what a build leaves beside the library, an + # import library and an exports file on MSVC, or a soname's versioned + # links, and packaging then fails on a layout that is perfectly valid. + BuiltFile( + src_dir="%CMAKE_CACHE_DIR%/extension/cuda/%BUILD_TYPE%/", + src_name="*extension_cuda" + _dynamic_lib_suffix(), + dst="executorch/lib/", + dependent_cmake_flags=["EXECUTORCH_BUILD_CUDA"], + ), # The quantized kernels, as their own library rather than code # fused into the AOT-only extension beside the Python bindings. # A C++ application running a quantized model could not link @@ -1550,23 +2028,8 @@ def run(self): # noqa C901 is_dynamic_lib=True, dependent_cmake_flags=["EXECUTORCH_BUILD_CUDA"], ), - # The stream helper the library above records as a dependency. It was - # never shipped, and resolved only because the copied library still - # carried the absolute directory it was linked in, which exists on a - # build machine and nowhere else. Stripping that path is what made the - # omission visible as a failed import. - # - # Shipped beside its consumer rather than in lib/, because that - # directory only exists in the shared build and this has to work - # without it. The glob covers both names the target can have: the - # shared build renames it to advertise it as a wheel component. - BuiltFile( - src_dir="%CMAKE_CACHE_DIR%/extension/cuda/%BUILD_TYPE%/", - src_name="*extension_cuda", - dst="executorch/backends/cuda/", - is_dynamic_lib=True, - dependent_cmake_flags=["EXECUTORCH_BUILD_CUDA"], - ), + # The stream helper this library needs ships in lib/ from here on, + # alongside the other components a C++ consumer links. BuiltFile( src_dir="%CMAKE_CACHE_DIR%/backends/qualcomm/%BUILD_TYPE%/", src_name="qnn_executorch_backend", diff --git a/tools/cmake/executorch-wheel-config.cmake b/tools/cmake/executorch-wheel-config.cmake index 02f9bb34a48..ec849a42954 100644 --- a/tools/cmake/executorch-wheel-config.cmake +++ b/tools/cmake/executorch-wheel-config.cmake @@ -313,9 +313,11 @@ if(_executorch_runtime_library AND NOT _executorch_targets_supported) # route has no per-component target to opt into, so they are offered through # EXECUTORCH_QUANTIZED_KERNELS_LIBRARY instead and a consumer that wants them # links that as well. - foreach(_executorch_component IN - ITEMS libexecutorch_kernels_optimized libexecutorch_backend_xnnpack - libexecutorch_threadpool libexecutorch_etdump + foreach( + _executorch_component IN + ITEMS libexecutorch_kernels_optimized libexecutorch_backend_xnnpack + libexecutorch_backend_cuda libexecutorch_extension_cuda + libexecutorch_threadpool libexecutorch_etdump ) _executorch_find_library( _executorch_component_library "${_executorch_component}" @@ -611,6 +613,11 @@ if(TARGET executorch::runtime AND TARGET executorch::threadpool) endif() _executorch_define_component(backend_xnnpack executorch_backend_xnnpack) +# The CUDA delegate and its stream helper, present only in a wheel built from a +# CUDA index. A CPU wheel defines neither, so a consumer asking for one is told +# while configuring. +_executorch_define_component(backend_cuda executorch_backend_cuda) +_executorch_define_component(extension_cuda executorch_extension_cuda) # Find prebuilt _portable_lib..so. This is the legacy contract used # to build custom-op extensions against the Python module, and is kept working diff --git a/tools/cmake/preset/pybind.cmake b/tools/cmake/preset/pybind.cmake index 609b9f9ddd9..957afd05742 100644 --- a/tools/cmake/preset/pybind.cmake +++ b/tools/cmake/preset/pybind.cmake @@ -113,15 +113,7 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") # consumers link, so a process has a single backend registry. Linux only: # macOS C++ consumers are served by the Swift package distribution, and the # runtime has no export annotations for a Windows DLL. - # - # Not with the CUDA backend, whose libraries this build does not ship yet. The - # CUDA libraries currently reach the wheel carrying the absolute path of the - # directory they were linked in, which resolves only on the machine that built - # them. The shared build removes those paths, so enabling it here before the - # CUDA libraries ship would leave the extension unable to load at all. - if(NOT EXECUTORCH_BUILD_CUDA) - set_overridable_option(EXECUTORCH_BUILD_SHARED ON) - endif() + set_overridable_option(EXECUTORCH_BUILD_SHARED ON) elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows" OR CMAKE_SYSTEM_NAME STREQUAL "WIN32" )