Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -260,12 +260,17 @@ jobs:
os: ubuntu-26.04-arm

build-macos-relocatable:
name: 'macOS (relocatable)'
name: 'macOS (relocatable ${{ matrix.framework }})'
needs: build-context
if: needs.build-context.outputs.run-macos == 'true'
strategy:
fail-fast: false
matrix:
framework: ['', 'Python', 'RelocatablePython']
uses: ./.github/workflows/reusable-relocatable.yml
with:
os: macos-26
framework: ${{ matrix.framework }}

build-ubuntu-ssltests:
name: 'Ubuntu SSL tests'
Expand Down
36 changes: 20 additions & 16 deletions .github/workflows/reusable-relocatable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ on:
os:
type: string
required: true
framework:
type: string
default: ''

permissions:
contents: read
Expand All @@ -22,6 +25,7 @@ jobs:
run:
shell: bash
env:
FRAMEWORK: ${{ inputs.framework }}
PYTHONSTRICTEXTENSIONBUILD: 1
HOMEBREW_NO_ANALYTICS: 1
HOMEBREW_NO_AUTO_UPDATE: 1
Expand All @@ -46,24 +50,16 @@ jobs:
echo "GDBM_CFLAGS=-I$(brew --prefix gdbm)/include" >> "$GITHUB_ENV"
echo "GDBM_LIBS=-L$(brew --prefix gdbm)/lib -lgdbm" >> "$GITHUB_ENV"
echo "OPENSSL_DIR=$(brew --prefix openssl@3.5)" >> "$GITHUB_ENV"
- name: Reject unsupported framework relocation
if: runner.os == 'macOS'
run: |
mkdir "$RUNNER_TEMP/framework-configure"
cd "$RUNNER_TEMP/framework-configure"
if "$GITHUB_WORKSPACE/configure" \
--enable-framework="$RUNNER_TEMP/framework-install" \
--enable-relocatable --with-openssl="$OPENSSL_DIR" > configure.log 2>&1; then
cat configure.log
exit 1
fi
cat configure.log
grep -F 'does not support framework builds yet' configure.log
- name: Configure CPython
run: |
mkdir "$BUILD_DIR"
cd "$BUILD_DIR"
configure_args=(--prefix="$INSTALL_DIR" --enable-shared --enable-relocatable)
configure_args=(--prefix="$INSTALL_DIR" --enable-relocatable)
if [ -n "$FRAMEWORK" ]; then
configure_args+=(--enable-framework="$INSTALL_DIR/Library/Frameworks" --with-framework-name="$FRAMEWORK")
else
configure_args+=(--enable-shared)
fi
if [ -n "${OPENSSL_DIR:-}" ]; then
configure_args+=(--with-openssl="$OPENSSL_DIR")
fi
Expand Down Expand Up @@ -100,7 +96,15 @@ jobs:
run: |
unset LD_LIBRARY_PATH LD_PRELOAD
unset DYLD_LIBRARY_PATH DYLD_FALLBACK_LIBRARY_PATH DYLD_INSERT_LIBRARIES
installed_python="$INSTALL_DIR/bin/python3"
unset DYLD_FRAMEWORK_PATH DYLD_FALLBACK_FRAMEWORK_PATH
export PYTHON_PREFIX="$INSTALL_DIR"
if [ -n "$FRAMEWORK" ]; then
framework_dir="$INSTALL_DIR/Library/Frameworks/$FRAMEWORK.framework"
framework_version=$(readlink "$framework_dir/Versions/Current")
export PYTHON_PREFIX="$framework_dir/Versions/$framework_version"
"$PYTHON_PREFIX/Resources/Python.app/Contents/MacOS/$FRAMEWORK" -I -c 'import ssl, ctypes; print("Direct app launch succeeded")'
fi
installed_python="$PYTHON_PREFIX/bin/python3"
"$installed_python" -I - <<'PY'
import os
import subprocess
Expand All @@ -109,7 +113,7 @@ jobs:
import tempfile
import venv

prefix = os.path.realpath(os.environ['INSTALL_DIR'])
prefix = os.path.realpath(os.environ['PYTHON_PREFIX'])
assert os.path.realpath(sys.prefix) == prefix, sys.prefix
assert sys.exec_prefix == sys.prefix, sys.exec_prefix
assert sysconfig.get_config_var('RELOCATABLE') == 'yes'
Expand Down
17 changes: 11 additions & 6 deletions Doc/using/configure.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1081,18 +1081,23 @@ Linker options

This feature is under active development. Do not use it in production.

Currently, this sets relative runtime library paths on Linux and for
non-framework macOS builds. Framework relocation is not implemented yet.
Combining this option with :option:`--enable-framework` is an error.
Currently, this sets relative runtime library paths on Linux and macOS,
including macOS framework builds.

Move the installation as a unit. It must use the default ``bin`` and ``lib``
directories under a common prefix. :option:`--with-platlibdir` may select
a different directory name for the standard library, but not a nested path.
On Linux, these paths use ``DT_RPATH``.

Copied virtual environments from non-framework shared builds require
directory symlinks to the base installation's libraries. Virtual environments
depend on their base installation and must be recreated if it moves.
When :mod:`venv` copies an interpreter from a shared or framework build,
it creates a symlink to the base library directory or the directory
containing the framework. This requires filesystem support for symlinks
even with ``--copies``. Virtual environments depend on their base
installation and must be recreated if it moves.

A macOS framework can be moved as a unit, including its internal
``Resources/Python.app``. External command symlinks and application launchers
are not rewritten; update any that point to the original location.

Dependencies stored in other directories, such as ``lib/tcltk``, may
need additional loader search paths.
Expand Down
43 changes: 29 additions & 14 deletions Lib/test/test_venv.py
Original file line number Diff line number Diff line change
Expand Up @@ -1079,7 +1079,15 @@ def test_zippath_from_non_installed_posix(self):
bindir = os.path.join(non_installed_dir, self.bindir)
os.mkdir(bindir)
python_exe = os.path.basename(sys.executable)
shutil.copy2(sys.executable, os.path.join(bindir, python_exe))
framework = sysconfig.get_config_var("PYTHONFRAMEWORK")
source_executable = sys.executable
if framework and not sysconfig.is_python_build():
# The framework launcher looks for the app relative to the loaded runtime.
# This fixture lacks that app bundle, so copy its interpreter directly.
source_executable = os.path.join(
sys.base_exec_prefix, 'Resources', 'Python.app', 'Contents',
'MacOS', framework)
shutil.copy2(source_executable, os.path.join(bindir, python_exe))
if sys.platform == 'cygwin':
# Copy libpython DLL
exe_path = os.path.dirname(sys.executable)
Expand Down Expand Up @@ -1132,27 +1140,34 @@ def test_zippath_from_non_installed_posix(self):
# venv module correctly.
pythonpath = os.pathsep.join(
additional_pythonpath_for_non_installed)
# For python built with shared enabled. We need to set
# LD_LIBRARY_PATH so the non-installed python can find and link
# libpython.so
ld_library_path = sysconfig.get_config_var("LIBDIR")
if (sysconfig.get_config_var("RELOCATABLE") == "yes"
and not sysconfig.get_config_var("PYTHONFRAMEWORK")):
ld_library_path = os.path.join(sys.base_exec_prefix, "lib")
# Set the library search path so the non-installed interpreter can
# load its shared library or framework.
if framework:
ld_library_path = sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX")
else:
ld_library_path = sysconfig.get_config_var("LIBDIR")
if sysconfig.get_config_var("RELOCATABLE") == "yes":
if framework:
ld_library_path = os.path.abspath(os.path.join(
sys.base_exec_prefix, '..', '..', '..'))
else:
ld_library_path = os.path.join(sys.base_exec_prefix, "lib")
if not ld_library_path or sysconfig.is_python_build():
ld_library_path = os.path.abspath(os.path.dirname(sys.executable))
if (sysconfig.get_config_var('Py_ENABLE_SHARED')
if ((sysconfig.get_config_var('Py_ENABLE_SHARED') or framework)
and sys.platform != 'cygwin'):
# Startup can search relative to libpython before the executable.
# Startup can search relative to the runtime library before the executable.
# Load a copy from the temporary installation to select its
# standard library.
shared_libdir = os.path.join(non_installed_dir, 'lib')
os.makedirs(shared_libdir, exist_ok=True)
instsoname = sysconfig.get_config_var('INSTSONAME')
shutil.copy2(os.path.join(ld_library_path, instsoname),
os.path.join(shared_libdir, instsoname))
copied_library = os.path.join(shared_libdir, instsoname)
os.makedirs(os.path.dirname(copied_library), exist_ok=True)
shutil.copy2(os.path.join(ld_library_path, instsoname), copied_library)
ld_library_path = os.pathsep.join((shared_libdir, ld_library_path))
if sys.platform == 'darwin':
if framework:
ld_library_path_env = "DYLD_FRAMEWORK_PATH"
elif sys.platform == 'darwin':
ld_library_path_env = "DYLD_LIBRARY_PATH"
else:
ld_library_path_env = "LD_LIBRARY_PATH"
Expand Down
8 changes: 6 additions & 2 deletions Lib/venv/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,12 +311,16 @@ def setup_python(self, context):
if not os.path.islink(path):
os.chmod(path, 0o755)
if (sysconfig.get_config_var('RELOCATABLE') == 'yes'
and sysconfig.get_config_var('Py_ENABLE_SHARED')
and not sysconfig.get_config_var('PYTHONFRAMEWORK')):
and (sysconfig.get_config_var('Py_ENABLE_SHARED')
or sysconfig.get_config_var('PYTHONFRAMEWORK'))):
# The copied executable searches relative to this environment.
# Link the base library directory so it can find libpython.
if sysconfig.is_python_build():
library_dir = context.python_dir
elif sysconfig.get_config_var('PYTHONFRAMEWORK'):
# The install name includes the framework directory.
library_dir = os.path.abspath(os.path.join(
sys.base_exec_prefix, '..', '..', '..'))
else:
library_dir = os.path.join(sys.base_exec_prefix, 'lib')
library_link = os.path.join(
Expand Down
14 changes: 13 additions & 1 deletion Mac/Makefile.in
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@ LIBDEST=$(prefix)/lib/python$(VERSION)
RUNSHARED=@RUNSHARED@
BUILDEXE=@BUILDEXEEXT@
BUILDPYTHON=$(builddir)/python$(BUILDEXE)
RELOCATABLE=@RELOCATABLE@
RELOCATABLE_DIR=$(builddir)/build/relocatable
RELOCATABLE_PYTHONW_LDFLAGS=@RELOCATABLE_PYTHONW_LDFLAGS@
INSTALL_PYTHON=$(BUILDPYTHON)
ifeq ($(RELOCATABLE),yes)
INSTALL_PYTHON=$(RELOCATABLE_DIR)/python$(BUILDEXE)
endif
DESTDIR=
LDFLAGS=@LDFLAGS@
FRAMEWORKUNIXTOOLSPREFIX=@FRAMEWORKUNIXTOOLSPREFIX@
Expand Down Expand Up @@ -177,6 +184,11 @@ pythonw: $(srcdir)/Tools/pythonw.c Makefile
$(srcdir)/Tools/pythonw.c -I.. -I$(srcdir)/../Include \
../$(PYTHONFRAMEWORK).framework/Versions/$(VERSION)/$(PYTHONFRAMEWORK)

pythonw-relocatable: $(srcdir)/Tools/pythonw.c Makefile $(RELOCATABLE_DIR)/$(PYTHONFRAMEWORK).framework/Versions/$(VERSION)/$(PYTHONFRAMEWORK)
$(CC) $(LDFLAGS) $(RELOCATABLE_PYTHONW_LDFLAGS) -DPYTHONFRAMEWORK='"$(PYTHONFRAMEWORK)"' -o $@ \
$(srcdir)/Tools/pythonw.c -I.. -I$(srcdir)/../Include \
$(RELOCATABLE_DIR)/$(PYTHONFRAMEWORK).framework/Versions/$(VERSION)/$(PYTHONFRAMEWORK)

install_PythonLauncher:
cd PythonLauncher && make install DESTDIR=$(DESTDIR)

Expand Down Expand Up @@ -223,7 +235,7 @@ install_Python:
esac; \
done; \
done
$(INSTALL_PROGRAM) $(STRIPFLAG) $(BUILDPYTHON) "$(DESTDIR)$(APPINSTALLDIR)/Contents/MacOS/$(PYTHONFRAMEWORK)"
$(INSTALL_PROGRAM) $(STRIPFLAG) $(INSTALL_PYTHON) "$(DESTDIR)$(APPINSTALLDIR)/Contents/MacOS/$(PYTHONFRAMEWORK)"
sed -e "s!%bundleid%!$(PYTHONFRAMEWORKIDENTIFIER)!g" \
-e "s!%version%!`$(RUNSHARED) $(BUILDPYTHON) \
-c 'import platform; print(platform.python_version())'`!g" \
Expand Down
27 changes: 20 additions & 7 deletions Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -321,16 +321,21 @@ BUILDPYTHON= python$(BUILDEXE)
# runtime paths relative to their installed locations.
RELOCATABLE_DIR=build/relocatable
INSTALL_PYTHON=$(BUILDPYTHON)
INSTALL_PYTHONW=Mac/pythonw
INSTALL_LDLIBRARY=$(LDLIBRARY)
INSTALL_PY3LIBRARY=$(PY3LIBRARY)
INSTALL_SHAREDMODS=$(SHAREDMODS)
ifeq ($(RELOCATABLE),yes)
INSTALL_PYTHON=$(RELOCATABLE_DIR)/$(BUILDPYTHON)
INSTALL_PYTHONW=Mac/pythonw-relocatable
INSTALL_SHAREDMODS=$(addprefix $(RELOCATABLE_DIR)/,$(SHAREDMODS))
ifeq ($(PY_ENABLE_SHARED),1)
INSTALL_LDLIBRARY=$(RELOCATABLE_DIR)/$(LDLIBRARY)
INSTALL_PY3LIBRARY=$(addprefix $(RELOCATABLE_DIR)/,$(PY3LIBRARY))
endif
ifneq ($(PYTHONFRAMEWORK),)
INSTALL_LDLIBRARY=$(RELOCATABLE_DIR)/$(LDLIBRARY)
endif
endif

HOSTRUNNER= @HOSTRUNNER@
Expand Down Expand Up @@ -1009,7 +1014,7 @@ $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS)

$(RELOCATABLE_DIR)/$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) $(INSTALL_LDLIBRARY)
@mkdir -p $(@D)
$(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) $(RELOCATABLE_PYTHON_LDFLAGS) -L$(RELOCATABLE_DIR) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS)
$(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(subst $(LDLIBRARY),$(INSTALL_LDLIBRARY),$(LINKFORSHARED)) $(RELOCATABLE_PYTHON_LDFLAGS) -L$(RELOCATABLE_DIR) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS)

platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt
$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform
Expand Down Expand Up @@ -1108,6 +1113,14 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \
$(LN) -fsn Versions/Current/$(PYTHONFRAMEWORK) $(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)
$(LN) -fsn Versions/Current/Resources $(PYTHONFRAMEWORKDIR)/Resources

$(RELOCATABLE_DIR)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): $(LIBRARY)
@mkdir -p $(@D)
$(CC) -o $@ $(PY_CORE_LDFLAGS) $(RELOCATABLE_LIBPYTHON_LDFLAGS) -dynamiclib \
-all_load $(LIBRARY) \
-install_name $(RELOCATABLE_LIBPYTHON_INSTALL_NAME) \
-compatibility_version $(VERSION) -current_version $(VERSION) \
-framework CoreFoundation $(LIBS)

# This rule is for iOS, which requires an annoyingly just slightly different
# format for frameworks to macOS. It *doesn't* use a versioned framework, and
# the Info.plist must be in the root of the framework.
Expand Down Expand Up @@ -2473,7 +2486,7 @@ endif
if test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \
$(INSTALL_PROGRAM) $(INSTALL_PYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \
else \
$(INSTALL_PROGRAM) $(STRIPFLAG) Mac/pythonw $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \
$(INSTALL_PROGRAM) $(STRIPFLAG) $(INSTALL_PYTHONW) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \
fi
-if test "$(VERSION)" != "$(LDVERSION)"; then \
if test -f $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE) -o -h $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE); \
Expand Down Expand Up @@ -3075,7 +3088,7 @@ frameworkinstallframework: @FRAMEWORKINSTALLFIRST@ install frameworkinstallmacli

# macOS uses a versioned frameworks structure that includes a full install
.PHONY: frameworkinstallversionedstructure
frameworkinstallversionedstructure: $(LDLIBRARY)
frameworkinstallversionedstructure: all
@if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \
echo Not configured with --enable-framework; \
exit 1; \
Expand All @@ -3094,7 +3107,7 @@ frameworkinstallversionedstructure: $(LDLIBRARY)
$(LN) -fsn Versions/Current/$(PYTHONFRAMEWORK) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/$(PYTHONFRAMEWORK)
$(LN) -fsn Versions/Current/Headers $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers
$(LN) -fsn Versions/Current/Resources $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Resources
$(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY)
$(INSTALL_SHARED) $(INSTALL_LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY)

# iOS/tvOS/watchOS uses a non-versioned framework with Info.plist in the
# framework root, no .lproj data, and only stub compilation assistance binaries
Expand Down Expand Up @@ -3141,8 +3154,8 @@ frameworkinstallapps:
# an app bundle within the framework. This allows the interpreter to
# run OS X GUI APIs.
.PHONY: frameworkpythonw
frameworkpythonw:
cd Mac && $(MAKE) pythonw
frameworkpythonw: $(INSTALL_LDLIBRARY)
cd Mac && $(MAKE) $(notdir $(INSTALL_PYTHONW))

# This installs the python* and other bin symlinks in $prefix/bin or in
# a bin directory relative to the framework root
Expand Down Expand Up @@ -3336,7 +3349,7 @@ clean-retain-profile: pycremoval
-rm -f pybuilddir.txt
-rm -f _bootstrap_python
-rm -rf web_example python.mjs python.wasm python*.symbols python*.map
-rm -f Programs/_testembed Programs/_freeze_module
-rm -f Programs/_testembed Programs/_freeze_module Mac/pythonw-relocatable
-rm -rf Python/deepfreeze
-rm -f Python/frozen_modules/*.h
-rm -f Python/frozen_modules/MANIFEST
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Support :option:`--enable-relocatable` for macOS framework builds. Use relative
framework library paths for installed interpreters and extensions, and link
copied virtual environments to the base framework.
23 changes: 6 additions & 17 deletions Modules/_testinternalcapi.c
Original file line number Diff line number Diff line change
Expand Up @@ -261,25 +261,14 @@ static const char *
classify_address(uintptr_t addr, int jit_enabled, PyInterpreterState *interp)
{
Dl_info info;
if (dladdr((void *)addr, &info) != 0
&& info.dli_fname != NULL
&& info.dli_fname[0] != '\0') {
const char *base = strrchr(info.dli_fname, '/');
base = base ? base + 1 : info.dli_fname;
if (strncmp(base, "python", 6) == 0) {
Dl_info python_info;
if (dladdr((void *)addr, &info) != 0) {
// Framework library names need not start with `python` or `libpython`.
// Identify Python by the image containing `Py_Initialize`.
if (dladdr(&Py_Initialize, &python_info) != 0
&& info.dli_fbase == python_info.dli_fbase) {
return "python";
}
#ifdef __CYGWIN__
// Match Cygwin "cygpython3.16.dll"
if (strncmp(base, "cygpython", 9) == 0) {
return "python";
}
#else
// Match "libpython3.15.so.1.0"
if (strncmp(base, "libpython", 9) == 0) {
return "python";
}
#endif
return "other";
}
#ifdef _Py_JIT
Expand Down
Loading
Loading