From a834bc0e998b67bfb357b1f2cd9ae8f12ec122c4 Mon Sep 17 00:00:00 2001 From: Mike Mattice Date: Wed, 21 Oct 2026 10:35:59 -0500 Subject: [PATCH] [Python] Export only PyInit_* from macOS extension modules Each extension statically links protobuf, abseil and cel-cpp. On macOS, pybind_extension links with only `-undefined dynamic_lookup`, so those copies are exported, including thousands of weak definitions. dyld coalesces exported weak definitions across every loaded image, so importing cel_expr_python alongside another extension that ships protobuf (e.g. ortools) binds one runtime's code to the other's parse tables and crashes in AddDescriptors at import time. Wrap pybind_extension to add `-Wl,-exported_symbol,_PyInit_` on macOS, matching the isolation Linux already gets from `-Wl,-Bsymbolic`, and add a macOS test that each extension exports only its init function. Fixes #115 --- cel_expr_python/BUILD | 34 +++++++++++- cel_expr_python/exported_symbols_test.py | 66 ++++++++++++++++++++++++ cel_expr_python/ext/BUILD | 2 +- cel_expr_python/pybind_extension.bzl | 40 ++++++++++++++ custom_ext/BUILD | 2 +- 5 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 cel_expr_python/exported_symbols_test.py create mode 100644 cel_expr_python/pybind_extension.bzl diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index 02c8f9c..323574d 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -1,6 +1,7 @@ -load("@pybind11_bazel//:build_defs.bzl", "pybind_extension", "pybind_library") +load("@pybind11_bazel//:build_defs.bzl", "pybind_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python:py_test.bzl", "py_test") +load(":pybind_extension.bzl", "pybind_extension") package(default_visibility = ["//visibility:public"]) @@ -209,6 +210,37 @@ py_test( }), ) +py_test( + name = "exported_symbols_test", + srcs = ["exported_symbols_test.py"], + data = [ + ":cel", + "//cel_expr_python/ext:ext_bindings", + "//cel_expr_python/ext:ext_encoders", + "//cel_expr_python/ext:ext_lists", + "//cel_expr_python/ext:ext_math", + "//cel_expr_python/ext:ext_optional", + "//cel_expr_python/ext:ext_proto", + "//cel_expr_python/ext:ext_strings", + ], + deps = [ + "@com_google_absl_py//absl/testing:absltest", + "@com_google_absl_py//absl/testing:parameterized", + ] + select({ + "@platforms//os:windows": [], + "//conditions:default": [ + ":cel", + "//cel_expr_python/ext:ext_bindings", + "//cel_expr_python/ext:ext_encoders", + "//cel_expr_python/ext:ext_lists", + "//cel_expr_python/ext:ext_math", + "//cel_expr_python/ext:ext_optional", + "//cel_expr_python/ext:ext_proto", + "//cel_expr_python/ext:ext_strings", + ], + }), +) + # Platform definition for macOS x86_64. # Used for cross-compiling from arm64 macOS hosts (Apple Silicon) to x86_64. # This is passed to Bazel via --platforms in setup.py when building x86_64 wheels. diff --git a/cel_expr_python/exported_symbols_test.py b/cel_expr_python/exported_symbols_test.py new file mode 100644 index 0000000..98b9785 --- /dev/null +++ b/cel_expr_python/exported_symbols_test.py @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests that macOS extension modules export only their init function. + +Exported copies of the statically linked protobuf and abseil runtimes are +coalesced by dyld with any other copy loaded in the process (for example +ortools'), which crashes at import time. +""" + +import subprocess +import sys +import types + +from absl.testing import absltest +from absl.testing import parameterized +from cel_expr_python import cel +from cel_expr_python.ext import ext_bindings +from cel_expr_python.ext import ext_encoders +from cel_expr_python.ext import ext_lists +from cel_expr_python.ext import ext_math +from cel_expr_python.ext import ext_optional +from cel_expr_python.ext import ext_proto +from cel_expr_python.ext import ext_strings + +_MODULES = ( + cel, + ext_bindings, + ext_encoders, + ext_lists, + ext_math, + ext_optional, + ext_proto, + ext_strings, +) + + +@absltest.skipUnless(sys.platform == "darwin", "macOS-specific linkage") +class ExportedSymbolsTest(parameterized.TestCase): + + @parameterized.parameters(*_MODULES) + def test_exports_only_init_function(self, module: types.ModuleType): + output = subprocess.run( + ["nm", "-gU", module.__file__], + check=True, + capture_output=True, + text=True, + ).stdout + exported = {line.split()[-1] for line in output.splitlines() if line} + init_symbol = "_PyInit_" + module.__name__.rpartition(".")[2] + self.assertEqual(exported, {init_symbol}) + + +if __name__ == "__main__": + absltest.main() diff --git a/cel_expr_python/ext/BUILD b/cel_expr_python/ext/BUILD index c90eb14..b391193 100644 --- a/cel_expr_python/ext/BUILD +++ b/cel_expr_python/ext/BUILD @@ -1,4 +1,4 @@ -load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") +load("//cel_expr_python:pybind_extension.bzl", "pybind_extension") package(default_visibility = ["//visibility:public"]) diff --git a/cel_expr_python/pybind_extension.bzl b/cel_expr_python/pybind_extension.bzl new file mode 100644 index 0000000..7aae2d2 --- /dev/null +++ b/cel_expr_python/pybind_extension.bzl @@ -0,0 +1,40 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""pybind_extension wrapper that keeps statically linked dependencies private.""" + +load("@pybind11_bazel//:build_defs.bzl", _pybind_extension = "pybind_extension") + +def pybind_extension(name, linkopts = [], **kwargs): + """Builds a pybind11 extension module that exports only its init function. + + Each extension statically links its own copy of protobuf, abseil and + cel-cpp. On macOS, dyld coalesces exported weak definitions across every + loaded image, so exporting those copies lets them bind to another + extension's protobuf or abseil (e.g. ortools') and crash at import time. + Linux is unaffected because pybind_extension links with -Bsymbolic there. + + Args: + name: The name of the extension module. + linkopts: Additional linker options. + **kwargs: Passed through to pybind11_bazel's pybind_extension. + """ + _pybind_extension( + name = name, + linkopts = linkopts + select({ + "@platforms//os:osx": ["-Wl,-exported_symbol,_PyInit_" + name], + "//conditions:default": [], + }), + **kwargs + ) diff --git a/custom_ext/BUILD b/custom_ext/BUILD index 2329467..dbc9e90 100644 --- a/custom_ext/BUILD +++ b/custom_ext/BUILD @@ -1,6 +1,6 @@ -load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") load("@rules_python//python:py_library.bzl", "py_library") load("@rules_python//python:py_test.bzl", "py_test") +load("//cel_expr_python:pybind_extension.bzl", "pybind_extension") package(default_visibility = ["//visibility:private"])