[mypyc] Internally support static linking mode

mypyc/codegen/emit.py

@@ -153,6 +153,9 @@ def __init__(
         self.names = names
         self.group_name = group_name
         self.group_map = group_map or {}
+        # Suffix that makes names of C globals local to the group unique across groups,
+        # so that groups can be statically linked together (empty if not using groups)
+        self.group_suffix = "_" + exported_name(group_name) if group_name else ""
         # Groups that this group depends on
         self.group_deps: set[str] = set()
 
@@ -349,7 +352,9 @@ def static_name(self, id: str, module: str | None, prefix: str = STATIC_PREFIX)
         # the pointer also.
         star_maybe = "*" if lib_prefix else ""
         suffix = self.names.private_name(module or "", id)
-        return f"{star_maybe}{lib_prefix}{prefix}{suffix}"
+        # Statics without a module are local to the group, so make them unique per group
+        group_suffix = "" if module else self.context.group_suffix
+        return f"{star_maybe}{lib_prefix}{prefix}{suffix}{group_suffix}"
 
     def type_struct_name(self, cl: ClassIR) -> str:
         return self.static_name(cl.name, cl.module_name, prefix=TYPE_PREFIX)

mypyc/codegen/emitfunc.py

@@ -394,10 +394,16 @@ def visit_load_error_value(self, op: LoadErrorValue) -> None:
     def visit_load_literal(self, op: LoadLiteral) -> None:
         index = self.literals.literal_index(op.value)
         if not is_int_rprimitive(op.type):
-            self.emit_line("%s = CPyStatics[%d];" % (self.reg(op), index), ann=op.value)
+            self.emit_line(
+                "%s = CPyStatics%s[%d];"
+                % (self.reg(op), self.emitter.context.group_suffix, index),
+                ann=op.value,
+            )
         else:
             self.emit_line(
-                "%s = (CPyTagged)CPyStatics[%d] | 1;" % (self.reg(op), index), ann=op.value
+                "%s = (CPyTagged)CPyStatics%s[%d] | 1;"
+                % (self.reg(op), self.emitter.context.group_suffix, index),
+                ann=op.value,
             )
 
     def get_attr_expr(self, obj: str, op: GetAttr | SetAttr, decl_cl: ClassIR) -> str:

mypyc/lib-rt/base64/librt_base64.c

@@ -4,6 +4,13 @@
 
 #define BASE64_EXPORTS
 
+// With static linking, compiled code calls the API functions and refers to the
+// type objects directly, so they need external linkage
+#ifdef MYPYC_STATIC_LINKING
+#define LIBRT_API_LINKAGE
+#else
+#define LIBRT_API_LINKAGE static
+#endif
 #include "librt_base64.h"
 #include "libbase64.h"
 #include "pythoncapi_compat.h"
@@ -44,8 +51,8 @@ convert_urlsafe_to_encoded(const char *src, size_t len, char *buf) {
     }
 }
 
-static PyObject *
-b64encode_internal(PyObject *obj, bool urlsafe) {
+LIBRT_API_LINKAGE PyObject *
+LibRTBase64_b64encode_internal(PyObject *obj, bool urlsafe) {
     unsigned char *ascii_data;
     char *bin_data;
     int leftbits = 0;
@@ -99,7 +106,7 @@ b64encode(PyObject *self, PyObject *const *args, size_t nargs) {
         PyErr_SetString(PyExc_TypeError, "b64encode() takes exactly one argument");
         return 0;
     }
-    return b64encode_internal(args[0], false);
+    return LibRTBase64_b64encode_internal(args[0], false);
 }
 
 static PyObject*
@@ -108,7 +115,7 @@ urlsafe_b64encode(PyObject *self, PyObject *const *args, size_t nargs) {
         PyErr_SetString(PyExc_TypeError, "urlsafe_b64encode() takes exactly one argument");
         return 0;
     }
-    return b64encode_internal(args[0], true);
+    return LibRTBase64_b64encode_internal(args[0], true);
 }
 
 static inline int
@@ -117,8 +124,8 @@ is_valid_base64_char(char c, bool allow_padding) {
             (c >= '0' && c <= '9') || (c == '+') || (c == '/') || (allow_padding && c == '='));
 }
 
-static PyObject *
-b64decode_internal(PyObject *arg, bool urlsafe) {
+LIBRT_API_LINKAGE PyObject *
+LibRTBase64_b64decode_internal(PyObject *arg, bool urlsafe) {
     const char *src;
     Py_ssize_t srclen_ssz;
 
@@ -307,7 +314,7 @@ b64decode(PyObject *self, PyObject *const *args, size_t nargs) {
         PyErr_SetString(PyExc_TypeError, "b64decode() takes exactly one argument");
         return 0;
     }
-    return b64decode_internal(args[0], false);
+    return LibRTBase64_b64decode_internal(args[0], false);
 }
 
 static PyObject*
@@ -316,7 +323,7 @@ urlsafe_b64decode(PyObject *self, PyObject *const *args, size_t nargs) {
         PyErr_SetString(PyExc_TypeError, "urlsafe_b64decode() takes exactly one argument");
         return 0;
     }
-    return b64decode_internal(args[0], true);
+    return LibRTBase64_b64decode_internal(args[0], true);
 }
 
 static PyMethodDef librt_base64_module_methods[] = {
@@ -327,30 +334,32 @@ static PyMethodDef librt_base64_module_methods[] = {
     {NULL, NULL, 0, NULL}
 };
 
-static int
-base64_abi_version(void) {
+LIBRT_API_LINKAGE int
+LibRTBase64_ABIVersion(void) {
     return LIBRT_BASE64_ABI_VERSION;
 }
 
-static int
-base64_api_version(void) {
+LIBRT_API_LINKAGE int
+LibRTBase64_APIVersion(void) {
     return LIBRT_BASE64_API_VERSION;
 }
 
 static int
 librt_base64_module_exec(PyObject *m)
 {
+#ifndef MYPYC_STATIC_LINKING
     // Export mypy internal C API, be careful with the order!
     static void *base64_api[LIBRT_BASE64_API_LEN] = {
-        (void *)base64_abi_version,
-        (void *)base64_api_version,
-        (void *)b64encode_internal,
-        (void *)b64decode_internal,
+        (void *)LibRTBase64_ABIVersion,
+        (void *)LibRTBase64_APIVersion,
+        (void *)LibRTBase64_b64encode_internal,
+        (void *)LibRTBase64_b64decode_internal,
     };
     PyObject *c_api_object = PyCapsule_New((void *)base64_api, "librt.base64._C_API", NULL);
     if (PyModule_Add(m, "_C_API", c_api_object) < 0) {
         return -1;
     }
+#endif
     return 0;
 }
 

mypyc/lib-rt/base64/librt_base64.h

@@ -7,4 +7,14 @@
 #define LIBRT_BASE64_API_VERSION 2
 #define LIBRT_BASE64_API_LEN 4
 
+#ifdef MYPYC_STATIC_LINKING
+// With static linking, the API is used directly (see librt_base64_api.h)
+#include <stdbool.h>
+#include <stdint.h>
+int LibRTBase64_ABIVersion(void);
+int LibRTBase64_APIVersion(void);
+PyObject *LibRTBase64_b64encode_internal(PyObject *obj, bool urlsafe);
+PyObject *LibRTBase64_b64decode_internal(PyObject *arg, bool urlsafe);
+#endif
+
 #endif  // LIBRT_BASE64_H

mypyc/lib-rt/base64/librt_base64_api.c

@@ -1,6 +1,8 @@
 #include "librt_base64_api.h"
 
+#ifndef MYPYC_STATIC_LINKING
 void *LibRTBase64_API[LIBRT_BASE64_API_LEN] = {0};
+#endif
 
 int
 import_librt_base64(void)
@@ -9,6 +11,11 @@ import_librt_base64(void)
     if (mod == NULL)
         return -1;
     Py_DECREF(mod);  // we import just for the side effect of making the below work.
+#ifdef MYPYC_STATIC_LINKING
+    // librt is statically linked, so the API is used directly. Importing the
+    // module above initializes it (e.g. its types).
+    return 0;
+#else
     void **capsule = (void **)PyCapsule_Import("librt.base64._C_API", 0);
     if (capsule == NULL)
         return -1;
@@ -40,4 +47,5 @@ import_librt_base64(void)
     // entries, so this copy is safe.
     memcpy(LibRTBase64_API, capsule, sizeof(LibRTBase64_API));
     return 0;
+#endif
 }

mypyc/lib-rt/base64/librt_base64_api.h

@@ -3,12 +3,17 @@
 
 #include "librt_base64.h"
 
+#ifdef MYPYC_STATIC_LINKING
+// librt is statically linked, so the API functions (declared in librt_base64.h) are
+// called directly, and type objects are referred to directly
+#else
 extern void *LibRTBase64_API[LIBRT_BASE64_API_LEN];
 
 #define LibRTBase64_ABIVersion (*(int (*)(void)) LibRTBase64_API[0])
 #define LibRTBase64_APIVersion (*(int (*)(void)) LibRTBase64_API[1])
 #define LibRTBase64_b64encode_internal (*(PyObject* (*)(PyObject *source, bool urlsafe)) LibRTBase64_API[2])
 #define LibRTBase64_b64decode_internal (*(PyObject* (*)(PyObject *source, bool urlsafe)) LibRTBase64_API[3])
+#endif
 
 int import_librt_base64(void);