From 993deaf823a151c102951a856af16ab51fb941ad Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Wed, 22 Jul 2026 08:22:55 -0700 Subject: [PATCH] Use ctypes for Python bindings --- SConscript | 10 +- SConstruct | 15 +- msgq/ipc.pxd | 68 -------- msgq/ipc_ctypes.cc | 64 ++++++++ msgq/ipc_ctypes.h | 49 ++++++ msgq/ipc_pyx.py | 189 +++++++++++++++++++++ msgq/ipc_pyx.pyx | 256 ----------------------------- msgq/visionipc/visionipc.pxd | 53 ------ msgq/visionipc/visionipc_ctypes.cc | 71 ++++++++ msgq/visionipc/visionipc_ctypes.h | 43 +++++ msgq/visionipc/visionipc_pyx.pxd | 10 -- msgq/visionipc/visionipc_pyx.py | 142 ++++++++++++++++ msgq/visionipc/visionipc_pyx.pyx | 170 ------------------- pyproject.toml | 5 - site_scons/site_tools/cython.py | 72 -------- 15 files changed, 564 insertions(+), 653 deletions(-) delete mode 100644 msgq/ipc.pxd create mode 100644 msgq/ipc_ctypes.cc create mode 100644 msgq/ipc_ctypes.h create mode 100644 msgq/ipc_pyx.py delete mode 100644 msgq/ipc_pyx.pyx delete mode 100644 msgq/visionipc/visionipc.pxd create mode 100644 msgq/visionipc/visionipc_ctypes.cc create mode 100644 msgq/visionipc/visionipc_ctypes.h delete mode 100644 msgq/visionipc/visionipc_pyx.pxd create mode 100644 msgq/visionipc/visionipc_pyx.py delete mode 100644 msgq/visionipc/visionipc_pyx.pyx delete mode 100644 site_scons/site_tools/cython.py diff --git a/SConscript b/SConscript index cde36529e..738f7c9a1 100644 --- a/SConscript +++ b/SConscript @@ -1,4 +1,4 @@ -Import('env', 'envCython', 'common') +Import('env', 'common') visionipc_dir = Dir('msgq/visionipc') @@ -12,7 +12,8 @@ msgq_objects = env.SharedObject([ 'msgq/msgq.cc', ]) msgq = env.Library('msgq', msgq_objects) -msgq_python = envCython.Program('msgq/ipc_pyx.so', 'msgq/ipc_pyx.pyx', LIBS=envCython["LIBS"]+[msgq]+common) +env.SharedLibrary('msgq/libipc_ctypes.so', ['msgq/ipc_ctypes.cc'], LIBS=[msgq]+common) +msgq_python = File('msgq/ipc_pyx.py') # Build Vision IPC vipc_files = ['visionipc.cc', 'visionipc_server.cc', 'visionipc_client.cc'] @@ -26,9 +27,8 @@ vipc_objects = env.SharedObject(vipc_sources) visionipc = env.Library('visionipc', vipc_objects) -vipc_libs = envCython["LIBS"] + [visionipc, msgq] + common -envCython.Program(f'{visionipc_dir.abspath}/visionipc_pyx.so', f'{visionipc_dir.abspath}/visionipc_pyx.pyx', - LIBS=vipc_libs) +env.SharedLibrary(f'{visionipc_dir.abspath}/libvisionipc_ctypes.so', [f'{visionipc_dir.abspath}/visionipc_ctypes.cc'], + LIBS=[visionipc, msgq]+common) if GetOption('extras'): env.Program('msgq/test_runner', ['msgq/msgq_tests.cc'], LIBS=[msgq]+common) diff --git a/SConstruct b/SConstruct index 0ee7a1dba..992c0149f 100644 --- a/SConstruct +++ b/SConstruct @@ -1,7 +1,6 @@ import os import platform import subprocess -import sysconfig arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() if platform.system() == "Darwin": @@ -13,7 +12,6 @@ cpppath = [ "#/", '#msgq/', '/usr/lib/include', - sysconfig.get_paths()['include'], ] AddOption('--minimal', @@ -61,21 +59,10 @@ env = Environment( CFLAGS="-std=gnu11", CXXFLAGS="-std=c++1z", CPPPATH=cpppath, - CYTHONCFILESUFFIX=".cpp", - tools=["default", "cython"] + tools=["default"] ) Export('env', 'arch', 'common') -envCython = env.Clone(LIBS=[]) -envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"] -envCython["CCFLAGS"].remove('-Werror') -if arch == "Darwin": - envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] -else: - envCython["LINKFLAGS"] = ["-pthread", "-shared"] - -Export('envCython') - SConscript(['SConscript']) diff --git a/msgq/ipc.pxd b/msgq/ipc.pxd deleted file mode 100644 index 0760561d2..000000000 --- a/msgq/ipc.pxd +++ /dev/null @@ -1,68 +0,0 @@ -# distutils: language = c++ -#cython: language_level=3 - -from libcpp.string cimport string -from libcpp.vector cimport vector -from libcpp cimport bool - - -cdef extern from "msgq/impl_fake.h": - cdef cppclass Event: - @staticmethod - int wait_for_one(vector[Event], int) except + - - Event() - Event(int) - void set() - int clear() - void wait(int) except + - bool peek() - int fd() - - cdef cppclass SocketEventHandle: - @staticmethod - void toggle_fake_events(bool) - @staticmethod - void set_fake_prefix(string) - @staticmethod - string fake_prefix() - - SocketEventHandle(string, string, bool) - bool is_enabled() - void set_enabled(bool) - Event recv_called() - Event recv_ready() - - -cdef extern from "msgq/ipc.h": - cdef cppclass Context: - @staticmethod - Context * create() - - cdef cppclass Message: - void init(size_t) - void init(char *, size_t) - void close() - size_t getSize() - char *getData() - - cdef cppclass SubSocket: - @staticmethod - SubSocket * create() nogil - int connect(Context *, string, string, bool, bool, size_t) nogil - Message * receive(bool) nogil - void setTimeout(int) nogil - - cdef cppclass PubSocket: - @staticmethod - PubSocket * create() - int connect(Context *, string, bool, size_t) - int sendMessage(Message *) - int send(char *, size_t) - bool all_readers_updated() - - cdef cppclass Poller: - @staticmethod - Poller * create() - void registerSocket(SubSocket *) - vector[SubSocket*] poll(int) nogil diff --git a/msgq/ipc_ctypes.cc b/msgq/ipc_ctypes.cc new file mode 100644 index 000000000..2cfb6d170 --- /dev/null +++ b/msgq/ipc_ctypes.cc @@ -0,0 +1,64 @@ +#include "msgq/ipc_ctypes.h" + +#include +#include +#include +#include + +#include "msgq/event.h" +#include "msgq/ipc.h" + +extern "C" void *msgq_context_create() { return Context::create(); } +extern "C" void msgq_context_delete(void *p) { delete static_cast(p); } +extern "C" void *msgq_sub_create() { return SubSocket::create(); } +extern "C" void msgq_sub_delete(void *p) { delete static_cast(p); } +extern "C" int msgq_sub_connect(void *s, void *c, const char *e, const char *a, int cf, size_t z) { + return static_cast(s)->connect(static_cast(c), e, a, cf, true, z); +} +extern "C" void msgq_sub_set_timeout(void *s, int t) { static_cast(s)->setTimeout(t); } +extern "C" void *msgq_sub_receive(void *s, int nb) { return static_cast(s)->receive(nb); } +extern "C" size_t msgq_message_size(void *m) { return static_cast(m)->getSize(); } +extern "C" const char *msgq_message_data(void *m) { return static_cast(m)->getData(); } +extern "C" void msgq_message_delete(void *m) { delete static_cast(m); } +extern "C" void *msgq_pub_create() { return PubSocket::create(); } +extern "C" void msgq_pub_delete(void *p) { delete static_cast(p); } +extern "C" int msgq_pub_connect(void *s, void *c, const char *e, size_t z) { + return static_cast(s)->connect(static_cast(c), e, true, z); +} +extern "C" int msgq_pub_send(void *s, const char *d, size_t z) { return static_cast(s)->send(const_cast(d), z); } +extern "C" int msgq_pub_all_readers_updated(void *s) { return static_cast(s)->all_readers_updated(); } +extern "C" void *msgq_poller_create() { return Poller::create(); } +extern "C" void msgq_poller_delete(void *p) { delete static_cast(p); } +extern "C" void msgq_poller_register(void *p, void *s) { static_cast(p)->registerSocket(static_cast(s)); } +extern "C" size_t msgq_poller_poll(void *p, int t, void **out, size_t capacity) { + const auto sockets = static_cast(p)->poll(t); + const size_t count = std::min(capacity, sockets.size()); + std::copy_n(sockets.begin(), count, out); + return count; +} + +extern "C" void msgq_toggle_fake_events(int e) { SocketEventHandle::toggle_fake_events(e); } +extern "C" void msgq_set_fake_prefix(const char *p) { SocketEventHandle::set_fake_prefix(p); } +extern "C" size_t msgq_get_fake_prefix(char *out, size_t capacity) { + const std::string prefix = SocketEventHandle::fake_prefix(); + if (capacity) memcpy(out, prefix.data(), std::min(capacity, prefix.size())); + return prefix.size(); +} +extern "C" void *msgq_event_handle_create(const char *e, const char *i, int o) { return new SocketEventHandle(e, i, o); } +extern "C" void msgq_event_handle_delete(void *h) { delete static_cast(h); } +extern "C" int msgq_event_handle_enabled(void *h) { return static_cast(h)->is_enabled(); } +extern "C" void msgq_event_handle_set_enabled(void *h, int e) { static_cast(h)->set_enabled(e); } +extern "C" void *msgq_event_handle_recv_called(void *h) { return new Event(static_cast(h)->recv_called()); } +extern "C" void *msgq_event_handle_recv_ready(void *h) { return new Event(static_cast(h)->recv_ready()); } +extern "C" void msgq_event_delete(void *e) { delete static_cast(e); } +extern "C" int msgq_event_set(void *e) { try { static_cast(e)->set(); return 0; } catch (...) { return -1; } } +extern "C" int msgq_event_clear(void *e) { try { return static_cast(e)->clear(); } catch (...) { return -1; } } +extern "C" int msgq_event_wait(void *e, int t) { try { static_cast(e)->wait(t); return 0; } catch (...) { return -1; } } +extern "C" int msgq_event_peek(void *e) { try { return static_cast(e)->peek(); } catch (...) { return -1; } } +extern "C" int msgq_event_fd(void *e) { return static_cast(e)->fd(); } +extern "C" int msgq_event_wait_for_one(void **items, size_t count, int timeout) { + std::vector events; + events.reserve(count); + for (size_t i = 0; i < count; ++i) events.push_back(*static_cast(items[i])); + try { return Event::wait_for_one(events, timeout); } catch (...) { return -1; } +} diff --git a/msgq/ipc_ctypes.h b/msgq/ipc_ctypes.h new file mode 100644 index 000000000..3f42e489d --- /dev/null +++ b/msgq/ipc_ctypes.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +void *msgq_context_create(void); +void msgq_context_delete(void *context); +void *msgq_sub_create(void); +void msgq_sub_delete(void *socket); +int msgq_sub_connect(void *socket, void *context, const char *endpoint, const char *address, int conflate, size_t segment_size); +void msgq_sub_set_timeout(void *socket, int timeout); +void *msgq_sub_receive(void *socket, int non_blocking); +size_t msgq_message_size(void *message); +const char *msgq_message_data(void *message); +void msgq_message_delete(void *message); +void *msgq_pub_create(void); +void msgq_pub_delete(void *socket); +int msgq_pub_connect(void *socket, void *context, const char *endpoint, size_t segment_size); +int msgq_pub_send(void *socket, const char *data, size_t size); +int msgq_pub_all_readers_updated(void *socket); +void *msgq_poller_create(void); +void msgq_poller_delete(void *poller); +void msgq_poller_register(void *poller, void *socket); +size_t msgq_poller_poll(void *poller, int timeout, void **sockets, size_t capacity); + +void msgq_toggle_fake_events(int enabled); +void msgq_set_fake_prefix(const char *prefix); +size_t msgq_get_fake_prefix(char *prefix, size_t capacity); +void *msgq_event_handle_create(const char *endpoint, const char *identifier, int override); +void msgq_event_handle_delete(void *handle); +int msgq_event_handle_enabled(void *handle); +void msgq_event_handle_set_enabled(void *handle, int enabled); +void *msgq_event_handle_recv_called(void *handle); +void *msgq_event_handle_recv_ready(void *handle); +void msgq_event_delete(void *event); +int msgq_event_set(void *event); +int msgq_event_clear(void *event); +int msgq_event_wait(void *event, int timeout); +int msgq_event_peek(void *event); +int msgq_event_fd(void *event); +int msgq_event_wait_for_one(void **events, size_t count, int timeout); + +#ifdef __cplusplus +} +#endif diff --git a/msgq/ipc_pyx.py b/msgq/ipc_pyx.py new file mode 100644 index 000000000..4c6b8b53f --- /dev/null +++ b/msgq/ipc_pyx.py @@ -0,0 +1,189 @@ +import ctypes +import errno +import os +import time +import weakref +from pathlib import Path + + +lib = ctypes.CDLL(str(Path(__file__).with_name("libipc_ctypes.so")), use_errno=True) +c_void_p = ctypes.c_void_p +c_size_t = ctypes.c_size_t +c_int = ctypes.c_int +c_char_p = ctypes.c_char_p + +lib.msgq_context_create.restype = c_void_p +lib.msgq_context_delete.argtypes = (c_void_p,) +lib.msgq_sub_create.restype = c_void_p +lib.msgq_sub_delete.argtypes = (c_void_p,) +lib.msgq_sub_connect.argtypes = (c_void_p, c_void_p, c_char_p, c_char_p, c_int, c_size_t) +lib.msgq_sub_set_timeout.argtypes = (c_void_p, c_int) +lib.msgq_sub_receive.argtypes = (c_void_p, c_int) +lib.msgq_sub_receive.restype = c_void_p +lib.msgq_message_size.argtypes = (c_void_p,) +lib.msgq_message_size.restype = c_size_t +lib.msgq_message_data.argtypes = (c_void_p,) +lib.msgq_message_data.restype = c_void_p +lib.msgq_message_delete.argtypes = (c_void_p,) +lib.msgq_pub_create.restype = c_void_p +lib.msgq_pub_delete.argtypes = (c_void_p,) +lib.msgq_pub_connect.argtypes = (c_void_p, c_void_p, c_char_p, c_size_t) +lib.msgq_pub_send.argtypes = (c_void_p, c_char_p, c_size_t) +lib.msgq_pub_all_readers_updated.argtypes = (c_void_p,) +lib.msgq_poller_create.restype = c_void_p +lib.msgq_poller_delete.argtypes = (c_void_p,) +lib.msgq_poller_register.argtypes = (c_void_p, c_void_p) +lib.msgq_poller_poll.argtypes = (c_void_p, c_int, ctypes.POINTER(c_void_p), c_size_t) +lib.msgq_poller_poll.restype = c_size_t +lib.msgq_toggle_fake_events.argtypes = (c_int,) +lib.msgq_set_fake_prefix.argtypes = (c_char_p,) +lib.msgq_get_fake_prefix.argtypes = (c_void_p, c_size_t) +lib.msgq_get_fake_prefix.restype = c_size_t +lib.msgq_event_handle_create.argtypes = (c_char_p, c_char_p, c_int) +lib.msgq_event_handle_create.restype = c_void_p +lib.msgq_event_handle_delete.argtypes = (c_void_p,) +lib.msgq_event_handle_enabled.argtypes = (c_void_p,) +lib.msgq_event_handle_set_enabled.argtypes = (c_void_p, c_int) +lib.msgq_event_handle_recv_called.argtypes = (c_void_p,) +lib.msgq_event_handle_recv_called.restype = c_void_p +lib.msgq_event_handle_recv_ready.argtypes = (c_void_p,) +lib.msgq_event_handle_recv_ready.restype = c_void_p +lib.msgq_event_delete.argtypes = (c_void_p,) +for name in ("set", "clear", "peek", "fd"): + getattr(lib, f"msgq_event_{name}").argtypes = (c_void_p,) +lib.msgq_event_wait.argtypes = (c_void_p, c_int) +lib.msgq_event_wait_for_one.argtypes = (ctypes.POINTER(c_void_p), c_size_t, c_int) + + +def _bytes(value): + return value.encode() if isinstance(value, str) else value + + +class IpcError(Exception): + def __init__(self, endpoint=None): + suffix = f"with {endpoint.decode('utf-8')}" if endpoint else "" + super().__init__(f"Messaging failure {suffix}: {os.strerror(ctypes.get_errno())}") + + +class MultiplePublishersError(IpcError): + pass + + +def _check(result, endpoint=None): + if result != 0: + if ctypes.get_errno() == errno.EADDRINUSE: + raise MultiplePublishersError(endpoint) + raise IpcError(endpoint) + + +def toggle_fake_events(enabled): lib.msgq_toggle_fake_events(enabled) +def set_fake_prefix(prefix): lib.msgq_set_fake_prefix(_bytes(prefix)) +def get_fake_prefix(): + size = lib.msgq_get_fake_prefix(None, 0) + result = ctypes.create_string_buffer(size) + lib.msgq_get_fake_prefix(result, size) + return result.raw +def delete_fake_prefix(): set_fake_prefix(b"") + + +class Event: + def __init__(self, ptr): + self._ptr = ptr + self._finalizer = weakref.finalize(self, lib.msgq_event_delete, ptr) + def set(self): + if lib.msgq_event_set(self._ptr) < 0: + raise RuntimeError("event operation failed") + def clear(self): return lib.msgq_event_clear(self._ptr) + def wait(self, timeout=-1): + if lib.msgq_event_wait(self._ptr, timeout) < 0: + raise RuntimeError("event timed out") + def peek(self): return bool(lib.msgq_event_peek(self._ptr)) + @property + def fd(self): return lib.msgq_event_fd(self._ptr) + @property + def ptr(self): return self._ptr + + +def wait_for_one_event(events, timeout=-1): + items = (c_void_p * len(events))(*(event._ptr for event in events)) + result = lib.msgq_event_wait_for_one(items, len(events), timeout) + if result < 0: + raise RuntimeError("event wait failed") + return result + + +class SocketEventHandle: + def __init__(self, endpoint, identifier, override): + self._ptr = lib.msgq_event_handle_create(_bytes(endpoint), _bytes(identifier), override) + self._finalizer = weakref.finalize(self, lib.msgq_event_handle_delete, self._ptr) + @property + def enabled(self): return bool(lib.msgq_event_handle_enabled(self._ptr)) + @enabled.setter + def enabled(self, value): lib.msgq_event_handle_set_enabled(self._ptr, value) + @property + def recv_called_event(self): return Event(lib.msgq_event_handle_recv_called(self._ptr)) + @property + def recv_ready_event(self): return Event(lib.msgq_event_handle_recv_ready(self._ptr)) + + +class Context: + def __init__(self): + self._ptr = lib.msgq_context_create() + self._finalizer = weakref.finalize(self, lib.msgq_context_delete, self._ptr) + def term(self): + if self._finalizer.alive: + self._finalizer() + self._ptr = None + + +class SubSocket: + def __init__(self): + self._ptr = lib.msgq_sub_create() + self._finalizer = weakref.finalize(self, lib.msgq_sub_delete, self._ptr) + def connect(self, context, endpoint, address=b"127.0.0.1", conflate=False, segment_size=0): + encoded_endpoint = _bytes(endpoint) + _check(lib.msgq_sub_connect(self._ptr, context._ptr, encoded_endpoint, _bytes(address), conflate, segment_size), encoded_endpoint) + def setTimeout(self, timeout): lib.msgq_sub_set_timeout(self._ptr, timeout) + def receive(self, non_blocking=False): + message = lib.msgq_sub_receive(self._ptr, non_blocking) + if not message: + return None + try: + return ctypes.string_at(lib.msgq_message_data(message), lib.msgq_message_size(message)) + finally: + lib.msgq_message_delete(message) + + +class PubSocket: + def __init__(self): + self._ptr = lib.msgq_pub_create() + self._finalizer = weakref.finalize(self, lib.msgq_pub_delete, self._ptr) + def connect(self, context, endpoint, segment_size=0): + encoded_endpoint = _bytes(endpoint) + _check(lib.msgq_pub_connect(self._ptr, context._ptr, encoded_endpoint, segment_size), encoded_endpoint) + def send(self, data): + result = lib.msgq_pub_send(self._ptr, data, len(data)) + if result != len(data): + _check(result) + def all_readers_updated(self): return bool(lib.msgq_pub_all_readers_updated(self._ptr)) + def wait_for_readers(self, timeout=1.0, interval=0.001): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self.all_readers_updated(): + return + time.sleep(interval) + raise TimeoutError("subscriber did not connect") + + +class Poller: + def __init__(self): + self._ptr = lib.msgq_poller_create() + self._finalizer = weakref.finalize(self, lib.msgq_poller_delete, self._ptr) + self._sockets = {} + def registerSocket(self, socket): + self._sockets[socket._ptr] = socket + lib.msgq_poller_register(self._ptr, socket._ptr) + def poll(self, timeout): + result = (c_void_p * max(1, len(self._sockets)))() + count = lib.msgq_poller_poll(self._ptr, int(timeout), result, len(self._sockets)) + return [self._sockets[result[i]] for i in range(count)] diff --git a/msgq/ipc_pyx.pyx b/msgq/ipc_pyx.pyx deleted file mode 100644 index b16e9702e..000000000 --- a/msgq/ipc_pyx.pyx +++ /dev/null @@ -1,256 +0,0 @@ -# distutils: language = c++ -# cython: c_string_encoding=ascii, language_level=3 - -import time -from libcpp.string cimport string -from libcpp.vector cimport vector -from libcpp cimport bool -from libc cimport errno -from libc.string cimport strerror -from cython.operator import dereference - - -from .ipc cimport Context as cppContext -from .ipc cimport SubSocket as cppSubSocket -from .ipc cimport PubSocket as cppPubSocket -from .ipc cimport Poller as cppPoller -from .ipc cimport Message as cppMessage -from .ipc cimport Event as cppEvent, SocketEventHandle as cppSocketEventHandle - - -class IpcError(Exception): - def __init__(self, endpoint=None): - suffix = f"with {endpoint.decode('utf-8')}" if endpoint else "" - message = f"Messaging failure {suffix}: {strerror(errno.errno).decode('utf-8')}" - super().__init__(message) - - -class MultiplePublishersError(IpcError): - pass - - -def toggle_fake_events(bool enabled): - cppSocketEventHandle.toggle_fake_events(enabled) - - -def set_fake_prefix(string prefix): - cppSocketEventHandle.set_fake_prefix(prefix) - - -def get_fake_prefix(): - return cppSocketEventHandle.fake_prefix() - - -def delete_fake_prefix(): - cppSocketEventHandle.set_fake_prefix(b"") - - -def wait_for_one_event(list events, int timeout=-1): - cdef vector[cppEvent] items - for event in events: - items.push_back(dereference(event.ptr)) - return cppEvent.wait_for_one(items, timeout) - - -cdef class Event: - cdef cppEvent event; - - cdef setEvent(self, cppEvent event): - self.event = event - - def set(self): - self.event.set() - - def clear(self): - return self.event.clear() - - def wait(self, int timeout=-1): - self.event.wait(timeout) - - def peek(self): - return self.event.peek() - - @property - def fd(self): - return self.event.fd() - - @property - def ptr(self): - return &self.event - - -cdef class SocketEventHandle: - cdef cppSocketEventHandle * handle; - - def __cinit__(self, string endpoint, string identifier, bool override): - self.handle = new cppSocketEventHandle(endpoint, identifier, override) - - def __dealloc__(self): - del self.handle - - @property - def enabled(self): - return self.handle.is_enabled() - - @enabled.setter - def enabled(self, bool value): - self.handle.set_enabled(value) - - @property - def recv_called_event(self): - e = Event() - e.setEvent(self.handle.recv_called()) - - return e - - @property - def recv_ready_event(self): - e = Event() - e.setEvent(self.handle.recv_ready()) - - return e - - -cdef class Context: - cdef cppContext * context - - def __cinit__(self): - self.context = cppContext.create() - - def term(self): - del self.context - self.context = NULL - - def __dealloc__(self): - pass - # Deleting the context will hang if sockets are still active - # TODO: Figure out a way to make sure the context is closed last - # del self.context - - -cdef class Poller: - cdef cppPoller * poller - cdef list sub_sockets - - def __cinit__(self): - self.sub_sockets = [] - self.poller = cppPoller.create() - - def __dealloc__(self): - del self.poller - - def registerSocket(self, SubSocket socket): - self.sub_sockets.append(socket) - self.poller.registerSocket(socket.socket) - - def poll(self, timeout): - sockets = [] - cdef int t = timeout - - with nogil: - result = self.poller.poll(t) - - for s in result: - socket = SubSocket() - socket.setPtr(s) - sockets.append(socket) - - return sockets - - -cdef class SubSocket: - cdef cppSubSocket * socket - cdef bool is_owner - - def __cinit__(self): - with nogil: - self.socket = cppSubSocket.create() - - self.is_owner = True - if self.socket == NULL: - raise IpcError - - def __dealloc__(self): - if self.is_owner: - with nogil: - del self.socket - - cdef setPtr(self, cppSubSocket * ptr): - if self.is_owner: - with nogil: - del self.socket - - self.is_owner = False - self.socket = ptr - - def connect(self, Context context, string endpoint, string address=b"127.0.0.1", bool conflate=False, size_t segment_size=0): - cdef int r - with nogil: - r = self.socket.connect(context.context, endpoint, address, conflate, True, segment_size) - - if r != 0: - if errno.errno == errno.EADDRINUSE: - raise MultiplePublishersError(endpoint) - else: - raise IpcError(endpoint) - - def setTimeout(self, int timeout): - with nogil: - self.socket.setTimeout(timeout) - - def receive(self, bool non_blocking=False): - cdef cppMessage *msg - with nogil: - msg = self.socket.receive(non_blocking) - - if msg == NULL: - return None - else: - sz = msg.getSize() - m = msg.getData()[:sz] - with nogil: - del msg - - return m - - -cdef class PubSocket: - cdef cppPubSocket * socket - - def __cinit__(self): - self.socket = cppPubSocket.create() - if self.socket == NULL: - raise IpcError - - def __dealloc__(self): - del self.socket - - def connect(self, Context context, string endpoint, size_t segment_size=0): - r = self.socket.connect(context.context, endpoint, True, segment_size) - - if r != 0: - if errno.errno == errno.EADDRINUSE: - raise MultiplePublishersError(endpoint) - else: - raise IpcError(endpoint) - - def send(self, bytes data): - length = len(data) - r = self.socket.send(data, length) - - if r != length: - if errno.errno == errno.EADDRINUSE: - raise MultiplePublishersError - else: - raise IpcError - - def all_readers_updated(self): - return self.socket.all_readers_updated() - - def wait_for_readers(self, double timeout=1.0, double interval=0.001): - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if self.all_readers_updated(): - return - time.sleep(interval) - raise TimeoutError("subscriber did not connect") diff --git a/msgq/visionipc/visionipc.pxd b/msgq/visionipc/visionipc.pxd deleted file mode 100644 index 602d0dc94..000000000 --- a/msgq/visionipc/visionipc.pxd +++ /dev/null @@ -1,53 +0,0 @@ -# distutils: language = c++ -#cython: language_level=3 - -from libcpp.string cimport string -from libcpp.vector cimport vector -from libcpp.set cimport set -from libc.stdint cimport uint32_t, uint64_t -from libcpp cimport bool, int - -cdef extern from "msgq/visionipc/visionbuf.h": - cdef enum VisionStreamType: - pass - - cdef cppclass VisionBuf: - void * addr - int fd - size_t len - size_t width - size_t height - size_t stride - size_t uv_offset - size_t idx - void set_frame_id(uint64_t id) - uint64_t get_frame_id() - -cdef extern from "msgq/visionipc/visionipc.h": - struct VisionIpcBufExtra: - uint32_t frame_id - uint64_t timestamp_sof - uint64_t timestamp_eof - bool valid - -cdef extern from "msgq/visionipc/visionipc_server.h": - string get_endpoint_name(string, VisionStreamType) - - cdef cppclass VisionIpcServer: - VisionIpcServer(string) - void create_buffers(VisionStreamType, size_t, size_t, size_t) - void create_buffers_with_sizes(VisionStreamType, size_t, size_t, size_t, size_t, size_t, size_t) - VisionBuf * get_buffer(VisionStreamType) - void send(VisionBuf *, VisionIpcBufExtra *, bool) - void start_listener() - -cdef extern from "msgq/visionipc/visionipc_client.h": - cdef cppclass VisionIpcClient: - int num_buffers - VisionBuf buffers[1] - VisionIpcClient(string, VisionStreamType, bool) - VisionBuf * recv(VisionIpcBufExtra *, int) nogil - bool connect(bool) - bool is_connected() - @staticmethod - set[VisionStreamType] getAvailableStreams(string, bool) diff --git a/msgq/visionipc/visionipc_ctypes.cc b/msgq/visionipc/visionipc_ctypes.cc new file mode 100644 index 000000000..51e928de3 --- /dev/null +++ b/msgq/visionipc/visionipc_ctypes.cc @@ -0,0 +1,71 @@ +#include "msgq/visionipc/visionipc_ctypes.h" + +#include +#include +#include + +#include "msgq/visionipc/visionipc_client.h" +#include "msgq/visionipc/visionipc_server.h" + +struct CffiVisionClient { + VisionIpcClient client; + VisionIpcBufExtra extra = {}; + CffiVisionClient(const char *name, VisionStreamType stream, bool conflate) : client(name, stream, conflate) {} +}; + +static VisionStreamType stream_type(int stream) { return static_cast(stream); } + +extern "C" size_t vipc_get_endpoint_name(const char *name, int stream, char *output, size_t capacity) { + const std::string endpoint = get_endpoint_name(name, stream_type(stream)); + if (capacity) memcpy(output, endpoint.data(), std::min(capacity, endpoint.size())); + return endpoint.size(); +} +extern "C" void *vipc_server_create(const char *name) { return new VisionIpcServer(name); } +extern "C" void vipc_server_delete(void *server) { delete static_cast(server); } +extern "C" void vipc_server_create_buffers(void *server, int stream, size_t count, size_t width, size_t height) { + static_cast(server)->create_buffers(stream_type(stream), count, width, height); +} +extern "C" void vipc_server_create_buffers_with_sizes(void *server, int stream, size_t count, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset) { + static_cast(server)->create_buffers_with_sizes(stream_type(stream), count, width, height, size, stride, uv_offset); +} +extern "C" int vipc_server_send(void *server, int stream, const unsigned char *data, size_t size, uint32_t frame_id, uint64_t timestamp_sof, uint64_t timestamp_eof) { + VisionIpcServer *s = static_cast(server); + VisionBuf *buffer = s->get_buffer(stream_type(stream)); + if (buffer->len != size) return -1; + memcpy(buffer->addr, data, size); + buffer->set_frame_id(frame_id); + VisionIpcBufExtra extra = {frame_id, timestamp_sof, timestamp_eof, false}; + s->send(buffer, &extra, false); + return 0; +} +extern "C" void vipc_server_start_listener(void *server) { static_cast(server)->start_listener(); } + +extern "C" void *vipc_client_create(const char *name, int stream, int conflate) { return new CffiVisionClient(name, stream_type(stream), conflate); } +extern "C" void vipc_client_delete(void *client) { delete static_cast(client); } +extern "C" int vipc_client_connect(void *client, int blocking) { return static_cast(client)->client.connect(blocking); } +extern "C" int vipc_client_is_connected(void *client) { return static_cast(client)->client.is_connected(); } +extern "C" void *vipc_client_recv(void *client, int timeout_ms) { + CffiVisionClient *c = static_cast(client); + return c->client.recv(&c->extra, timeout_ms); +} +extern "C" uint64_t vipc_client_available_streams(const char *name, int blocking) { + uint64_t streams = 0; + for (const auto stream : VisionIpcClient::getAvailableStreams(name, blocking)) streams |= UINT64_C(1) << stream; + return streams; +} +extern "C" size_t vipc_client_num_buffers(void *client) { return static_cast(client)->client.num_buffers; } +extern "C" void *vipc_client_buffer(void *client, size_t index) { return &static_cast(client)->client.buffers[index]; } +extern "C" uint32_t vipc_client_frame_id(void *client) { return static_cast(client)->extra.frame_id; } +extern "C" uint64_t vipc_client_timestamp_sof(void *client) { return static_cast(client)->extra.timestamp_sof; } +extern "C" uint64_t vipc_client_timestamp_eof(void *client) { return static_cast(client)->extra.timestamp_eof; } +extern "C" int vipc_client_valid(void *client) { return static_cast(client)->extra.valid; } + +extern "C" void *vipc_buffer_data(void *buffer) { return static_cast(buffer)->addr; } +extern "C" size_t vipc_buffer_len(void *buffer) { return static_cast(buffer)->len; } +extern "C" size_t vipc_buffer_width(void *buffer) { return static_cast(buffer)->width; } +extern "C" size_t vipc_buffer_height(void *buffer) { return static_cast(buffer)->height; } +extern "C" size_t vipc_buffer_stride(void *buffer) { return static_cast(buffer)->stride; } +extern "C" size_t vipc_buffer_uv_offset(void *buffer) { return static_cast(buffer)->uv_offset; } +extern "C" size_t vipc_buffer_idx(void *buffer) { return static_cast(buffer)->idx; } +extern "C" int vipc_buffer_fd(void *buffer) { return static_cast(buffer)->fd; } +extern "C" uint64_t vipc_buffer_frame_id(void *buffer) { return static_cast(buffer)->get_frame_id(); } diff --git a/msgq/visionipc/visionipc_ctypes.h b/msgq/visionipc/visionipc_ctypes.h new file mode 100644 index 000000000..a68bbc519 --- /dev/null +++ b/msgq/visionipc/visionipc_ctypes.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +size_t vipc_get_endpoint_name(const char *name, int stream, char *output, size_t capacity); +void *vipc_server_create(const char *name); +void vipc_server_delete(void *server); +void vipc_server_create_buffers(void *server, int stream, size_t count, size_t width, size_t height); +void vipc_server_create_buffers_with_sizes(void *server, int stream, size_t count, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset); +int vipc_server_send(void *server, int stream, const unsigned char *data, size_t size, uint32_t frame_id, uint64_t timestamp_sof, uint64_t timestamp_eof); +void vipc_server_start_listener(void *server); + +void *vipc_client_create(const char *name, int stream, int conflate); +void vipc_client_delete(void *client); +int vipc_client_connect(void *client, int blocking); +int vipc_client_is_connected(void *client); +void *vipc_client_recv(void *client, int timeout_ms); +uint64_t vipc_client_available_streams(const char *name, int blocking); +size_t vipc_client_num_buffers(void *client); +void *vipc_client_buffer(void *client, size_t index); +uint32_t vipc_client_frame_id(void *client); +uint64_t vipc_client_timestamp_sof(void *client); +uint64_t vipc_client_timestamp_eof(void *client); +int vipc_client_valid(void *client); + +void *vipc_buffer_data(void *buffer); +size_t vipc_buffer_len(void *buffer); +size_t vipc_buffer_width(void *buffer); +size_t vipc_buffer_height(void *buffer); +size_t vipc_buffer_stride(void *buffer); +size_t vipc_buffer_uv_offset(void *buffer); +size_t vipc_buffer_idx(void *buffer); +int vipc_buffer_fd(void *buffer); +uint64_t vipc_buffer_frame_id(void *buffer); + +#ifdef __cplusplus +} +#endif diff --git a/msgq/visionipc/visionipc_pyx.pxd b/msgq/visionipc/visionipc_pyx.pxd deleted file mode 100644 index 6e2d5ed05..000000000 --- a/msgq/visionipc/visionipc_pyx.pxd +++ /dev/null @@ -1,10 +0,0 @@ -# distutils: language = c++ -#cython: language_level=3 - -from .visionipc cimport VisionBuf as cppVisionBuf - -cdef class VisionBuf: - cdef cppVisionBuf * buf - - @staticmethod - cdef create(cppVisionBuf*) diff --git a/msgq/visionipc/visionipc_pyx.py b/msgq/visionipc/visionipc_pyx.py new file mode 100644 index 000000000..85349969e --- /dev/null +++ b/msgq/visionipc/visionipc_pyx.py @@ -0,0 +1,142 @@ +import ctypes +import weakref +from enum import IntEnum +from pathlib import Path + + +lib = ctypes.CDLL(str(Path(__file__).with_name("libvisionipc_ctypes.so"))) +c_void_p = ctypes.c_void_p +c_size_t = ctypes.c_size_t +c_int = ctypes.c_int +c_char_p = ctypes.c_char_p +c_uint64 = ctypes.c_uint64 + +lib.vipc_get_endpoint_name.argtypes = (c_char_p, c_int, c_void_p, c_size_t) +lib.vipc_get_endpoint_name.restype = c_size_t +lib.vipc_server_create.argtypes = (c_char_p,) +lib.vipc_server_create.restype = c_void_p +lib.vipc_server_delete.argtypes = (c_void_p,) +lib.vipc_server_create_buffers.argtypes = (c_void_p, c_int, c_size_t, c_size_t, c_size_t) +lib.vipc_server_create_buffers_with_sizes.argtypes = (c_void_p, c_int, c_size_t, c_size_t, c_size_t, c_size_t, c_size_t, c_size_t) +lib.vipc_server_send.argtypes = (c_void_p, c_int, c_void_p, c_size_t, ctypes.c_uint32, c_uint64, c_uint64) +lib.vipc_server_start_listener.argtypes = (c_void_p,) +lib.vipc_client_create.argtypes = (c_char_p, c_int, c_int) +lib.vipc_client_create.restype = c_void_p +lib.vipc_client_delete.argtypes = (c_void_p,) +lib.vipc_client_connect.argtypes = (c_void_p, c_int) +lib.vipc_client_is_connected.argtypes = (c_void_p,) +lib.vipc_client_recv.argtypes = (c_void_p, c_int) +lib.vipc_client_recv.restype = c_void_p +lib.vipc_client_available_streams.argtypes = (c_char_p, c_int) +lib.vipc_client_available_streams.restype = c_uint64 +lib.vipc_client_num_buffers.argtypes = (c_void_p,) +lib.vipc_client_num_buffers.restype = c_size_t +lib.vipc_client_buffer.argtypes = (c_void_p, c_size_t) +lib.vipc_client_buffer.restype = c_void_p +for name in ("data",): + getattr(lib, f"vipc_buffer_{name}").argtypes = (c_void_p,) + getattr(lib, f"vipc_buffer_{name}").restype = c_void_p +for name in ("len", "width", "height", "stride", "uv_offset", "idx"): + getattr(lib, f"vipc_buffer_{name}").argtypes = (c_void_p,) + getattr(lib, f"vipc_buffer_{name}").restype = c_size_t +lib.vipc_buffer_fd.argtypes = (c_void_p,) +lib.vipc_buffer_frame_id.argtypes = (c_void_p,) +lib.vipc_buffer_frame_id.restype = c_uint64 +for name in ("frame_id", "timestamp_sof", "timestamp_eof", "valid"): + getattr(lib, f"vipc_client_{name}").argtypes = (c_void_p,) + + +def _bytes(value): + return value.encode() if isinstance(value, str) else value + + +class VisionStreamType(IntEnum): + VISION_STREAM_ROAD = 0 + VISION_STREAM_DRIVER = 1 + VISION_STREAM_WIDE_ROAD = 2 + VISION_STREAM_MAP = 3 + + +def get_endpoint_name(name, stream): + encoded_name = _bytes(name) + size = lib.vipc_get_endpoint_name(encoded_name, stream, None, 0) + output = ctypes.create_string_buffer(size) + lib.vipc_get_endpoint_name(encoded_name, stream, output, size) + return output.raw.decode() + + +class VisionBuf: + def __init__(self, buffer): self._buffer = buffer + @property + def data(self): + size = lib.vipc_buffer_len(self._buffer) + return memoryview((ctypes.c_ubyte * size).from_address(lib.vipc_buffer_data(self._buffer))) + @property + def width(self): return lib.vipc_buffer_width(self._buffer) + @property + def height(self): return lib.vipc_buffer_height(self._buffer) + @property + def stride(self): return lib.vipc_buffer_stride(self._buffer) + @property + def uv_offset(self): return lib.vipc_buffer_uv_offset(self._buffer) + @property + def idx(self): return lib.vipc_buffer_idx(self._buffer) + @property + def fd(self): return lib.vipc_buffer_fd(self._buffer) + @property + def frame_id(self): return lib.vipc_buffer_frame_id(self._buffer) + + +class VisionIpcServer: + def __init__(self, name): + self._server = lib.vipc_server_create(_bytes(name)) + self._finalizer = weakref.finalize(self, lib.vipc_server_delete, self._server) + def create_buffers(self, stream, num_buffers, width, height): + lib.vipc_server_create_buffers(self._server, stream, num_buffers, width, height) + def create_buffers_with_sizes(self, stream, num_buffers, width, height, size, stride, uv_offset): + lib.vipc_server_create_buffers_with_sizes(self._server, stream, num_buffers, width, height, size, stride, uv_offset) + def send(self, stream, data, frame_id=0, timestamp_sof=0, timestamp_eof=0): + if isinstance(data, bytes): + source = ctypes.c_char_p(data) + else: + source = (ctypes.c_ubyte * len(data)).from_buffer(data) + if lib.vipc_server_send(self._server, stream, source, len(data), frame_id, timestamp_sof, timestamp_eof) != 0: + raise AssertionError("buffer size mismatch") + def start_listener(self): lib.vipc_server_start_listener(self._server) + + +class VisionIpcClient: + def __init__(self, name, stream, conflate): + self._client = lib.vipc_client_create(_bytes(name), stream, conflate) + self._finalizer = weakref.finalize(self, lib.vipc_client_delete, self._client) + def _first_buffer_value(self, function): + return function(lib.vipc_client_buffer(self._client, 0)) if self.num_buffers else None + @property + def width(self): return self._first_buffer_value(lib.vipc_buffer_width) + @property + def height(self): return self._first_buffer_value(lib.vipc_buffer_height) + @property + def stride(self): return self._first_buffer_value(lib.vipc_buffer_stride) + @property + def uv_offset(self): return self._first_buffer_value(lib.vipc_buffer_uv_offset) + @property + def buffer_len(self): return self._first_buffer_value(lib.vipc_buffer_len) + @property + def num_buffers(self): return lib.vipc_client_num_buffers(self._client) + @property + def frame_id(self): return lib.vipc_client_frame_id(self._client) + @property + def timestamp_sof(self): return lib.vipc_client_timestamp_sof(self._client) + @property + def timestamp_eof(self): return lib.vipc_client_timestamp_eof(self._client) + @property + def valid(self): return bool(lib.vipc_client_valid(self._client)) + def recv(self, timeout_ms=100): + buffer = lib.vipc_client_recv(self._client, timeout_ms) + return None if not buffer else VisionBuf(buffer) + def connect(self, blocking): return bool(lib.vipc_client_connect(self._client, blocking)) + def is_connected(self): return bool(lib.vipc_client_is_connected(self._client)) + @staticmethod + def available_streams(name, block): + mask = lib.vipc_client_available_streams(_bytes(name), block) + return {stream.value for stream in VisionStreamType if mask & (1 << stream.value)} diff --git a/msgq/visionipc/visionipc_pyx.pyx b/msgq/visionipc/visionipc_pyx.pyx deleted file mode 100644 index 533762bb7..000000000 --- a/msgq/visionipc/visionipc_pyx.pyx +++ /dev/null @@ -1,170 +0,0 @@ -# distutils: language = c++ -# cython: c_string_encoding=ascii, language_level=3 - -from libc.string cimport memcpy -from libc.stdint cimport uint32_t, uint64_t -from libcpp cimport bool -from libcpp.string cimport string - -from .visionipc cimport VisionIpcServer as cppVisionIpcServer -from .visionipc cimport VisionIpcClient as cppVisionIpcClient -from .visionipc cimport VisionBuf as cppVisionBuf -from .visionipc cimport VisionIpcBufExtra -from .visionipc cimport get_endpoint_name as cpp_get_endpoint_name - - -def get_endpoint_name(string name, VisionStreamType stream): - return cpp_get_endpoint_name(name, stream).decode('utf-8') - - -cpdef enum VisionStreamType: - VISION_STREAM_ROAD - VISION_STREAM_DRIVER - VISION_STREAM_WIDE_ROAD - VISION_STREAM_MAP - - -cdef class VisionBuf: - @staticmethod - cdef create(cppVisionBuf * cbuf): - buf = VisionBuf() - buf.buf = cbuf - return buf - - @property - def data(self): - cdef unsigned char[:] data = self.buf.addr - return memoryview(data) - - @property - def width(self): - return self.buf.width - - @property - def height(self): - return self.buf.height - - @property - def stride(self): - return self.buf.stride - - @property - def uv_offset(self): - return self.buf.uv_offset - - @property - def idx(self): - return self.buf.idx - - @property - def fd(self): - return self.buf.fd - - @property - def frame_id(self): - return self.buf.get_frame_id() - - -cdef class VisionIpcServer: - cdef cppVisionIpcServer * server - - def __init__(self, string name): - self.server = new cppVisionIpcServer(name) - - def create_buffers(self, VisionStreamType tp, size_t num_buffers, size_t width, size_t height): - self.server.create_buffers(tp, num_buffers, width, height) - - def create_buffers_with_sizes(self, VisionStreamType tp, size_t num_buffers, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset): - self.server.create_buffers_with_sizes(tp, num_buffers, width, height, size, stride, uv_offset) - - def send(self, VisionStreamType tp, const unsigned char[:] data, uint32_t frame_id=0, uint64_t timestamp_sof=0, uint64_t timestamp_eof=0): - cdef cppVisionBuf * buf = self.server.get_buffer(tp) - - # Populate buffer - assert buf.len == len(data) - memcpy(buf.addr, &data[0], len(data)) - buf.set_frame_id(frame_id) - - cdef VisionIpcBufExtra extra - extra.frame_id = frame_id - extra.timestamp_sof = timestamp_sof - extra.timestamp_eof = timestamp_eof - extra.valid = False - - self.server.send(buf, &extra, False) - - def start_listener(self): - self.server.start_listener() - - def __dealloc__(self): - del self.server - - -cdef class VisionIpcClient: - cdef cppVisionIpcClient * client - cdef VisionIpcBufExtra extra - - def __cinit__(self, string name, VisionStreamType stream, bool conflate): - self.client = new cppVisionIpcClient(name, stream, conflate) - - def __dealloc__(self): - del self.client - - @property - def width(self): - return self.client.buffers[0].width if self.client.num_buffers else None - - @property - def height(self): - return self.client.buffers[0].height if self.client.num_buffers else None - - @property - def stride(self): - return self.client.buffers[0].stride if self.client.num_buffers else None - - @property - def uv_offset(self): - return self.client.buffers[0].uv_offset if self.client.num_buffers else None - - @property - def buffer_len(self): - return self.client.buffers[0].len if self.client.num_buffers else None - - @property - def num_buffers(self): - return self.client.num_buffers - - @property - def frame_id(self): - return self.extra.frame_id - - @property - def timestamp_sof(self): - return self.extra.timestamp_sof - - @property - def timestamp_eof(self): - return self.extra.timestamp_eof - - @property - def valid(self): - return self.extra.valid - - def recv(self, int timeout_ms=100): - cdef cppVisionBuf * buf - # release the GIL, this can block for timeout_ms - with nogil: - buf = self.client.recv(&self.extra, timeout_ms) - if not buf: - return None - return VisionBuf.create(buf) - - def connect(self, bool blocking): - return self.client.connect(blocking) - - def is_connected(self): - return self.client.is_connected() - - @staticmethod - def available_streams(string name, bool block): - return cppVisionIpcClient.getAvailableStreams(name, block) diff --git a/pyproject.toml b/pyproject.toml index df5aac13f..9520adad7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ dependencies = [] [project.optional-dependencies] dev = [ "setuptools", # for distutils - "Cython", "scons", "ruff", "cppcheck", @@ -37,7 +36,3 @@ target-version="py311" [tool.ty.src] exclude = ["site_scons/"] - -[tool.ty.rules] -# Cython modules are compiled at build time, not available for static analysis -unresolved-import = "ignore" diff --git a/site_scons/site_tools/cython.py b/site_scons/site_tools/cython.py deleted file mode 100644 index c29147553..000000000 --- a/site_scons/site_tools/cython.py +++ /dev/null @@ -1,72 +0,0 @@ -import re -import SCons -from SCons.Action import Action -from SCons.Scanner import Scanner - -pyx_from_import_re = re.compile(r'^from\s+(\S+)\s+cimport', re.M) -pyx_import_re = re.compile(r'^cimport\s+(\S+)', re.M) -cdef_import_re = re.compile(r'^cdef extern from\s+.(\S+).:', re.M) - - -def pyx_scan(node, env, path, arg=None): - contents = node.get_text_contents() - - # from cimport ... - matches = pyx_from_import_re.findall(contents) - # cimport - matches += pyx_import_re.findall(contents) - - # Modules can be either .pxd or .pyx files - files = [m.replace('.', '/') + '.pxd' for m in matches] - files += [m.replace('.', '/') + '.pyx' for m in matches] - - # cdef extern from - files += cdef_import_re.findall(contents) - - # Handle relative imports - cur_dir = str(node.get_dir()) - files = [cur_dir + f if f.startswith('/') else f for f in files] - - # Filter out non-existing files (probably system imports) - files = [f for f in files if env.File(f).exists()] - return env.File(files) - - -pyxscanner = Scanner(function=pyx_scan, skeys=['.pyx', '.pxd'], recursive=True) -cythonAction = Action("$CYTHONCOM") - - -def create_builder(env): - try: - cython = env['BUILDERS']['Cython'] - except KeyError: - cython = SCons.Builder.Builder( - action=cythonAction, - emitter={}, - suffix=cython_suffix_emitter, - single_source=1 - ) - env.Append(SCANNERS=pyxscanner) - env['BUILDERS']['Cython'] = cython - return cython - -def cython_suffix_emitter(env, source): - return "$CYTHONCFILESUFFIX" - -def generate(env): - env["CYTHON"] = "cythonize" - env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE" - env["CYTHONCFILESUFFIX"] = ".cpp" - - c_file, _ = SCons.Tool.createCFileBuilders(env) - - c_file.suffix['.pyx'] = cython_suffix_emitter - c_file.add_action('.pyx', cythonAction) - - c_file.suffix['.py'] = cython_suffix_emitter - c_file.add_action('.py', cythonAction) - - create_builder(env) - -def exists(env): - return True