Repository navigation
fix(realtime): surface terminal startup rejection reasons #399
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,7 @@ | |
|
|
||
| import asyncio | ||
| import inspect | ||
| from contextlib import suppress | ||
| from dataclasses import dataclass | ||
| from itertools import count | ||
| from typing import ( | ||
|
|
@@ -63,6 +64,7 @@ | |
| invoke_async, | ||
| response_payload, | ||
| ) | ||
| from .errors import RealtimeConnectionError | ||
|
|
||
| if TYPE_CHECKING: | ||
| from collections.abc import Callable, Iterator, Mapping | ||
|
|
@@ -89,9 +91,13 @@ class ManagedChannel(Generic[FacadeT]): | |
| class ClientEvents(ClientEventHandler): | ||
| def __init__(self, enqueue: Callable[[ConnectionContext], None]) -> None: | ||
| self.enqueue: Callable[[ConnectionContext], None] = enqueue | ||
| self.started: asyncio.Event = asyncio.Event() | ||
| self.failure: RealtimeConnectionError | None = None | ||
| self.is_terminal: Callable[[], bool] = lambda: False | ||
|
|
||
| @override | ||
| async def on_connected(self, ctx: object) -> None: | ||
| self.started.set() | ||
| client = _native.native_attribute(ctx, "client") | ||
| self.enqueue( | ||
| RealtimeConnectContext(client=client if isinstance(client, str) else None), | ||
|
|
@@ -101,13 +107,27 @@ async def on_connected(self, ctx: object) -> None: | |
| async def on_disconnected(self, ctx: object) -> None: | ||
| code = _native.native_attribute(ctx, "code") | ||
| reason = _native.native_attribute(ctx, "reason") | ||
| if self.is_terminal(): | ||
| self.failure = RealtimeConnectionError( | ||
| reason | ||
| if isinstance(reason, str) and reason | ||
| else "Connection disconnected", | ||
| code=code if isinstance(code, int) else None, | ||
| ) | ||
| self.started.set() | ||
| self.enqueue( | ||
| RealtimeDisconnectContext( | ||
| code=code if isinstance(code, int) else None, | ||
| reason=reason if isinstance(reason, str) else None, | ||
| ), | ||
| ) | ||
|
|
||
| async def wait_ready(self, connection: _native.VolcanoCentrifugeConnection) -> None: | ||
| if not connection.is_connected: | ||
| _ = await self.started.wait() | ||
| if self.failure is not None: | ||
| raise self.failure | ||
|
|
||
| @override | ||
| async def on_error(self, ctx: object) -> None: | ||
| code = _native.native_attribute(ctx, "code") | ||
|
|
@@ -489,21 +509,26 @@ async def _connect_locked(self) -> _native.VolcanoCentrifugeConnection: | |
| _generation, lineage, session = self.client_context.capture_session_binding() | ||
| if session is None: | ||
| raise RuntimeError(NO_ACTIVE_SESSION) | ||
| events = ClientEvents(self.enqueue_connection_callbacks) | ||
| connection = _native.VolcanoCentrifugeConnection( | ||
| self.client_factory( | ||
| self._address(), | ||
| events=ClientEvents(self.enqueue_connection_callbacks), | ||
| events=events, | ||
| token=session.access_token, | ||
| get_token=self._token, | ||
| ) | ||
| ) | ||
| events.is_terminal = lambda: connection.is_disconnected | ||
| self.connection_session_lineage = lineage | ||
| self.connection_access_token = session.access_token | ||
| try: | ||
| await connection.connect() | ||
| async with asyncio.timeout(10): | ||
| await connection.connect() | ||
| await events.wait_ready(connection) | ||
| except BaseException: | ||
| self.connection_session_lineage = None | ||
| self.connection_access_token = None | ||
| await self._discard_failed_connection(connection) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If transport shutdown is slow after startup times out, the caller receives the exception after more than ten seconds: cleanup is awaited outside the timeout scope. Non-blocking: add a delayed-close test to establish the caller-visible bound, since the current deadline test uses immediate cleanup. |
||
| raise | ||
| try: | ||
| current_session = self._session_for_lineage(lineage) | ||
|
|
@@ -518,6 +543,14 @@ async def _connect_locked(self) -> _native.VolcanoCentrifugeConnection: | |
| self.connection = connection | ||
| return connection | ||
|
|
||
| @staticmethod | ||
| async def _discard_failed_connection( | ||
| connection: _native.VolcanoCentrifugeConnection, | ||
| ) -> None: | ||
| # Cleanup must not replace the original connection failure/cancellation. | ||
| with suppress(Exception, asyncio.CancelledError): | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ⚪ P3: Failed-connection cleanup swallows an external cancellation and can leave disconnect half-done
Found by claude-opus-5-5 |
||
| await connection.disconnect() | ||
|
|
||
| async def subscribe(self, channel: ChannelState) -> None: | ||
| # A later stop supersedes this request, including time spent waiting for locks. | ||
| generation = channel.subscribe_generation | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,7 @@ | |
|
|
||
| from volcano_sdk import ( | ||
| RealtimeConnectContext, | ||
| RealtimeConnectionError, | ||
| RealtimeDisconnectContext, | ||
| RealtimeErrorContext, | ||
| Session, | ||
|
|
@@ -32,7 +33,12 @@ | |
|
|
||
| from .client_inspection import InspectedClient | ||
| from .realtime_probes import channel_state, completed_operation, realtime_state | ||
| from .test_realtime import FakeCentrifugeClient, FakeCentrifugeFactory, FakeSubscription | ||
| from .test_realtime import ( | ||
| FakeCentrifugeClient, | ||
| FakeCentrifugeFactory, | ||
| FakeConnectionState, | ||
| FakeSubscription, | ||
| ) | ||
|
|
||
| if TYPE_CHECKING: | ||
| from collections.abc import Awaitable, Callable | ||
|
|
@@ -70,6 +76,7 @@ async def test_failed_connect_clears_credentials_before_retry( | |
| await channel.subscribe() | ||
|
|
||
| assert caught.value is failure | ||
| assert native.calls == ["disconnect"] | ||
| assert realtime_state(client.realtime).connection is None | ||
| assert realtime_state(client.realtime).connection_session_lineage is None | ||
| assert realtime_state(client.realtime).connection_access_token is None | ||
|
|
@@ -84,6 +91,112 @@ async def test_failed_connect_clears_credentials_before_retry( | |
| await client.realtime.disconnect() | ||
|
|
||
|
|
||
| class RejectedConnection(FakeCentrifugeClient): | ||
| @override | ||
| async def connect(self) -> None: | ||
| assert self.events is not None | ||
| await self.events.on_disconnected( | ||
| SimpleNamespace(code=3507, reason="permission denied") | ||
| ) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "cleanup_failure", | ||
| [None, RuntimeError("cleanup failed"), asyncio.CancelledError("cleanup cancelled")], | ||
| ) | ||
| async def test_terminal_connection_failure_preserves_reason_and_stops_transport( | ||
| cleanup_failure: BaseException | None, | ||
| ) -> None: | ||
| native = RejectedConnection() | ||
| native.disconnect_error = cleanup_failure | ||
| client = VolcanoClient( | ||
| anon_key="anon", | ||
| access_token="access", | ||
| _realtime_client_factory=FakeCentrifugeFactory(native), | ||
| ) | ||
| try: | ||
| with pytest.raises( | ||
| RealtimeConnectionError, match="permission denied" | ||
| ) as caught: | ||
| await asyncio.wait_for(client.realtime.channel("room").subscribe(), 0.2) | ||
| assert caught.value.code == 3507 | ||
| assert native.calls == ["disconnect"] | ||
| assert realtime_state(client.realtime).connection is None | ||
| finally: | ||
| await client.realtime.disconnect() | ||
|
|
||
|
|
||
| async def test_unbound_connection_events_do_not_treat_a_disconnect_as_terminal() -> ( | ||
| None | ||
| ): | ||
| events = ClientEvents(lambda _context: None) | ||
| await events.on_disconnected(SimpleNamespace(code=3001, reason="transport closed")) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This handler has no connection binding, so the test exercises the default false predicate rather than the production native-state check. Non-blocking: add a bound connection case that emits a retryable disconnect while connecting, then connects successfully, to protect recovery within the startup deadline. |
||
|
|
||
| assert events.failure is None | ||
| assert not events.started.is_set() | ||
|
|
||
|
|
||
| async def test_connect_waits_for_a_delayed_native_connection( | ||
| monkeypatch: pytest.MonkeyPatch, | ||
| ) -> None: | ||
| native = FakeCentrifugeClient() | ||
| entered = asyncio.Event() | ||
|
|
||
| async def connect() -> None: | ||
| native.state = FakeConnectionState(value="connecting") | ||
| entered.set() | ||
| await completed_operation(None) | ||
|
|
||
| monkeypatch.setattr(native, "connect", connect) | ||
| client = VolcanoClient( | ||
| anon_key="anon", | ||
| access_token="access", | ||
| _realtime_client_factory=FakeCentrifugeFactory(native), | ||
| ) | ||
| connecting = asyncio.create_task(realtime_state(client.realtime).connect()) | ||
| try: | ||
| _ = await asyncio.wait_for(entered.wait(), timeout=0.2) | ||
| assert not connecting.done() | ||
| assert not client.realtime.is_connected | ||
| native.state = FakeConnectionState(value="connected") | ||
| assert native.events is not None | ||
| await native.events.on_connected(SimpleNamespace(client="client-123")) | ||
| _ = await asyncio.wait_for(connecting, timeout=0.2) | ||
| assert client.realtime.is_connected | ||
| finally: | ||
| _ = connecting.cancel() | ||
| _ = await asyncio.gather(connecting, return_exceptions=True) | ||
| await client.realtime.disconnect() | ||
|
|
||
|
|
||
| async def test_connection_startup_times_out_within_its_ten_second_deadline( | ||
| monkeypatch: pytest.MonkeyPatch, | ||
| ) -> None: | ||
| native = FakeCentrifugeClient() | ||
|
|
||
| async def connect() -> None: | ||
| _ = await asyncio.Event().wait() | ||
|
|
||
| monkeypatch.setattr(native, "connect", connect) | ||
| client = VolcanoClient( | ||
| anon_key="anon", | ||
| access_token="access", | ||
| _realtime_client_factory=FakeCentrifugeFactory(native), | ||
| ) | ||
| try: | ||
| # The outer deadline distinguishes the SDK's ten-second timeout from | ||
| # an absent deadline or one extended beyond its documented bound. | ||
| async with asyncio.timeout(10.75): | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ⚪ P3: Hard-coded 10s startup deadline forces a unit test that waits 10 real seconds
Found by claude-opus-5-5 |
||
| with pytest.raises(TimeoutError): | ||
| _ = await realtime_state(client.realtime).connect() | ||
| assert native.calls == ["disconnect"] | ||
| assert realtime_state(client.realtime).connection_session_lineage is None | ||
| assert realtime_state(client.realtime).connection_access_token is None | ||
| assert not client.realtime.is_connected | ||
| finally: | ||
| await client.realtime.disconnect() | ||
|
|
||
|
|
||
| async def test_connection_requires_a_session_before_constructing_transport() -> None: | ||
| native = FakeCentrifugeClient() | ||
| client = VolcanoClient( | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -67,3 +67,17 @@ class ServerError(VolcanoError): | |
|
|
||
| class TransportError(VolcanoError): | ||
| """The request failed before receiving an API response.""" | ||
|
|
||
|
|
||
| class RealtimeConnectionError(RuntimeError): | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ⚪ P3: New public Every other class in Found by claude-opus-5-5 |
||
| """A terminal realtime disconnect before a connection becomes ready.""" | ||
|
|
||
| def __init__(self, message: str, *, code: int | None = None) -> None: | ||
| """Preserve the disconnect reason and optional numeric transport code.""" | ||
| super().__init__(message) | ||
| self._code: int | None = code | ||
|
|
||
| @property | ||
| def code(self) -> int | None: | ||
| """The transport's disconnect code, when available.""" | ||
| return self._code | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
⚪ P3: Readiness wait holds
connection_lockup to 10s, stallingdisconnect()during a retrying first connectBefore this change, if the first native attempt failed with a retryable error (for example, connection refused), native
connect()scheduled a reconnect and returned quickly. That releasedconnection_lock, and the subscription's readiness wait ran outside the lock. Now_connect_lockedholds the lock throughevents.wait_ready()for up to 10 seconds. A concurrentclient.realtime.disconnect(),channel.unsubscribe()orremove_channel(), such as a shutdown handler, now blocks until the deadline passes instead of aborting the startup. A fix would be to letdisconnect()cancel or supersede an in-flight startup, or to wait for readiness without holding the lock.Found by claude-opus-5-5