Skip to content

Commit 2cf1510

Browse files
committed
feat(bigtable): fall back to native when the accelerator fails health checks
fix(bigtable): lock all reads and writes in AcceleratorBreaker Change-Id: I9d1daffbe086788481d6802cec035a3d4d284729
1 parent 475c837 commit 2cf1510

11 files changed

Lines changed: 1448 additions & 17 deletions

File tree

‎packages/google-cloud-bigtable/google/cloud/bigtable/data/_accelerator/_fallback.py‎

Lines changed: 50 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@
2020
stops being dialed at all. A daemon whose subprocess has died mid-flight trips
2121
the breaker immediately — it will never recover.
2222
23+
The same breaker also carries a reversible "degraded" flag, set by the health
24+
monitor when the daemon is up but too starved to answer its own health check;
25+
see :class:`AcceleratorBreaker`.
26+
2327
Any other gRPC error is a real, daemon-served result the native client would
2428
reproduce (the daemon owns retries, so it has already exhausted them), so it is
2529
translated to the corresponding ``google.api_core`` exception and raised without
@@ -57,25 +61,60 @@ class _AcceleratorFallback(Exception):
5761
class AcceleratorBreaker:
5862
"""Tracks accelerator health and decides when to stop using it.
5963
60-
One instance per data target. Thread-safe so the generated sync client can
61-
share a target across threads. Two triggers permanently bypass the
62-
accelerator:
63-
64-
* the first ``UNIMPLEMENTED`` reply (the daemon understands the RPC shape but
65-
has no working sessions), and
66-
* an explicit :meth:`trip` when the daemon subprocess is found dead.
64+
One instance per Table. Thread-safe so the generated sync client can share a
65+
Table across threads. There are two independent reasons to bypass, and they
66+
differ in whether they can be undone:
67+
68+
* **Tripped** — permanent. The first ``UNIMPLEMENTED`` reply (the daemon
69+
understands the RPC shape but has no working sessions), or an explicit
70+
:meth:`trip` when the daemon subprocess is found dead. Neither condition
71+
recovers, so the accelerator is abandoned for the life of the Table.
72+
* **Degraded** — reversible. Set by the health monitor when the daemon stops
73+
answering its own health check promptly (see
74+
``_async/_accelerator_health.py``). A daemon starved of CPU recovers when
75+
the load that starved it goes away, so this must clear again — routing
76+
permanently away from the accelerator on one bad minute would be a far
77+
worse outcome than the slow calls it avoided.
78+
79+
Keeping the two flags separate is what makes that safe: :meth:`set_degraded`
80+
can never resurrect an accelerator that :meth:`trip` has given up on.
6781
"""
6882

6983
def __init__(self):
70-
self._tripped = threading.Event()
84+
self._tripped = False
85+
self._degraded = False
86+
self._lock = threading.Lock()
7187

7288
def bypass(self) -> bool:
73-
"""Whether the accelerator should be skipped entirely from now on."""
74-
return self._tripped.is_set()
89+
"""Whether the accelerator should be skipped for calls made right now."""
90+
with self._lock:
91+
return self._tripped or self._degraded
7592

7693
def trip(self) -> None:
7794
"""Permanently bypass the accelerator (e.g. the daemon process died)."""
78-
self._tripped.set()
95+
with self._lock:
96+
self._tripped = True
97+
98+
@property
99+
def is_tripped(self) -> bool:
100+
"""Whether the accelerator has been permanently abandoned."""
101+
with self._lock:
102+
return self._tripped
103+
104+
@property
105+
def is_degraded(self) -> bool:
106+
"""Whether the accelerator is currently being skipped for poor health."""
107+
with self._lock:
108+
return self._degraded
109+
110+
def set_degraded(self, degraded: bool) -> None:
111+
"""Start or stop bypassing the accelerator for poor health.
112+
113+
Reversible, and orthogonal to :meth:`trip`: clearing it does not undo a
114+
permanent trip.
115+
"""
116+
with self._lock:
117+
self._degraded = degraded
79118

80119

81120
def _grpc_code(exc: BaseException) -> StatusCode | None:
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
"""Wire codec for ``grpc.health.v1.Health/Check`` against the daemon.
16+
17+
The daemon serves the standard gRPC health service. The generated Python stubs
18+
for it live in ``grpcio-health-checking``, a separate distribution this package
19+
does not depend on, and health.proto is far too small to justify adding one: the
20+
request we send is the empty message, and the response is a single varint.
21+
22+
HealthCheckRequest{service: ""} -> b""
23+
HealthCheckResponse{status: S} -> b"\\x08" + varint(S)
24+
b"" -> UNKNOWN (proto3 omits 0)
25+
b"\\x08\\x01" -> SERVING
26+
b"\\x08\\x02" -> NOT_SERVING
27+
28+
So the two functions below are the whole codec. Note that we always probe the
29+
empty service name (""), which is the daemon's overall-process status.
30+
31+
This module is plain sync-only logic, shared verbatim by the async and generated
32+
sync clients; no CrossSync branching is needed.
33+
"""
34+
35+
from __future__ import annotations
36+
37+
import enum
38+
39+
#: Full method name of the health check RPC, as registered by the daemon.
40+
HEALTH_CHECK_METHOD = "/grpc.health.v1.Health/Check"
41+
42+
#: Tag byte for ``HealthCheckResponse.status``: field 1, wire type 0 (varint).
43+
_STATUS_TAG = 0x08
44+
45+
46+
class ServingStatus(enum.IntEnum):
47+
"""Mirror of ``grpc.health.v1.HealthCheckResponse.ServingStatus``."""
48+
49+
UNKNOWN = 0
50+
SERVING = 1
51+
NOT_SERVING = 2
52+
SERVICE_UNKNOWN = 3
53+
54+
55+
def serialize_health_request(_request: object = None) -> bytes:
56+
"""Serialize ``HealthCheckRequest{service: ""}``.
57+
58+
Every field is at its proto3 default, so the encoding is empty. Takes and
59+
ignores an argument because gRPC calls it with the request object.
60+
"""
61+
return b""
62+
63+
64+
def parse_health_response(payload: bytes) -> ServingStatus:
65+
"""Decode a ``HealthCheckResponse`` into its status.
66+
67+
Anything that isn't a recognizable status field decodes to ``UNKNOWN``,
68+
which callers treat as a failed probe. That is the right default: the only
69+
field the message has is ``status`` at field 1, and proto3 serializes fields
70+
in ascending field-number order, so a payload that doesn't start with tag 1
71+
is not a response we understand.
72+
"""
73+
if not payload or payload[0] != _STATUS_TAG:
74+
return ServingStatus.UNKNOWN
75+
value = 0
76+
shift = 0
77+
for byte in payload[1:]:
78+
value |= (byte & 0x7F) << shift
79+
if not byte & 0x80:
80+
break
81+
shift += 7
82+
else:
83+
# Ran off the end of a continued varint: truncated message.
84+
return ServingStatus.UNKNOWN
85+
try:
86+
return ServingStatus(value)
87+
except ValueError:
88+
# A status this client predates. Not SERVING as far as we know.
89+
return ServingStatus.UNKNOWN

‎packages/google-cloud-bigtable/google/cloud/bigtable/data/_async/_accelerator_client.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@
2121

2222
from __future__ import annotations
2323

24+
from google.cloud.bigtable.data._accelerator._health import (
25+
HEALTH_CHECK_METHOD,
26+
ServingStatus,
27+
parse_health_response,
28+
serialize_health_request,
29+
)
2430
from google.cloud.bigtable.data._cross_sync import CrossSync
2531
from google.cloud.bigtable_v2.types import (
2632
MutateRowRequest,
@@ -63,6 +69,13 @@ def __init__(self, uds_path: str, auth_secret: str):
6369
request_serializer=ReadRowsRequest.serialize,
6470
response_deserializer=ReadRowsResponse.deserialize,
6571
)
72+
# Hand-rolled codec instead of the generated health stubs, which live in
73+
# a distribution we don't depend on. See ``_accelerator/_health.py``.
74+
self._health_stub = self._channel.unary_unary(
75+
HEALTH_CHECK_METHOD,
76+
request_serializer=serialize_health_request,
77+
response_deserializer=parse_health_response,
78+
)
6679

6780
@property
6881
def uds_path(self) -> str:
@@ -89,6 +102,17 @@ async def read_rows(
89102
"""
90103
return self._read_rows_stub(request, timeout=timeout, metadata=self._metadata)
91104

105+
@CrossSync.convert
106+
async def check_health(self, *, timeout: float | None = None) -> ServingStatus:
107+
"""Probe the daemon's overall serving status.
108+
109+
The daemon answers this out of its own process without touching a
110+
session, the Channel, or the network, so how long it takes is almost
111+
purely a measure of how contended that process is. That makes the
112+
``timeout`` the real signal here, more than the returned status.
113+
"""
114+
return await self._health_stub(None, timeout=timeout, metadata=self._metadata)
115+
92116
@CrossSync.convert
93117
async def close(self) -> None:
94118
if CrossSync.is_async:

0 commit comments

Comments
 (0)