diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 1b77f506..6538ca91 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.7.0" + ".": "0.8.0" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 645b1025..e42497ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.8.0](https://github.com/tembo/sdk-python/compare/v0.7.0...v0.8.0) (2026-11-03) + + +### Features + +* **api:** add 403 error response on agents.runs.create (+1 more change) ([351f019](https://github.com/tembo/sdk-python/commit/351f019cefa6e229c4b3d1c94005a0fe58d99512)) + ## [0.7.0](https://github.com/tembo/sdk-python/compare/v0.6.1...v0.7.0) (2026-10-29) diff --git a/README.md b/README.md index 29f60b7a..dea0c3f1 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ Generated clients support request timeouts and retry temporary failures such as - Set the `TEMBO_LOG` environment variable to `info` or `debug` to enable HTTP logging. - Logs are emitted through the standard `logging` module under the `tembo` logger. +- Importing the SDK wraps the process's log record factory: records from the SDK, `httpx`, `httpcore`, `hpack` and `websockets` loggers have credential headers, credential query parameters and URL userinfo logged as `***`. A record factory installed later that does not chain to the previous one turns this off.
diff --git a/api.md b/api.md index 204cba9d..5764147e 100644 --- a/api.md +++ b/api.md @@ -326,7 +326,7 @@ Create an organization skill with its files. skill = client.skills.create( description="", name="x", - files=[{"content": "", "filename": "x"}], + files=[{"content": "", "encoding": "utf8", "filename": "x"}], ) ``` @@ -1032,7 +1032,7 @@ Enable or disable an organization model, or select its inference provider. ```python model = client.models.update( - model_name="claude-opus-5-5", + model_name="modelName", ) ``` @@ -2439,7 +2439,7 @@ integration = client.integrations.retrieve( ### Update an integration -Update integration settings such as the default repository, Slack channel restrictions, automatic solving, or Sentry environments; replace AWS or PostgreSQL configuration; set or remove Atlassian service-account credentials. +Update integration settings such as the default repository, Slack channel restrictions, automatic solving, or Sentry environments; replace AWS or PostgreSQL configuration; rotate GitLab service account access tokens; set or remove Atlassian service-account credentials. | Direction | Type | | --- | --- | diff --git a/pyproject.toml b/pyproject.toml index 5cedd3db..c2f7c85d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tembo-sdk-python" -version = "0.7.0" # x-release-please-version +version = "0.8.0" # x-release-please-version description = "Manage agents, sessions, repositories, and organization resources with the Tembo API." readme = "README.md" license = "Apache-2.0" @@ -16,6 +16,11 @@ dependencies = [ "sniffio", ] +[project.urls] +Source = "https://github.com/tembo/sdk-python" +Issues = "https://github.com/tembo/sdk-python/issues" +Changelog = "https://github.com/tembo/sdk-python/blob/main/CHANGELOG.md" + [project.optional-dependencies] aiohttp = ["aiohttp", "httpx_aiohttp>=0.1.9"] websockets = ["websockets >= 13, < 16"] diff --git a/scalar-sdk.manifest.json b/scalar-sdk.manifest.json index 8bb4afe8..653bee36 100644 --- a/scalar-sdk.manifest.json +++ b/scalar-sdk.manifest.json @@ -1,7 +1,7 @@ { "name": "TemboSdk", "slug": "temboSdk", - "generatorVersion": "0.36.11", + "generatorVersion": "0.37.1", "servers": [ "https://api.tembo.io/" ], @@ -24974,6 +24974,7 @@ "successStatus": "201", "errorStatuses": [ "400", + "403", "404", "409", "500" @@ -24991,6 +24992,17 @@ } ] }, + { + "status": "403", + "contentType": "application/json", + "encoding": "json", + "contents": [ + { + "contentType": "application/json", + "encoding": "json" + } + ] + }, { "status": "404", "contentType": "application/json", diff --git a/src/tembo/_base_client.py b/src/tembo/_base_client.py index 7a4b7a89..cf3d8d51 100644 --- a/src/tembo/_base_client.py +++ b/src/tembo/_base_client.py @@ -4,6 +4,7 @@ import json import time import uuid +import zlib import email import asyncio import inspect @@ -42,6 +43,7 @@ from . import _exceptions from ._qs import Querystring +from ._decoding import CodingRefused, BoundedDecoder, decoding_error from ._files import to_httpx_files, async_to_httpx_files from ._types import ( Body, @@ -69,6 +71,10 @@ APIResponse, BaseAPIResponse, AsyncAPIResponse, + BinaryAPIResponse, + AsyncBinaryAPIResponse, + StreamedBinaryAPIResponse, + AsyncStreamedBinaryAPIResponse, extract_response_type, ) from ._constants import ( @@ -82,6 +88,7 @@ ) from ._streaming import Stream, SSEDecoder, AsyncStream, SSEBytesDecoder from ._exceptions import ( + APIError, APIStatusError, APITimeoutError, APIConnectionError, @@ -112,10 +119,389 @@ try: from httpx._config import DEFAULT_TIMEOUT_CONFIG as HTTPX_DEFAULT_TIMEOUT except ImportError: - # taken from https://github.com/encode/httpx/blob/3ba5fe0d7ac70222590e759c31442b1cab263791/httpx/_config.py#L366 + # httpx's own default of five seconds, for a version that no longer exposes the constant. HTTPX_DEFAULT_TIMEOUT = Timeout(5.0) +# The redirect statuses the SDK follows, the ones the fetch spec follows. A 300 only offers choices +# and a 304 tells a conditional request its copy is current, so neither is a hop: like a redirect +# without a `Location`, each reaches the caller as a status error. +_REDIRECT_STATUSES: frozenset[int] = frozenset({301, 302, 303, 307, 308}) + +# How many redirects one request follows before giving up with a connection error. +MAX_REDIRECTS = 20 + +# Every header a credential travels in, lower-cased: `Authorization`, `Proxy-Authorization`, `Cookie`, +# the conventional key headers, and each header this SDK's auth schemes place (an `apiKey` scheme +# names its own). A redirect that leaves the origin drops all of them; the log masks the same set. +_CREDENTIAL_HEADERS: frozenset[str] = frozenset( + ("authorization", "proxy-authorization", "api-key", "x-api-key", "cookie", "set-cookie") +) + +# The query parameters this SDK's `apiKey` schemes place a credential in. +_CREDENTIAL_QUERY: tuple[str, ...] = () + +# The headers that describe a request body, dropped with the body when a redirect turns the request +# into a bodiless GET (the fetch spec's list, plus the framing headers httpx sets). +_BODY_HEADERS: tuple[str, ...] = ( + "content-type", + "content-length", + "content-encoding", + "content-language", + "content-location", + "transfer-encoding", +) + +_DEFAULT_PORTS: Dict[str, int] = {"http": 80, "https": 443} + +# The most bytes a buffered response body may decode to (after transparent decompression), so a +# hostile or broken server cannot make the client hold an unbounded body in memory. A binary +# download or a streamed response is handed to the caller unread and is not capped. +MAX_RESPONSE_BODY_BYTES = 64 * 1024 * 1024 + +# The most bytes of an error response's body read for its message and `body`; the rest is dropped. +MAX_ERROR_BODY_BYTES = 64 * 1024 + + +class _ResponseRefused(Exception): + """A response the SDK will not act on, reported to the caller as an SDK error with `message`.""" + + def __init__(self, message: str, *, request: httpx.Request) -> None: + super().__init__(message) + self.message = message + self.request = request + + +class _RedirectRefused(_ResponseRefused): + """A redirect the SDK will not follow; it reaches the caller as an `APIConnectionError`.""" + + +class _BodyRefused(_ResponseRefused): + """A buffered body the SDK will not read; it reaches the caller as an `APIError` and is not retried.""" + + +class _BodyTooLarge(_BodyRefused): + """A buffered body larger than `MAX_RESPONSE_BODY_BYTES`.""" + + +def _origin(url: URL) -> tuple[str, str, int | None]: + """Scheme, host and port of `url`, the default port spelled out, so `https://h:443` is `https://h`.""" + scheme = url.scheme.lower() + return (scheme, url.host.lower(), url.port if url.port is not None else _DEFAULT_PORTS.get(scheme)) + + +def _is_followed_redirect(response: httpx.Response) -> bool: + """Whether `response` is a redirect the SDK follows: a 301, 302, 303, 307 or 308 naming a `Location`.""" + return response.status_code in _REDIRECT_STATUSES and bool(response.headers.get("location", "").strip()) + + +def _is_replayable(stream: object) -> bool: + """Whether a request body can be sent again on the next hop. + + Bytes can, and so can a multipart body whose files are bytes or seekable (httpx rewinds each one + as it renders the part). An iterator or an unseekable file was consumed by the first request, so + resending it would deliver an empty or truncated body the server would act on. + """ + if isinstance(stream, httpx.ByteStream): + return True + fields = getattr(stream, "fields", None) + if fields is None: + return False + for field in cast("list[Any]", fields): + file = getattr(field, "file", None) + if file is None or isinstance(file, (bytes, str)): + continue + seekable = getattr(file, "seekable", None) + try: + if not (callable(seekable) and seekable()): + return False + except Exception: + return False + return True + + +def _redirect_request( + request: httpx.Request, response: httpx.Response, *, sent_headers: httpx.Headers, keep_credentials: bool +) -> tuple[httpx.Request, bool]: + """The request a followed redirect sends next, and whether it still carries the credentials. + + `Location` is resolved against the URL that was redirected (RFC 3986 section 5.2). Only http and + https are followed, and never from https down to http, which would send the request and its + credentials in clear. + + A 303 answering anything but HEAD, and a 301 or 302 answering a POST, is fetched again as a + bodiless GET, as the fetch spec does. Every other redirect keeps its method and resends its body + (a PUT answered with a 301 must not become a GET that succeeds without doing what was asked), and + a body that cannot be replayed is refused rather than resent empty. + + Leaving the origin (scheme, host and port) drops every credential header, whether the client or a + per-call `extra_headers` set it, and the drop holds for the rest of the chain, so a chain that + comes back to the first origin arrives without them. A query-placed key cannot leave at all: the + target's query is the `Location`'s own. On the same origin a relative `Location` cannot carry the + query key, so it is carried over exactly as a header credential is. + + The next hop is built from `sent_headers`, the headers as they stood before httpx ran the auth flow: + an `httpx.Auth` signs the request object in place, so its headers (whatever their names) must not + be copied onto a hop it did not sign. A hop that stays on the origin is signed again on send. + """ + location = response.headers.get("location", "").strip() + try: + target = request.url.join(location) + except Exception as exc: + raise _RedirectRefused("Redirect with an unusable Location header", request=request) from exc + + scheme = target.scheme.lower() + if scheme not in ("http", "https") or not target.host: + raise _RedirectRefused("Redirect to an unsupported URL scheme", request=request) + if request.url.scheme.lower() == "https" and scheme == "http": + raise _RedirectRefused("Redirect from https to insecure http", request=request) + target = target.copy_with(fragment=None) + + headers = httpx.Headers(sent_headers) + method = request.method.upper() + status = response.status_code + stream: Any = request.stream + if (status == 303 and method != "HEAD") or (status in (301, 302) and method == "POST"): + method = "GET" + stream = None + for name in _BODY_HEADERS: + headers.pop(name, None) + elif not _is_replayable(stream): + raise _RedirectRefused("Redirect would resend a request body that cannot be replayed", request=request) + + if keep_credentials and _origin(request.url) == _origin(target): + present = set(target.params.keys()) + carried = [ + (key, value) + for key, value in request.url.params.multi_items() + if key in _CREDENTIAL_QUERY and key not in present + ] + if carried: + target = target.copy_merge_params(tuple(carried)) + _merge_set_cookies(headers, response, target) + else: + keep_credentials = False + for name in [name for name in headers if name.lower() in _CREDENTIAL_HEADERS]: + del headers[name] + + # httpx only fills `Host` in for a request built without a stream, so it is set here for both. + headers["Host"] = target.netloc.decode("ascii") + extensions = dict(request.extensions) + if "_" in target.host: + # work around https://github.com/encode/httpx/discussions/2880, as `_build_request` does + extensions["sni_hostname"] = target.host.replace("_", "-") + else: + extensions.pop("sni_hostname", None) + + return httpx.Request(method, target, headers=headers, stream=stream, extensions=extensions), keep_credentials + + +def _merge_set_cookies(headers: httpx.Headers, response: httpx.Response, target: httpx.URL) -> None: + """Folds the cookies a same-origin redirect sets into the next hop's `Cookie` header. + + httpx writes a request's cookies into a header when the request is built, so without this a cookie + the 3xx itself sets (a session, say) would be missing from the next hop, as it is not when httpx + follows. A cookie the redirect sets replaces the one of the same name the header carried. + """ + if not response.headers.get_list("set-cookie"): + return + fresh = httpx.Cookies() + fresh.extract_cookies(response) + probe = httpx.Request("GET", target) + fresh.set_cookie_header(probe) + added = probe.headers.get("cookie") + if not added: + return + pairs: Dict[str, str] = {} + for part in [*headers.get("cookie", "").split(";"), *added.split(";")]: + part = part.strip() + if part: + pairs[part.partition("=")[0]] = part + headers["Cookie"] = "; ".join(pairs.values()) + + +def _describe_body(value: object) -> str: + """A request body as the debug log shows it: its shape and size, never its values. + + A mapping lists its keys, which are the API's field names; every value, nested ones included, + stays out of the log. + """ + if isinstance(value, (bytes, bytearray)): + return f"<{len(value)} bytes>" + if isinstance(value, str): + return f"<{len(value)} characters>" + if is_mapping(value): + keys = ", ".join(sorted(str(key) for key in value)) + return f"" + if is_list(value) or isinstance(value, tuple): + return f"<{len(cast('list[object]', value))} items>" + return f"<{type(value).__name__}>" + + +def _read_deadline(request: httpx.Request, started: float) -> float | None: + """When reading a buffered body must have finished: the request's read timeout after `started`, when its headers arrived. + + httpx applies its read timeout to each socket read, so a server trickling one byte per read keeps + a body arriving forever; this turns the same number into a deadline for reading the whole body. + """ + timeout = cast("Mapping[str, Any]", request.extensions.get("timeout") or {}) + read = timeout.get("read") + return None if read is None else started + float(read) + + +def _is_unbuffered_cast(cast_to: object) -> bool: + """Whether `cast_to` hands the body to the caller as it arrived: a binary download or a raw `httpx.Response`. + + Those bodies are the caller's to bound, so neither the size cap nor the read deadline applies. + The legacy binary wrapper is matched by name, since its module ships only with the legacy surface. + """ + origin = get_origin(cast_to) or cast_to + if not inspect.isclass(origin): + return False + if issubclass( + origin, + ( + httpx.Response, + BinaryAPIResponse, + AsyncBinaryAPIResponse, + StreamedBinaryAPIResponse, + AsyncStreamedBinaryAPIResponse, + ), + ): + return True + return any(base.__name__ == "HttpxBinaryResponseContent" for base in origin.__mro__) + + +def _body_limit(response: httpx.Response, *, capped: bool) -> tuple[int | None, bool]: + """The byte limit for reading `response`, and whether reaching it truncates rather than refuses. + + An error body is read for its message only, so it is cut at `MAX_ERROR_BODY_BYTES`; a success body + is refused past `MAX_RESPONSE_BODY_BYTES`, unless `capped` is off for a body the caller takes raw. + """ + if not response.is_success: + return MAX_ERROR_BODY_BYTES, True + return (MAX_RESPONSE_BODY_BYTES if capped else None), False + + +def _too_large(response: httpx.Response, limit: int) -> _BodyTooLarge: + """The error for a success body that decoded past `limit` bytes.""" + return _BodyTooLarge( + f"Response body exceeds the {limit} byte limit ({limit // (1024 * 1024)} MiB) for a buffered response", + request=response.request, + ) + + +class _BodyBuffer: + """The decoded bytes of a buffered body read so far, held to the limit `_body_limit` sets.""" + + def __init__(self, response: httpx.Response, *, capped: bool) -> None: + self._response = response + self._limit, self._truncate = _body_limit(response, capped=capped) + self._chunks: List[bytes] = [] + self._size = 0 + + def add(self, pieces: Iterable[bytes]) -> bool: + """Appends `pieces`; True once a truncated body is full and reading should stop. + + A body refused past the limit raises `_BodyTooLarge` as soon as a piece would cross it, so a + decompression bomb stops being inflated there. + """ + for piece in pieces: + if self._limit is not None and self._size + len(piece) > self._limit: + if not self._truncate: + raise _too_large(self._response, self._limit) + self._chunks.append(piece[: self._limit - self._size]) + return True + self._size += len(piece) + self._chunks.append(piece) + return False + + @property + def truncates(self) -> bool: + """Whether this is an error body, read for its message: past the limit or the deadline it is cut, not refused.""" + return self._truncate + + def content(self) -> bytes: + """The body read, decoded and joined.""" + return b"".join(self._chunks) + + +def _read_body(response: httpx.Response, *, capped: bool, deadline: float | None) -> None: + """Reads `response`'s body into memory, bounded in size and, when `deadline` is set, in time. + + The bytes are counted after transparent decompression, and gzip and deflate (stacked or not) are + inflated by `BoundedDecoder` in bounded steps, so a small compressed body cannot expand past the + limit. A read still running at `deadline` raises `httpx.ReadTimeout`, which the request loop + retries or reports like any other timeout, except on an error body, which keeps what arrived by + then so its status error still stands; httpx cannot interrupt one blocked socket read, so the + deadline can be overrun by at most one read timeout. + """ + body = _BodyBuffer(response, capped=capped) + try: + decoder = BoundedDecoder.for_response(response, capped=capped) + if decoder is None: + for chunk in response.iter_bytes(): + if body.add((chunk,)): + break + if deadline is not None and time.monotonic() > deadline: + if body.truncates: + break + raise httpx.ReadTimeout("Timed out reading the response body", request=response.request) + else: + for raw in response.iter_raw(): + if body.add(decoder.decode(raw)): + break + if deadline is not None and time.monotonic() > deadline: + if body.truncates: + break + raise httpx.ReadTimeout("Timed out reading the response body", request=response.request) + else: + body.add(decoder.flush()) + except CodingRefused as exc: + raise _BodyRefused(exc.message, request=response.request) from None + except zlib.error as exc: + raise decoding_error(response, exc) from exc + finally: + response.close() + response._content = body.content() # pyright: ignore[reportPrivateUsage] + + +async def _aread_body(response: httpx.Response, *, capped: bool, deadline: float | None) -> None: + """The async twin of `_read_body`, where the deadline cancels a read that is still blocked.""" + body = _BodyBuffer(response, capped=capped) + + async def read() -> None: + """Reads the body into `body` until it ends or `body` is full.""" + decoder = BoundedDecoder.for_response(response, capped=capped) + if decoder is None: + async for chunk in response.aiter_bytes(): + if body.add((chunk,)): + return + return + async for raw in response.aiter_raw(): + if body.add(decoder.decode(raw)): + return + body.add(decoder.flush()) + + try: + if deadline is None: + await read() + else: + try: + with anyio.fail_after(max(0.0, deadline - time.monotonic())): + await read() + except TimeoutError as exc: + if not body.truncates: + raise httpx.ReadTimeout("Timed out reading the response body", request=response.request) from exc + except CodingRefused as exc: + raise _BodyRefused(exc.message, request=response.request) from None + except zlib.error as exc: + raise decoding_error(response, exc) from exc + finally: + await response.aclose() + response._content = body.content() # pyright: ignore[reportPrivateUsage] + + @dataclass class CursorPageConfig: """Per-operation cursor pagination descriptor baked into a page by the generated list method. @@ -238,6 +624,17 @@ def _info_to_options(self, info: PageInfo) -> FinalRequestOptions: return options if not isinstance(info.url, NotGiven): + # The next request carries this client's credentials, so a page URL a response names on + # another scheme, host or port is refused rather than dialed: a server must not be able + # to send the caller's key elsewhere by naming it as the next page. The message names + # neither URL, since either may carry a token. The origin compared is the one that served + # this page: an operation with its own `servers` entry is sent to an absolute URL, else + # to the client's base URL. `_client` is declared by the sync and async page classes, + # each with its own client type. + current = URL(self._options.url) + served_by: URL = current if current.is_absolute_url else cast(Any, self)._client.base_url + if info.url.is_absolute_url and _origin(info.url) != _origin(served_by): + raise RuntimeError("Refusing to follow a next page URL that points away from the API's origin") params = self._params_from_url(info.url) url = info.url.copy_with(params=params) options.params = dict(url.params) @@ -259,6 +656,25 @@ def _info_to_options(self, info: PageInfo) -> FinalRequestOptions: raise ValueError("Unexpected PageInfo state") + def _next_page_repeats(self) -> bool: + """Whether the next page's request would be exactly the request that fetched this page. + + A server that answers with the cursor (or link, or body token) it was just given would make + auto-pagination fetch the same page forever, so the walk stops there instead. + """ + info = self.next_page_info() + if info is None: + return False + try: + options = self._info_to_options(info) + except (TypeError, ValueError): + return False + return ( + options.url == self._options.url + and options.params == self._options.params + and options.json_data == self._options.json_data + ) + class BaseSyncPage(BasePage[_T], Generic[_T]): _client: SyncAPIClient = pydantic.PrivateAttr() @@ -295,7 +711,7 @@ def iter_pages(self: SyncPageT) -> Iterator[SyncPageT]: page = self while True: yield page - if page.has_next_page(): + if page.has_next_page() and not page._next_page_repeats(): page = page.get_next_page() else: return @@ -383,7 +799,7 @@ async def iter_pages(self: AsyncPageT) -> AsyncIterator[AsyncPageT]: page = self while True: yield page - if page.has_next_page(): + if page.has_next_page() and not page._next_page_repeats(): page = await page.get_next_page() else: return @@ -540,13 +956,15 @@ def _prepare_url(self, url: str) -> URL: Merge a URL argument together with any 'base_url' on the client, to create the URL used for the outgoing request. """ - # Copied from httpx's `_merge_url` method. - merge_url = URL(url) - if merge_url.is_relative_url: - merge_raw_path = self.base_url.raw_path + merge_url.raw_path.lstrip(b"/") - return self.base_url.copy_with(raw_path=merge_raw_path) + target = URL(url) + if target.is_absolute_url: + return target - return merge_url + # A relative URL always extends the base path, even when it starts with "/": operation paths + # are written from the API root, which is wherever `base_url` points. Exactly one "/" joins + # the two halves. `raw_path` carries the query string, so a query on `url` is kept. + base_path = self.base_url.raw_path.rstrip(b"/") + return self.base_url.copy_with(raw_path=b"/".join((base_path, target.raw_path.lstrip(b"/")))) def _make_sse_decoder(self) -> SSEDecoder | SSEBytesDecoder: return SSEDecoder() @@ -558,19 +976,19 @@ def _build_request( retries_taken: int = 0, ) -> httpx.Request: if log.isEnabledFor(logging.DEBUG): - log.debug( - "Request options: %s", - model_dump( - options, - exclude_unset=True, - # Pydantic v1 can't dump every type we support in content, so we exclude it for now. - exclude={ - "content", - } - if PYDANTIC_V1 - else {}, - ), + # The request bodies are described, never dumped: a body carries whatever the caller + # sends (passwords, tokens, personal data), and the log redaction only knows credential + # header and query names. The headers and params are dumped, masked by that redaction. + dumped = model_dump( + options, + exclude_unset=True, + exclude={"content", "json_data", "extra_json", "files"}, ) + for name in ("content", "json_data", "extra_json", "files"): + value: object = getattr(options, name, None) + if value is not None and is_given(value): + dumped[name] = _describe_body(value) + log.debug("Request options: %s", dumped) kwargs: dict[str, Any] = {} json_data = options.json_data @@ -751,6 +1169,9 @@ def auth_headers(self) -> dict[str, str]: def default_headers(self) -> dict[str, str | Omit]: return { "Accept": "application/json", + # Only the codings `BoundedDecoder` inflates in bounded steps; httpx would otherwise add br + # and zstd whenever their optional packages are installed, and inflates those unbounded. + "Accept-Encoding": "gzip, deflate", "Content-Type": "application/json", "User-Agent": self.user_agent, **self.platform_headers(), @@ -896,7 +1317,8 @@ class _DefaultHttpxClient(httpx.Client): def __init__(self, **kwargs: Any) -> None: kwargs.setdefault("timeout", DEFAULT_TIMEOUT) kwargs.setdefault("limits", DEFAULT_CONNECTION_LIMITS) - kwargs.setdefault("follow_redirects", True) + # The SDK follows redirects itself (see `_send_following_redirects`), so httpx never does. + kwargs.setdefault("follow_redirects", False) super().__init__(**kwargs) @@ -1018,6 +1440,48 @@ def _prepare_request( """ return None + def _send_following_redirects( + self, + request: httpx.Request, + *, + options: FinalRequestOptions, + auth: httpx.Auth | None, + ) -> httpx.Response: + """Sends `request`, following redirects in the SDK rather than in httpx, and returns the final response unread. + + httpx is told not to follow on every send, so a caller-supplied `http_client` built with + `follow_redirects=True` cannot follow either: httpx strips only `Authorization` on a + cross-origin hop and would resend an API key header, cookies and a 307/308 body to whatever + host the server names. See `_redirect_request` for what each hop keeps. A hop that has left + the origin is sent with a no-op `auth`, so neither `custom_auth`, the caller's client-level + `auth=` nor a `.netrc` entry (httpx < 0.24 reads one by default) signs it. + + `options.follow_redirects=False` turns following off altogether: the 3xx itself reaches the + caller as a status error. Unset or `True`, the SDK follows at most `MAX_REDIRECTS` hops. + """ + keep_credentials = True + hops = 0 + while True: + kwargs: HttpxSendArgs = {"follow_redirects": False} + if not keep_credentials: + kwargs["auth"] = httpx.Auth() + elif auth is not None: + kwargs["auth"] = auth + sent_headers = httpx.Headers(request.headers) + response = self._client.send(request, stream=True, **kwargs) + if options.follow_redirects is False or not _is_followed_redirect(response): + return response + try: + if hops >= MAX_REDIRECTS: + raise _RedirectRefused(f"Maximum redirects ({MAX_REDIRECTS}) exceeded", request=request) + request, keep_credentials = _redirect_request( + request, response, sent_headers=sent_headers, keep_credentials=keep_credentials + ) + finally: + response.close() + hops += 1 + log.debug("Following redirect %i to %s", response.status_code, request.url) + @overload def request( self, @@ -1077,23 +1541,32 @@ def request( request = self._build_request(options, retries_taken=retries_taken) self._prepare_request(request) - kwargs: HttpxSendArgs = {} custom_auth = self._custom_auth(options.security) - if custom_auth is not None: - kwargs["auth"] = custom_auth - - if options.follow_redirects is not None: - kwargs["follow_redirects"] = options.follow_redirects + streamed = stream or self._should_stream_response_body(request=request) log.debug("Sending HTTP Request: %s %s", request.method, request.url) response = None try: - response = self._client.send( - request, - stream=stream or self._should_stream_response_body(request=request), - **kwargs, - ) + response = self._send_following_redirects(request, options=options, auth=custom_auth) + # The body's deadline starts once the headers are in: sending the request and waiting + # for the answer are bounded by httpx's own write and read timeouts. + started = time.monotonic() + if not streamed: + # Only a successful download is the caller's to bound; an error body is read for + # its message, capped and on the deadline like any other. + unbounded = _is_unbuffered_cast(cast_to) and response.is_success + _read_body( + response, + capped=not unbounded, + deadline=None if unbounded else _read_deadline(request, started), + ) + except _RedirectRefused as err: + log.debug("Refusing redirect: %s", err.message) + raise APIConnectionError(message=err.message, request=err.request) from None + except _BodyRefused as err: + log.debug("Refusing response body: %s", err.message) + raise APIError(err.message, err.request, body=None) from None except httpx.TimeoutException as err: log.debug("Encountered httpx.TimeoutException", exc_info=True) @@ -1125,8 +1598,8 @@ def request( log.debug( 'HTTP Response: %s %s "%i %s" %s', - request.method, - request.url, + response.request.method, + response.request.url, response.status_code, response.reason_phrase, response.headers, @@ -1134,7 +1607,7 @@ def request( try: response.raise_for_status() - except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code + except httpx.HTTPStatusError as err: # thrown on 3xx (one not followed), 4xx and 5xx log.debug("Encountered httpx.HTTPStatusError", exc_info=True) if remaining_retries > 0 and self._should_retry(err.response): @@ -1147,10 +1620,13 @@ def request( ) continue - # If the response is streamed then we need to explicitly read the response - # to completion before attempting to access the response text. + # A streamed response is still unread: read the first `MAX_ERROR_BODY_BYTES` of it + # for the message, within the read deadline, never the whole body a hostile server + # could make endless or trickle forever. if not err.response.is_closed: - err.response.read() + _read_body( + err.response, capped=True, deadline=_read_deadline(err.response.request, time.monotonic()) + ) log.debug("Re-raising status error") raise self._make_status_error_from_response(err.response) from None @@ -1465,7 +1941,8 @@ class _DefaultAsyncHttpxClient(httpx.AsyncClient): def __init__(self, **kwargs: Any) -> None: kwargs.setdefault("timeout", DEFAULT_TIMEOUT) kwargs.setdefault("limits", DEFAULT_CONNECTION_LIMITS) - kwargs.setdefault("follow_redirects", True) + # The SDK follows redirects itself (see `_send_following_redirects`), so httpx never does. + kwargs.setdefault("follow_redirects", False) super().__init__(**kwargs) @@ -1482,7 +1959,8 @@ class _DefaultAioHttpClient(httpx_aiohttp.HttpxAiohttpClient): # type: ignore def __init__(self, **kwargs: Any) -> None: kwargs.setdefault("timeout", DEFAULT_TIMEOUT) kwargs.setdefault("limits", DEFAULT_CONNECTION_LIMITS) - kwargs.setdefault("follow_redirects", True) + # The SDK follows redirects itself (see `_send_following_redirects`), so httpx never does. + kwargs.setdefault("follow_redirects", False) super().__init__(**kwargs) @@ -1607,6 +2085,37 @@ async def _prepare_request( """ return None + async def _send_following_redirects( + self, + request: httpx.Request, + *, + options: FinalRequestOptions, + auth: httpx.Auth | None, + ) -> httpx.Response: + """The async twin of `SyncAPIClient._send_following_redirects`, which documents the rules.""" + keep_credentials = True + hops = 0 + while True: + kwargs: HttpxSendArgs = {"follow_redirects": False} + if not keep_credentials: + kwargs["auth"] = httpx.Auth() + elif auth is not None: + kwargs["auth"] = auth + sent_headers = httpx.Headers(request.headers) + response = await self._client.send(request, stream=True, **kwargs) + if options.follow_redirects is False or not _is_followed_redirect(response): + return response + try: + if hops >= MAX_REDIRECTS: + raise _RedirectRefused(f"Maximum redirects ({MAX_REDIRECTS}) exceeded", request=request) + request, keep_credentials = _redirect_request( + request, response, sent_headers=sent_headers, keep_credentials=keep_credentials + ) + finally: + await response.aclose() + hops += 1 + log.debug("Following redirect %i to %s", response.status_code, request.url) + @overload async def request( self, @@ -1671,23 +2180,32 @@ async def request( request = self._build_request(options, retries_taken=retries_taken) await self._prepare_request(request) - kwargs: HttpxSendArgs = {} custom_auth = self._custom_auth(options.security) - if custom_auth is not None: - kwargs["auth"] = custom_auth - - if options.follow_redirects is not None: - kwargs["follow_redirects"] = options.follow_redirects + streamed = stream or self._should_stream_response_body(request=request) log.debug("Sending HTTP Request: %s %s", request.method, request.url) response = None try: - response = await self._client.send( - request, - stream=stream or self._should_stream_response_body(request=request), - **kwargs, - ) + response = await self._send_following_redirects(request, options=options, auth=custom_auth) + # The body's deadline starts once the headers are in: sending the request and waiting + # for the answer are bounded by httpx's own write and read timeouts. + started = time.monotonic() + if not streamed: + # Only a successful download is the caller's to bound; an error body is read for + # its message, capped and on the deadline like any other. + unbounded = _is_unbuffered_cast(cast_to) and response.is_success + await _aread_body( + response, + capped=not unbounded, + deadline=None if unbounded else _read_deadline(request, started), + ) + except _RedirectRefused as err: + log.debug("Refusing redirect: %s", err.message) + raise APIConnectionError(message=err.message, request=err.request) from None + except _BodyRefused as err: + log.debug("Refusing response body: %s", err.message) + raise APIError(err.message, err.request, body=None) from None except httpx.TimeoutException as err: log.debug("Encountered httpx.TimeoutException", exc_info=True) @@ -1719,8 +2237,8 @@ async def request( log.debug( 'HTTP Response: %s %s "%i %s" %s', - request.method, - request.url, + response.request.method, + response.request.url, response.status_code, response.reason_phrase, response.headers, @@ -1728,7 +2246,7 @@ async def request( try: response.raise_for_status() - except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code + except httpx.HTTPStatusError as err: # thrown on 3xx (one not followed), 4xx and 5xx log.debug("Encountered httpx.HTTPStatusError", exc_info=True) if remaining_retries > 0 and self._should_retry(err.response): @@ -1741,10 +2259,13 @@ async def request( ) continue - # If the response is streamed then we need to explicitly read the response - # to completion before attempting to access the response text. + # A streamed response is still unread: read the first `MAX_ERROR_BODY_BYTES` of it + # for the message, within the read deadline, never the whole body a hostile server + # could make endless or trickle forever. if not err.response.is_closed: - await err.response.aread() + await _aread_body( + err.response, capped=True, deadline=_read_deadline(err.response.request, time.monotonic()) + ) log.debug("Re-raising status error") raise self._make_status_error_from_response(err.response) from None diff --git a/src/tembo/_client.py b/src/tembo/_client.py index 501bfa3b..fd379bbf 100644 --- a/src/tembo/_client.py +++ b/src/tembo/_client.py @@ -104,7 +104,8 @@ def __init__( default_headers: Mapping[str, str] | None = None, default_query: Mapping[str, object] | None = None, # Configure a custom httpx client. - # We provide a `DefaultHttpxClient` class that you can pass to retain the default values we use for `limits`, `timeout` & `follow_redirects`. + # We provide a `DefaultHttpxClient` class that you can pass to retain the default values we use for `limits` & `timeout`. + # The SDK follows redirects itself, whatever the client's `follow_redirects` says. # See the [httpx documentation](https://www.python-httpx.org/api/#client) for more details. http_client: httpx.Client | None = None, # Enable or disable schema validation for data returned by the API. @@ -421,7 +422,8 @@ def __init__( default_headers: Mapping[str, str] | None = None, default_query: Mapping[str, object] | None = None, # Configure a custom httpx client. - # We provide a `DefaultAsyncHttpxClient` class that you can pass to retain the default values we use for `limits`, `timeout` & `follow_redirects`. + # We provide a `DefaultAsyncHttpxClient` class that you can pass to retain the default values we use for `limits` & `timeout`. + # The SDK follows redirects itself, whatever the client's `follow_redirects` says. # See the [httpx documentation](https://www.python-httpx.org/api/#asyncclient) for more details. http_client: httpx.AsyncClient | None = None, # Enable or disable schema validation for data returned by the API. diff --git a/src/tembo/_compat.py b/src/tembo/_compat.py index e6690a4f..fb3d7634 100644 --- a/src/tembo/_compat.py +++ b/src/tembo/_compat.py @@ -1,15 +1,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Union, Generic, TypeVar, Callable, cast, overload +from typing import TYPE_CHECKING, Any, Union, TypeVar, cast from datetime import date, datetime -from typing_extensions import Self, Literal, TypedDict +from typing_extensions import Literal, TypedDict import pydantic from pydantic.fields import FieldInfo from ._types import IncEx, StrBytesIntFloat -_T = TypeVar("_T") _ModelT = TypeVar("_ModelT", bound=pydantic.BaseModel) # --------------- Pydantic v2, v3 compatibility --------------- @@ -192,35 +191,5 @@ class GenericModel(pydantic.BaseModel): ... # cached properties if TYPE_CHECKING: cached_property = property - - # we define a separate type (copied from typeshed) - # that represents that `cached_property` is `set`able - # at runtime, which differs from `@property`. - # - # this is a separate type as editors likely special case - # `@property` and we don't want to cause issues just to have - # more helpful internal types. - - class typed_cached_property(Generic[_T]): - func: Callable[[Any], _T] - attrname: str | None - - def __init__(self, func: Callable[[Any], _T]) -> None: ... - - @overload - def __get__(self, instance: None, owner: type[Any] | None = None) -> Self: ... - - @overload - def __get__(self, instance: object, owner: type[Any] | None = None) -> _T: ... - - def __get__(self, instance: object, owner: type[Any] | None = None) -> _T | Self: - raise NotImplementedError() - - def __set_name__(self, owner: type[Any], name: str) -> None: ... - - # __set__ is not defined at runtime, but @cached_property is designed to be settable - def __set__(self, instance: object, value: _T) -> None: ... else: from functools import cached_property as cached_property - - typed_cached_property = cached_property diff --git a/src/tembo/_decoding.py b/src/tembo/_decoding.py new file mode 100644 index 00000000..2b38a5ca --- /dev/null +++ b/src/tembo/_decoding.py @@ -0,0 +1,187 @@ +"""Bounded decoding of a response's content codings. + +httpx inflates a whole network read in one call, so with stacked codings (`gzip, gzip`) a few KiB read +can expand to gigabytes before any size limit sees a byte. Buffered bodies and streamed ones (SSE, +JSON Lines) read their bytes through `BoundedDecoder` instead, which inflates gzip and deflate in +steps of at most `INFLATE_STEP` bytes, so every reader can stop at its own limit after each step. +""" + +from __future__ import annotations + +import zlib +from typing import Dict, List, Mapping, Iterator, AsyncIterator, cast + +import httpx + +from ._exceptions import APIError + +# The most bytes one inflate step produces. +INFLATE_STEP = 64 * 1024 + +# The content codings `BoundedDecoder` undoes itself, with the zlib window bits that read each one. +ZLIB_CODINGS: Dict[str, int] = { + "gzip": zlib.MAX_WBITS | 16, + "x-gzip": zlib.MAX_WBITS | 16, + "deflate": zlib.MAX_WBITS, +} + +# Codings only httpx can undo, each through an optional package and without bounding the output, so a +# body that uses one is refused. httpx skips one whose package is missing, and so does `BoundedDecoder`. +HTTPX_ONLY_CODINGS = frozenset({"br", "zstd"}) + + +class CodingRefused(Exception): + """A success body coded br or zstd, which no limit could bound; the caller reports it as an SDK error.""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.message = message + + +class Inflater: + """One gzip or deflate layer, inflated at most `INFLATE_STEP` bytes per step. + + A deflate body should be zlib-wrapped, but some servers send it raw; as httpx does, a stream whose + first bytes are not a zlib header is read again as raw deflate. + """ + + def __init__(self, wbits: int) -> None: + self._wbits = wbits + self._decompressor = zlib.decompressobj(wbits) + self._started = False + + def feed(self, data: bytes) -> Iterator[bytes]: + """Inflates `data`, yielding the output in pieces of at most `INFLATE_STEP` bytes.""" + while not self._decompressor.eof: + try: + out = self._decompressor.decompress(data, INFLATE_STEP) + except zlib.error: + if self._started or self._wbits != zlib.MAX_WBITS: + raise + self._wbits = -zlib.MAX_WBITS + self._decompressor = zlib.decompressobj(self._wbits) + continue + self._started = True + # Input past the step stays in `unconsumed_tail`; an empty step with nothing left means + # zlib needs more input. + data = self._decompressor.unconsumed_tail + if out: + yield out + elif not data: + break + + def flush(self) -> Iterator[bytes]: + """Whatever the layer still holds once the body has ended; `feed` has drained it to a few bytes.""" + out = self._decompressor.flush() + if out: + yield out + + +def httpx_decoders() -> Mapping[str, object]: + """The content codings the installed httpx decodes; br and zstd appear only with their optional packages.""" + try: + from httpx._decoders import SUPPORTED_DECODERS # pyright: ignore[reportPrivateUsage] + except ImportError: + return {} + return cast("Mapping[str, object]", SUPPORTED_DECODERS) + + +class BoundedDecoder: + """Undoes a response's gzip and deflate content codings, stacked ones included, in bounded steps. + + Each layer hands the next a piece of at most `INFLATE_STEP` bytes, so a reader can stop at its + limit after every step instead of after httpx has inflated a whole network read at once. + """ + + def __init__(self, layers: List[Inflater]) -> None: + self._layers = layers + + @classmethod + def for_response(cls, response: httpx.Response, *, capped: bool) -> BoundedDecoder | None: + """The decoder for `response`'s `Content-Encoding`, or None when httpx must decode it. + + httpx decodes a body read uncapped and a body it already holds (a transport may hand back a + response read in full): that one has no raw stream left and is served from memory. Codings + are undone last-applied first. `identity` and codings httpx does not know pass through + unchanged, as they do in httpx. A br or zstd coding httpx would decode raises `CodingRefused` + on a success; an error body with one is read raw. + """ + if hasattr(response, "_content") or not capped: + # An uncapped body (a successful binary download) is the caller's to bound, so httpx + # decodes it as it would. + return None + codings = [coding.strip().lower() for coding in response.headers.get("content-encoding", "").split(",")] + if any(coding in HTTPX_ONLY_CODINGS and coding in httpx_decoders() for coding in codings): + # httpx inflates br and zstd a whole network read per call, which no limit can stop, and the + # SDK never asks for them (`Accept-Encoding: gzip, deflate`). + if not response.is_success: + # An error body is read for its message only: keep its raw bytes, so the status error + # (and its retry) still stands. + return cls([]) + raise CodingRefused( + "A br or zstd content coding is not decoded for a buffered or streamed response; " + "the SDK accepts gzip and deflate" + ) + return cls([Inflater(ZLIB_CODINGS[coding]) for coding in reversed(codings) if coding in ZLIB_CODINGS]) + + def decode(self, data: bytes) -> Iterator[bytes]: + """Decodes one network read, yielding the body in bounded pieces.""" + return self._push(0, data) + + def flush(self) -> Iterator[bytes]: + """Decodes what the layers still hold once the body has ended.""" + for index, layer in enumerate(self._layers): + for piece in layer.flush(): + yield from self._push(index + 1, piece) + + def _push(self, index: int, data: bytes) -> Iterator[bytes]: + """Runs `data` through the layers from `index` on.""" + if index == len(self._layers): + yield data + return + for piece in self._layers[index].feed(data): + yield from self._push(index + 1, piece) + + +def decoding_error(response: httpx.Response, exc: zlib.error) -> httpx.DecodingError: + """The error for a body that is not valid for its `Content-Encoding`, as httpx raises it.""" + return httpx.DecodingError(f"Error decoding the response body: {exc}", request=response.request) + + +def _stream_decoder(response: httpx.Response) -> BoundedDecoder | None: + """The bounded decoder for a streamed body, reporting a br or zstd one as an `APIError`.""" + try: + return BoundedDecoder.for_response(response, capped=True) + except CodingRefused as exc: + raise APIError(exc.message, response.request, body=None) from None + + +def iter_decoded(response: httpx.Response) -> Iterator[bytes]: + """A streamed body's bytes, decoded in bounded pieces as they arrive (SSE and JSON Lines read this).""" + decoder = _stream_decoder(response) + if decoder is None: + yield from response.iter_bytes() + return + try: + for raw in response.iter_raw(): + yield from decoder.decode(raw) + yield from decoder.flush() + except zlib.error as exc: + raise decoding_error(response, exc) from exc + + +async def aiter_decoded(response: httpx.Response) -> AsyncIterator[bytes]: + """The async twin of `iter_decoded`.""" + decoder = _stream_decoder(response) + if decoder is None: + async for chunk in response.aiter_bytes(): + yield chunk + return + try: + async for raw in response.aiter_raw(): + for piece in decoder.decode(raw): + yield piece + for piece in decoder.flush(): + yield piece + except zlib.error as exc: + raise decoding_error(response, exc) from exc diff --git a/src/tembo/_models.py b/src/tembo/_models.py index ce728eb7..4ae7a440 100644 --- a/src/tembo/_models.py +++ b/src/tembo/_models.py @@ -237,6 +237,12 @@ def construct( # pyright: ignore[reportIncompatibleMethodOverride] _extra = {} for key, value in values.items(): if key not in model_fields: + if PYDANTIC_V1 and hasattr(__cls, key): + # Pydantic v1 keeps an extra key in the instance `__dict__`, which outranks the + # class: a response key named `json`, `dict` or `to_dict` would replace that method + # on this object with the server's value. Such a key is dropped instead; v2 keeps + # extras in `__pydantic_extra__`, which only answers names the class lacks. + continue parsed = construct_type(value=value, type_=extra_field_type) if extra_field_type is not None else value if PYDANTIC_V1: @@ -251,10 +257,11 @@ def construct( # pyright: ignore[reportIncompatibleMethodOverride] # init_private_attributes() does not exist in v2 m._init_private_attributes() # type: ignore - # copied from Pydantic v1's `construct()` method + # Record which fields were set, under the attribute Pydantic v1 reads them from. object.__setattr__(m, "__fields_set__", _fields_set) else: - # these properties are copied from Pydantic's `model_construct()` method + # Pydantic v2 reads private, extra and set fields from these attributes; a model built + # without validation has to set all three itself. object.__setattr__(m, "__pydantic_private__", None) object.__setattr__(m, "__pydantic_extra__", _extra) object.__setattr__(m, "__pydantic_fields_set__", _fields_set) @@ -291,33 +298,26 @@ def model_dump( fallback: Callable[[Any], Any] | None = None, serialize_as_any: bool = False, ) -> dict[str, Any]: - """Usage docs: https://docs.pydantic.dev/2.4/concepts/serialization/#modelmodel_dump + """Returns the model's fields as a dictionary, the Pydantic v2 `model_dump` call on Pydantic v1. - Generate a dictionary representation of the model, optionally specifying which fields to include or exclude. + See https://docs.pydantic.dev/latest/api/base_model/#pydantic.BaseModel.model_dump for the + full semantics. Args: - mode: The mode in which `to_python` should run. - If mode is 'json', the output will only contain JSON serializable types. - If mode is 'python', the output may contain non-JSON-serializable Python objects. - include: A set of fields to include in the output. - exclude: A set of fields to exclude from the output. - context: Additional context to pass to the serializer. - by_alias: Whether to use the field's alias in the dictionary key if defined. - exclude_unset: Whether to exclude fields that have not been explicitly set. - exclude_defaults: Whether to exclude fields that are set to their default value. - exclude_none: Whether to exclude fields that have a value of `None`. - exclude_computed_fields: Whether to exclude computed fields. - While this can be useful for round-tripping, it is usually recommended to use the dedicated - `round_trip` parameter instead. - round_trip: If True, dumped values should be valid as input for non-idempotent types such as Json[T]. - warnings: How to handle serialization errors. False/"none" ignores them, True/"warn" logs errors, - "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError]. - fallback: A function to call when an unknown value is encountered. If not provided, - a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. - serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. - - Returns: - A dictionary representation of the model. + mode: `"python"` keeps Python objects such as `datetime`; `"json"` converts every value + to a JSON-compatible type. + include: The fields to keep, as a set or a nested mapping. + exclude: The fields to drop, as a set or a nested mapping. + by_alias: Key the output by each field's alias rather than its attribute name. + exclude_unset: Drop fields that were never set explicitly. + exclude_defaults: Drop fields still equal to their default. + exclude_none: Drop fields whose value is `None`. + context: Pydantic v2 only; must keep its default, or this raises `ValueError`. + exclude_computed_fields: Pydantic v2 only; must keep its default, or this raises `ValueError`. + round_trip: Pydantic v2 only; must keep its default, or this raises `ValueError`. + warnings: Pydantic v2 only; must keep its default, or this raises `ValueError`. + fallback: Pydantic v2 only; must keep its default, or this raises `ValueError`. + serialize_as_any: Pydantic v2 only; must keep its default, or this raises `ValueError`. """ if mode not in {"json", "python"}: raise ValueError("mode must be either 'json' or 'python'") @@ -363,23 +363,26 @@ def model_dump_json( fallback: Callable[[Any], Any] | None = None, serialize_as_any: bool = False, ) -> str: - """Usage docs: https://docs.pydantic.dev/2.4/concepts/serialization/#modelmodel_dump_json + """Returns the model serialized as a JSON string, the Pydantic v2 `model_dump_json` call on Pydantic v1. - Generates a JSON representation of the model using Pydantic's `to_json` method. + See https://docs.pydantic.dev/latest/api/base_model/#pydantic.BaseModel.model_dump_json + for the full semantics. Args: - indent: Indentation to use in the JSON output. If None is passed, the output will be compact. - include: Field(s) to include in the JSON output. Can take either a string or set of strings. - exclude: Field(s) to exclude from the JSON output. Can take either a string or set of strings. - by_alias: Whether to serialize using field aliases. - exclude_unset: Whether to exclude fields that have not been explicitly set. - exclude_defaults: Whether to exclude fields that have the default value. - exclude_none: Whether to exclude fields that have a value of `None`. - round_trip: Whether to use serialization/deserialization between JSON and class instance. - warnings: Whether to show any warnings that occurred during serialization. - - Returns: - A JSON string representation of the model. + indent: Spaces to indent nested values by; `None` writes compact JSON. + include: The fields to keep, as a set or a nested mapping. + exclude: The fields to drop, as a set or a nested mapping. + by_alias: Key the output by each field's alias rather than its attribute name. + exclude_unset: Drop fields that were never set explicitly. + exclude_defaults: Drop fields still equal to their default. + exclude_none: Drop fields whose value is `None`. + ensure_ascii: Pydantic v2 only; must keep its default, or this raises `ValueError`. + context: Pydantic v2 only; must keep its default, or this raises `ValueError`. + exclude_computed_fields: Pydantic v2 only; must keep its default, or this raises `ValueError`. + round_trip: Pydantic v2 only; must keep its default, or this raises `ValueError`. + warnings: Pydantic v2 only; must keep its default, or this raises `ValueError`. + fallback: Pydantic v2 only; must keep its default, or this raises `ValueError`. + serialize_as_any: Pydantic v2 only; must keep its default, or this raises `ValueError`. """ if round_trip != False: raise ValueError("round_trip is only supported in Pydantic v2") diff --git a/src/tembo/_qs.py b/src/tembo/_qs.py index 4127c19c..7ab08023 100644 --- a/src/tembo/_qs.py +++ b/src/tembo/_qs.py @@ -118,13 +118,11 @@ def _stringify_item( return [(key, serialised)] def _primitive_value_to_str(self, value: PrimitiveData) -> str: - # copied from httpx - if value is True: - return "true" - elif value is False: - return "false" - elif value is None: + """Renders one query value: booleans as lowercase `true` / `false`, and `None` as empty.""" + if value is None: return "" + if isinstance(value, bool): + return "true" if value else "false" return str(value) diff --git a/src/tembo/_streaming.py b/src/tembo/_streaming.py index 78b80803..dc1d9ac3 100644 --- a/src/tembo/_streaming.py +++ b/src/tembo/_streaming.py @@ -1,6 +1,7 @@ -# Note: initially copied from https://github.com/florimondmanca/httpx-sse/blob/master/src/httpx_sse/_decoders.py from __future__ import annotations +import re +import sys import json import inspect from types import TracebackType @@ -10,6 +11,8 @@ import httpx from ._utils import extract_type_var_from_base +from ._decoding import iter_decoded, aiter_decoded +from ._exceptions import TemboError if TYPE_CHECKING: from ._client import Tembo, AsyncTembo @@ -49,7 +52,7 @@ def __iter__(self) -> Iterator[_T]: yield item def _iter_events(self) -> Iterator[ServerSentEvent]: - yield from self._decoder.iter_bytes(self.response.iter_bytes()) + yield from self._decoder.iter_bytes(iter_decoded(self.response)) def __stream__(self) -> Iterator[_T]: cast_to = cast(Any, self._cast_to) @@ -120,7 +123,7 @@ async def __aiter__(self) -> AsyncIterator[_T]: yield item async def _iter_events(self) -> AsyncIterator[ServerSentEvent]: - async for sse in self._decoder.aiter_bytes(self.response.aiter_bytes()): + async for sse in self._decoder.aiter_bytes(aiter_decoded(self.response)): yield sse async def __stream__(self) -> AsyncIterator[_T]: @@ -203,110 +206,169 @@ def __repr__(self) -> str: return f"ServerSentEvent(event={self.event}, data={self.data}, id={self.id}, retry={self.retry})" +_LINE_TERMINATOR = re.compile(rb"\r\n|\r|\n") +_UTF8_BOM = b"\xef\xbb\xbf" + +# The most bytes one line still waiting for its terminator, or one event still waiting for the blank +# line that dispatches it, may hold. A server that never ends a line or an event would otherwise +# make the decoder buffer without bound. +MAX_PENDING_BYTES = 64 * 1024 * 1024 + +# What one pending `data` line costs besides the string itself: its slot in the list of lines. +_LIST_SLOT_BYTES = 8 + + +def _pending_too_large(what: str) -> TemboError: + """The error for a stream whose pending `what` grew past `MAX_PENDING_BYTES`.""" + return TemboError( + f"Server-sent event stream {what} exceeds the {MAX_PENDING_BYTES} byte limit " + f"({MAX_PENDING_BYTES // (1024 * 1024)} MiB)" + ) + + class SSEDecoder: - _data: list[str] - _event: str | None - _retry: int | None - _last_event_id: str | None + """Incremental decoder for a `text/event-stream` body. + + Follows "Interpreting an event stream" in the WHATWG HTML standard: + https://html.spec.whatwg.org/multipage/server-sent-events.html#event-stream-interpretation + + Bytes arrive in arbitrary network chunks, so an unfinished line is buffered until its line + terminator arrives. As the standard requires, a byte order mark opening the stream is skipped, + bytes that are not valid UTF-8 decode to U+FFFD, and an event still pending when the stream ends + (it never saw the blank line that dispatches it) is dropped. + """ def __init__(self) -> None: - self._event = None - self._data = [] - self._last_event_id = None - self._retry = None + self._event: str | None = None + self._data: list[str] = [] + # Characters held in `_data`, kept beside it so the event cap costs nothing per line. + self._data_size = 0 + self._last_event_id: str | None = None + self._retry: int | None = None + # The pieces of a line whose terminator has not arrived yet, joined once it does so a long + # line arriving in many chunks is copied once rather than once per chunk. + self._pending: list[bytes] = [] + # Bytes held in `_pending`, kept beside it so the line cap costs nothing per chunk. + self._pending_size = 0 + # Set when a chunk ended on `\r`: a `\n` opening the next chunk completes that `\r\n`, so it + # must not be read as a second, empty line (which would dispatch an event). + self._skip_leading_lf = False + # The opening bytes of the stream, held until there are enough to tell whether they are a + # byte order mark (which can itself arrive split across chunks); `None` once that is settled. + self._head: bytes | None = b"" def iter_bytes(self, iterator: Iterator[bytes]) -> Iterator[ServerSentEvent]: - """Given an iterator that yields raw binary data, iterate over it & yield every event encountered""" - for chunk in self._iter_chunks(iterator): - # Split before decoding so splitlines() only uses \r and \n - for raw_line in chunk.splitlines(): - line = raw_line.decode("utf-8") - sse = self.decode(line) - if sse: - yield sse - - def _iter_chunks(self, iterator: Iterator[bytes]) -> Iterator[bytes]: - """Given an iterator that yields raw binary data, iterate over it and yield individual SSE chunks""" - data = b"" + """Decodes raw body chunks and yields each event as soon as its blank line arrives.""" for chunk in iterator: - for line in chunk.splitlines(keepends=True): - data += line - if data.endswith((b"\r\r", b"\n\n", b"\r\n\r\n")): - yield data - data = b"" - if data: - yield data + yield from self._feed(chunk) async def aiter_bytes(self, iterator: AsyncIterator[bytes]) -> AsyncIterator[ServerSentEvent]: - """Given an iterator that yields raw binary data, iterate over it & yield every event encountered""" - async for chunk in self._aiter_chunks(iterator): - # Split before decoding so splitlines() only uses \r and \n - for raw_line in chunk.splitlines(): - line = raw_line.decode("utf-8") - sse = self.decode(line) - if sse: - yield sse - - async def _aiter_chunks(self, iterator: AsyncIterator[bytes]) -> AsyncIterator[bytes]: - """Given an iterator that yields raw binary data, iterate over it and yield individual SSE chunks""" - data = b"" + """Decodes raw async body chunks and yields each event as soon as its blank line arrives.""" async for chunk in iterator: - for line in chunk.splitlines(keepends=True): - data += line - if data.endswith((b"\r\r", b"\n\n", b"\r\n\r\n")): - yield data - data = b"" - if data: - yield data + for sse in self._feed(chunk): + yield sse + + def _feed(self, chunk: bytes) -> Iterator[ServerSentEvent]: + """Consumes one chunk and yields the events completed by the lines it finishes.""" + if not chunk: + return + if self._head is not None: + chunk = self._head + chunk + if len(chunk) < len(_UTF8_BOM) and _UTF8_BOM.startswith(chunk): + self._head = chunk + return + self._head = None + if chunk.startswith(_UTF8_BOM): + chunk = chunk[len(_UTF8_BOM) :] + if not chunk: + return + if self._skip_leading_lf and chunk.startswith(b"\n"): + chunk = chunk[1:] + self._skip_leading_lf = chunk.endswith(b"\r") + + # Lines are split on the raw bytes, which no UTF-8 continuation byte can be mistaken for, + # and each complete line is decoded on its own so a character split across chunks waits + # in `_pending` with the rest of its line. Only the new chunk is scanned: `_pending` holds + # no terminator, and a `\r\n` split across chunks was handled above. + *finished, unfinished = _LINE_TERMINATOR.split(chunk) + if not finished: + self._hold(unfinished) + return + if self._pending_size + len(finished[0]) > MAX_PENDING_BYTES: + raise _pending_too_large("line") + lines = [b"".join([*self._pending, finished[0]]), *finished[1:]] + self._pending = [] + self._pending_size = 0 + self._hold(unfinished) + for line in lines: + sse = self.decode(line.decode("utf-8", errors="replace")) + if sse is not None: + yield sse + + def _hold(self, piece: bytes) -> None: + """Keeps `piece`, the start of a line whose terminator has not arrived, within the line cap.""" + if not piece: + return + self._pending_size += len(piece) + if self._pending_size > MAX_PENDING_BYTES: + raise _pending_too_large("line") + self._pending.append(piece) def decode(self, line: str) -> ServerSentEvent | None: - # See: https://html.spec.whatwg.org/multipage/server-sent-events.html#event-stream-interpretation # noqa: E501 - + """Processes one line, without its terminator, and returns the event a blank line dispatches.""" if not line: - if not self._event and not self._data and not self._last_event_id and self._retry is None: - return None - - sse = ServerSentEvent( - event=self._event, - data="\n".join(self._data), - id=self._last_event_id, - retry=self._retry, - ) - - # NOTE: as per the SSE spec, do not reset last_event_id. - self._event = None - self._data = [] - self._retry = None - - return sse + return self._dispatch() if line.startswith(":"): + # A comment, such as a keep-alive. return None - fieldname, _, value = line.partition(":") - + # A line with no colon names a field whose value is empty. + name, _, value = line.partition(":") if value.startswith(" "): value = value[1:] - if fieldname == "event": - self._event = value - elif fieldname == "data": + if name == "data": + # Each `data` line is held as its own string in a list until the event ends, so it is + # charged what that costs in memory: the string's own size (up to 4 bytes a character + # for astral text, plus its header) and its list slot. An empty line is not free. + self._data_size += sys.getsizeof(value) + _LIST_SLOT_BYTES + if self._data_size > MAX_PENDING_BYTES: + raise _pending_too_large("event") self._data.append(value) - elif fieldname == "id": - if "\0" in value: - pass - else: + elif name == "event": + self._event = value + elif name == "id": + # The standard ignores an id containing NUL rather than truncating it. + if "\0" not in value: self._last_event_id = value - elif fieldname == "retry": - try: + elif name == "retry": + # Only ASCII digits count; `int()` alone would also take a sign, spaces or other scripts' digits. + if value.isascii() and value.isdigit(): self._retry = int(value) - except (TypeError, ValueError): - pass - else: - pass # Field is ignored. - + # Any other field name is ignored. return None + def _dispatch(self) -> ServerSentEvent | None: + """Builds the event a blank line completes, and resets the per-event fields.""" + # An event with no data still dispatches when it set a name, an id or a retry, so a consumer + # sees named events that carry no payload. The last event id persists across events, so + # once one is set every blank line dispatches. + if not (self._data or self._event or self._last_event_id or self._retry is not None): + return None + + sse = ServerSentEvent( + event=self._event, + data="\n".join(self._data), + id=self._last_event_id, + retry=self._retry, + ) + self._event = None + self._data = [] + self._data_size = 0 + self._retry = None + return sse + @runtime_checkable class SSEBytesDecoder(Protocol): diff --git a/src/tembo/_types.py b/src/tembo/_types.py index 9ad42025..813aaa45 100644 --- a/src/tembo/_types.py +++ b/src/tembo/_types.py @@ -223,9 +223,14 @@ def get(self, __key: str) -> str | None: ... StrBytesIntFloat = Union[str, bytes, int, float] -# Note: copied from Pydantic -# https://github.com/pydantic/pydantic/blob/6f31f8f68ef011f84357330186f603ff295312fd/pydantic/main.py#L79 -IncEx: TypeAlias = Union[Set[int], Set[str], Mapping[int, Union["IncEx", bool]], Mapping[str, Union["IncEx", bool]]] +# The `include` / `exclude` selection `model_dump` takes: a set of field names (or sequence indexes), +# or a mapping from each one to `True` or a nested selection for that field's own value. +IncEx: TypeAlias = Union[ + Set[str], + Set[int], + Mapping[str, Union["IncEx", bool]], + Mapping[int, Union["IncEx", bool]], +] PostParser = Callable[[Any], Any] diff --git a/src/tembo/_utils/_datetime_parse.py b/src/tembo/_utils/_datetime_parse.py index 7cb9d9e6..3c6b5255 100644 --- a/src/tembo/_utils/_datetime_parse.py +++ b/src/tembo/_utils/_datetime_parse.py @@ -1,136 +1,156 @@ """ -This file contains code from https://github.com/pydantic/pydantic/blob/main/pydantic/v1/datetime_parse.py -without the Pydantic v1 specific errors. +Parsing for the `date` and `date-time` values a response carries. + +Models built without validation (`construct_type`) turn these values into `date` and `datetime` +objects here. A string is read as a date or date-time in the RFC 3339 / ISO 8601 shape, with the +departures `_DATE` and `_DATE_TIME` list; a number, or a string or bytes holding one, is read as a +Unix timestamp. + +`datetime.fromisoformat` is not used: before Python 3.11 it rejects a `Z` offset and fractions that +are not exactly three or six digits, and the supported floor is lower than that. """ from __future__ import annotations import re -from typing import Dict, Union, Optional +from typing import Union, Optional from datetime import date, datetime, timezone, timedelta from .._types import StrBytesIntFloat -date_expr = r"(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})" -time_expr = ( - r"(?P\d{1,2}):(?P\d{1,2})" - r"(?::(?P\d{1,2})(?:\.(?P\d{1,6})\d{0,6})?)?" - r"(?PZ|[+-]\d{2}(?::?\d{2})?)?$" +# RFC 3339 `full-date`, loosened to accept a one-digit month or day. +_DATE = re.compile(r"(\d{4})-(\d\d?)-(\d\d?)$") + +# RFC 3339 `date-time`, loosened the same way: a space may stand in for `T`, the seconds may be left +# out, and the offset may be `Z`, `+HH`, `+HHMM` or `+HH:MM`, or missing for a local time. It is also +# stricter in two places: `T` and `Z` must be uppercase, and a leap second (`:60`) is rejected, +# because `datetime` cannot hold one. +_DATE_TIME = re.compile( + r""" + (\d{4}) - (\d\d?) - (\d\d?) + [T ] + (\d\d?) : (\d\d?) + (?: : (\d\d?) (?: \. (\d{1,12}) )? )? + (Z | [+-] \d\d (?: :? \d\d )? )? + $ + """, + re.VERBOSE, ) -date_re = re.compile(f"{date_expr}$") -datetime_re = re.compile(f"{date_expr}[T ]{time_expr}") +_UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) +# A timestamp whose magnitude is above this is taken to be in milliseconds (or a finer unit): as +# seconds it would land after the year 2603. +_SECONDS_LIMIT = 2e10 +# Past this magnitude no unit brings the value inside the range `datetime` can represent. +_CLAMP_LIMIT = 3e20 -EPOCH = datetime(1970, 1, 1) -# if greater than this, the number is in ms, if less than or equal it's in seconds -# (in seconds this is 11th October 2603, in ms it's 20th August 1970) -MS_WATERSHED = int(2e10) -# slightly more than datetime.max in ns - (datetime.max - EPOCH).total_seconds() * 1e9 -MAX_NUMBER = int(3e20) +def _number_or_text(value: object, kind: str) -> Union[int, float, str]: + """Sorts an input into a timestamp (returned as a number) or text to parse as a `kind`. - -def _get_numeric(value: StrBytesIntFloat, native_expected_type: str) -> Union[None, int, float]: + Text that reads as a number, such as `"1700000000"` or even `"2024"`, is a timestamp. Any other + type `float()` accepts, such as `Decimal` or a NumPy scalar, is a timestamp too; everything else + raises `TypeError`. + """ if isinstance(value, (int, float)): return value + if isinstance(value, bytes): + value = value.decode() + if isinstance(value, str): + try: + return float(value) + except ValueError: + return value try: - return float(value) - except ValueError: - return None - except TypeError: - raise TypeError(f"invalid type; expected {native_expected_type}, string, bytes, int or float") from None + return float(value) # type: ignore[arg-type] + except (TypeError, ValueError): + raise TypeError(f"cannot parse a {type(value).__name__} as a {kind}") from None -def _from_unix_seconds(seconds: Union[int, float]) -> datetime: - if seconds > MAX_NUMBER: - return datetime.max - elif seconds < -MAX_NUMBER: - return datetime.min +def _from_timestamp(number: Union[int, float]) -> datetime: + """Converts a Unix timestamp to an aware UTC `datetime`. - while abs(seconds) > MS_WATERSHED: - seconds /= 1000 - dt = EPOCH + timedelta(seconds=seconds) - return dt.replace(tzinfo=timezone.utc) - - -def _parse_timezone(value: Optional[str]) -> Union[None, int, timezone]: - if value == "Z": - return timezone.utc - elif value is not None: - offset_mins = int(value[-2:]) if len(value) > 3 else 0 - offset = 60 * int(value[1:3]) + offset_mins - if value[0] == "-": - offset = -offset - return timezone(timedelta(minutes=offset)) - else: + The unit is inferred from the size: a value too large to be seconds is read as milliseconds, + one too large for that as microseconds, and so on. A value too large for any unit is clamped to + the naive `datetime.max` or `datetime.min`. + """ + magnitude = abs(number) + if magnitude > _CLAMP_LIMIT: + return datetime.max if number > 0 else datetime.min + divisor = 1 + while magnitude > _SECONDS_LIMIT * divisor: + divisor *= 1000 + return _UNIX_EPOCH + timedelta(seconds=number / divisor) + + +def _zone(offset: Optional[str]) -> Optional[timezone]: + """Returns the fixed-offset zone a `Z` or `±HH[[:]MM]` suffix names, or `None` for no suffix.""" + if offset is None: return None + if offset == "Z": + return timezone.utc + digits = offset[1:].replace(":", "") + delta = timedelta(hours=int(digits[:2]), minutes=int(digits[2:] or 0)) + # `timezone` itself rejects an offset of a day or more with a `ValueError`. + return timezone(-delta if offset.startswith("-") else delta) def parse_datetime(value: Union[datetime, StrBytesIntFloat]) -> datetime: """ - Parse a datetime/int/float/string and return a datetime.datetime. + Parses a date-time string, or a Unix timestamp in seconds or milliseconds, into a `datetime`. - This function supports time zone offsets. When the input contains one, - the output uses a timezone with a fixed offset from UTC. + A string with an offset gives an aware `datetime` with that fixed offset, and one without gives + a naive one. A timestamp gives an aware UTC `datetime`, except that one beyond the representable + range is clamped to the naive `datetime.max` or `datetime.min`. A `datetime` is returned unchanged. - Raise ValueError if the input is well formatted but not a valid datetime. - Raise ValueError if the input isn't well formatted. + Raises `ValueError` for a malformed string or out-of-range fields, and `TypeError` for any other + input type. """ if isinstance(value, datetime): return value - number = _get_numeric(value, "datetime") - if number is not None: - return _from_unix_seconds(number) - - if isinstance(value, bytes): - value = value.decode() - - assert not isinstance(value, (float, int)) + parsed = _number_or_text(value, "datetime") + if not isinstance(parsed, str): + return _from_timestamp(parsed) - match = datetime_re.match(value) + match = _DATE_TIME.match(parsed) if match is None: - raise ValueError("invalid datetime format") - - kw = match.groupdict() - if kw["microsecond"]: - kw["microsecond"] = kw["microsecond"].ljust(6, "0") - - tzinfo = _parse_timezone(kw.pop("tzinfo")) - kw_: Dict[str, Union[None, int, timezone]] = {k: int(v) for k, v in kw.items() if v is not None} - kw_["tzinfo"] = tzinfo - - return datetime(**kw_) # type: ignore + raise ValueError("invalid date-time format") + year, month, day, hour, minute, second, fraction, offset = match.groups() + # Pad the fraction out to microseconds, or cut it down to them. + microsecond = int(f"{fraction or ''}000000"[:6]) + return datetime( + int(year), + int(month), + int(day), + int(hour), + int(minute), + int(second or 0), + microsecond, + tzinfo=_zone(offset), + ) def parse_date(value: Union[date, StrBytesIntFloat]) -> date: """ - Parse a date/int/float/string and return a datetime.date. + Parses a date string, or a Unix timestamp in seconds or milliseconds, into a `date`. + + A `datetime` gives its date part, and a `date` is returned unchanged. - Raise ValueError if the input is well formatted but not a valid date. - Raise ValueError if the input isn't well formatted. + Raises `ValueError` for a malformed string or an impossible date, and `TypeError` for any other + input type. """ + if isinstance(value, datetime): + return value.date() if isinstance(value, date): - if isinstance(value, datetime): - return value.date() - else: - return value - - number = _get_numeric(value, "date") - if number is not None: - return _from_unix_seconds(number).date() + return value - if isinstance(value, bytes): - value = value.decode() + parsed = _number_or_text(value, "date") + if not isinstance(parsed, str): + return _from_timestamp(parsed).date() - assert not isinstance(value, (float, int)) - match = date_re.match(value) + match = _DATE.match(parsed) if match is None: raise ValueError("invalid date format") - - kw = {k: int(v) for k, v in match.groupdict().items()} - - try: - return date(**kw) - except ValueError: - raise ValueError("invalid date format") from None + year, month, day = (int(part) for part in match.groups()) + return date(year, month, day) diff --git a/src/tembo/_utils/_logs.py b/src/tembo/_utils/_logs.py index 4dc36eb1..37fc727b 100644 --- a/src/tembo/_utils/_logs.py +++ b/src/tembo/_utils/_logs.py @@ -1,5 +1,336 @@ +from __future__ import annotations + import os +import re import logging +from typing import Any, cast +from collections.abc import Mapping + +REDACTED = "***" + +# Credential headers, lower-cased: the conventional ones plus every header this SDK's auth schemes +# write (an `apiKey` scheme names its own, which no fixed list could know). +_REDACTED_HEADERS: frozenset[str] = frozenset( + ("authorization", "proxy-authorization", "api-key", "x-api-key", "cookie", "set-cookie") +) + +# Query parameters this SDK sends a credential in, from its `apiKey` schemes placed in the query. +# A credential there is worse off than one in a header: httpx logs every request URL at `info`. +_REDACTED_QUERY: tuple[str, ...] = () + +# The user part of a URL's authority, where a token often rides (`https://user:token@host`). Stops +# at `/`, `?`, `#` and whitespace, so an `@` in a path or query is left alone. +_USERINFO = re.compile(r"(//)[^/?#@\s]+@") + +# How deep `_redact_value` follows nested mappings and sequences before it falls back to masking +# the value's text, so a self-referencing structure cannot recurse without end. +_MAX_DEPTH = 8 + + +def _name_pattern(name: str) -> str: + """A pattern matching one credential query name in every spelling a URL can carry it in. + + Each character as written or percent-encoded, at any depth and in either hex case (`k`, `%6B`, + `%6b`, `%256B`), and a space also as `+`. A non-ASCII character is matched as written or as its + UTF-8 bytes encoded. A lone surrogate has no UTF-8 bytes, so it is matched only as written: + this runs at import time, and a spec can name a scheme that way. Only the hex ignores case, not + the name. + """ + parts: list[str] = [] + for char in name: + try: + utf8 = char.encode("utf-8") + except UnicodeEncodeError: + utf8 = b"" + encoded = "".join(f"%(?:25)*(?i:{byte:02x})" for byte in utf8) + alternatives = f"|{encoded}" if encoded else "" + plus = "|\\+" if char == " " else "" + parts.append(f"(?:{re.escape(char)}{alternatives}{plus})") + return "".join(parts) + + +def _query_pattern() -> re.Pattern[str] | None: + """The pattern behind `redact()`'s query masking, or `None` when no credential is placed in the query.""" + names = [_name_pattern(name) for name in _REDACTED_QUERY if name] + if not names: + return None + # A name counts after a separator, `?`, `&` or `;`, either literal or percent-encoded any number + # of times over (`%3F`, `%253F`), since a server can echo a whole URL encoded inside another + # parameter's value, as a redirect's `return_to` does. `;` also counts as a separator; masking an + # unrelated `;name=` value as well is harmless. The `=` after it can be encoded the same way. The + # value then runs to the next literal `&`, `#` or whitespace: an encoded `%26` may end a nested + # value, but stopping there would leak the rest of a value a sloppy encoder left half-encoded, + # and masking too much is harmless. `<`, `>` and `"` end it too, since no valid URL carries them + # unencoded, so a `Link` header's `` keeps its closing delimiter. `'` does not: a query value + # can hold it raw. A JSON-escaped `\u0026` (hex digits in either case) separates too, since + # `encoding/json` and others write `&` that way in a URL quoted inside a JSON body. No quantifier + # nests inside another, so matching stays linear on hostile input. + return re.compile( + r"((?:[?&;]|%(?:25)*(?i:3f|26|3b)|\\(?i:u0026))(?:" + "|".join(names) + r")(?:=|%(?:25)*(?i:3d)))[^&#\s<>\"]*" + ) + + +_QUERY = _query_pattern() + +# A header as httpcore's trace logging prints it, `(b'set-cookie', b'session=...')`: the response +# headers of every attempt go out at DEBUG as a list of these byte-string pairs, in whichever quote +# style each bytes repr picks. Matched pair by pair so a value is masked inside its own quotes and the +# trace keeps its `'),` structure. Each quoted part is a run of non-quote, non-backslash characters +# or escapes, so matching stays linear. +_BYTE_PAIR = re.compile(r"""(\(b?['"])([^'"\\]*)(['"],\s*)(b?)(?:'((?:[^'\\]|\\.)*)'|"((?:[^"\\]|\\.)*)")(\))""") + + +def _redact_plain(text: str) -> str: + """Masks URL userinfo and every credential query value in `text`.""" + text = _USERINFO.sub(r"\1" + REDACTED + "@", text) + if _QUERY is None: + return text + return _QUERY.sub(r"\1" + REDACTED, text) + + +def _redact_pair(match: re.Match[str]) -> str: + """One httpcore header pair with its value masked: whole for a credential header, else inside.""" + open_, name, between, prefix, single, double, close = match.groups() + if name.lower() in _REDACTED_HEADERS: + return f"{open_}{name}{between}{prefix}'{REDACTED}'{close}" + if single is not None: + return f"{open_}{name}{between}{prefix}'{_redact_plain(single)}'{close}" + return f'{open_}{name}{between}{prefix}"{_redact_plain(double)}"{close}' + + +def redact(text: str) -> str: + """Masks URL userinfo, credential header pairs and every credential query value in `text`. + + Works on free text rather than a parsed URL, so it covers a URL quoted in an error message, a + `Location` or `Link` header, and a credential nested, percent-encoded, in another parameter's + value (`/login?return_to=%2Fv1%3Fapi_key%3D...`) alike. httpcore's header pairs are masked one + by one and the text between them on its own, so a masked value cannot run past its pair. + """ + parts: list[str] = [] + end = 0 + for match in _BYTE_PAIR.finditer(text): + parts.append(_redact_plain(text[end : match.start()])) + parts.append(_redact_pair(match)) + end = match.end() + parts.append(_redact_plain(text[end:])) + return "".join(parts) + + +def _is_secret_key(key: object) -> bool: + """Whether a mapping key names a credential: a redacted header (any case) or query parameter.""" + return isinstance(key, str) and (key.lower() in _REDACTED_HEADERS or key in _REDACTED_QUERY) + + +def _is_secret_header(name: object) -> bool: + """Whether `name`, as text or bytes (hpack and httpcore hold headers as bytes), is a credential header.""" + if isinstance(name, (bytes, bytearray)): + name = bytes(name).decode("latin-1") + return isinstance(name, str) and name.lower() in _REDACTED_HEADERS + + +def _masked_like(value: object) -> object: + """`REDACTED` in the type `value` had, bytes for bytes, so a `b'...'` repr stays one.""" + return REDACTED.encode("ascii") if isinstance(value, (bytes, bytearray)) else REDACTED + + +def _redact_value(value: object, depth: int = 0) -> object: + """A log argument with every credential in it masked. + + A mapping (response headers, a request's `headers` or `params`) has the values of credential + keys replaced whole, and so does a `(name, value)` pair whose name is a credential header, the + shape hpack logs a header in (`HeaderTuple`, text or bytes). Every string, and every byte string + read as Latin-1, is passed through `redact()`. Numbers keep their type, so a `%i` or `%f` + placeholder still formats. Any other object is replaced by its masked text only when that text + held a credential (an `httpx.URL`, a request's repr), so its own formatting survives otherwise. + """ + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, str): + return redact(value) + if isinstance(value, (bytes, bytearray)): + text = bytes(value).decode("latin-1") + masked = redact(text) + return value if masked == text else masked.encode("latin-1") + if depth < _MAX_DEPTH: + if isinstance(value, Mapping): + return { + key: REDACTED if _is_secret_key(key) else _redact_value(item, depth + 1) + for key, item in cast("Mapping[Any, Any]", value).items() + } + if isinstance(value, list): + return [_redact_value(item, depth + 1) for item in cast("list[Any]", value)] + if isinstance(value, tuple): + items = cast("tuple[Any, ...]", value) + if len(items) == 2 and _is_secret_header(items[0]): + return (items[0], _masked_like(items[1])) + return tuple(_redact_value(item, depth + 1) for item in items) + text = str(value) + masked = redact(text) + return value if masked == text else masked + + +# The loggers whose records are masked, each with its children: this package's own, and the HTTP +# libraries under it, which log on loggers of their own: the request URL (httpx, at `info`), every +# response header (httpcore, at `debug`), every header an HTTP/2 connection encodes or decodes +# (hpack, at `debug`), and a websocket handshake's request line and headers (websockets, at `debug`). +_REDACTED_LOGGERS: tuple[str, ...] = ("tembo", "httpx", "httpcore", "hpack", "websockets") + + +def _redacts(name: object) -> bool: + """Whether a record logged under `name` is one this SDK masks. + + `logging.makeLogRecord` builds its record with no name and fills it in afterwards, so a name + that is not a string is simply not this SDK's. + """ + return isinstance(name, str) and any(name == root or name.startswith(root + ".") for root in _REDACTED_LOGGERS) + + +def _redact_arg(value: object) -> object: + """`_redact_value`, failing closed: an argument that cannot be masked is logged as `REDACTED`.""" + try: + return _redact_value(value) + except Exception: + return REDACTED + + +def _redact_text(text: str) -> str: + """`redact`, failing closed: text that cannot be masked is logged as `REDACTED`.""" + try: + return redact(text) + except Exception: + return REDACTED + + +# The placeholders a line logging one header as two arguments, its name then its value, joins them +# with: websockets' handshake trace (`> %s: %s`), hpack's `Adding %s=%s to the header table` and +# `Evicting %s: %s from the header table`. +_HEADER_LINE_JOINS: tuple[str, ...] = ("%s: %s", "%s=%s") + + +def _redact_header_lines(msg: str, args: tuple[object, ...]) -> tuple[object, ...]: + """Masks the value of a credential header a line logs as a name argument and a value argument. + + The name alone says which value is a secret, so the value is masked whole, in its own type. + """ + if not any(join in msg for join in _HEADER_LINE_JOINS): + return args + masked = list(args) + for index, arg in enumerate(args[:-1]): + if _is_secret_header(arg): + masked[index + 1] = _masked_like(args[index + 1]) + return tuple(masked) + + +def _is_hpack(name: str) -> bool: + """Whether a record comes from hpack's own loggers.""" + return name == "hpack" or name.startswith("hpack.") + + +def _is_byte_string(value: object) -> bool: + """Whether `value` is raw bytes, the way hpack hands it a header block.""" + return isinstance(value, (bytes, bytearray, memoryview)) + + +def _redact_header_blocks(msg: str, args: tuple[object, ...]) -> tuple[object, ...]: + """Masks every byte string an hpack record logs other than one header's name and value. + + `Encoded header block to %s` and `Decoding %s` log a whole encoded header block, each header's + name and value in it as written or Huffman-coded, which is no protection: no pattern can find a + credential in it, so the block is masked whole. A line that logs one header as a name and a + value (`_HEADER_LINE_JOINS`) keeps both, its credential values already masked. + """ + if any(join in msg for join in _HEADER_LINE_JOINS): + return args + return tuple(_masked_like(arg) if _is_byte_string(arg) else arg for arg in args) + + +def _redact_record(record: logging.LogRecord) -> None: + """Masks every credential a log record carries, in place. + + The arguments are masked one by one rather than the formatted message as a whole, so a header + mapping is masked by its keys and a URL argument's masked value ends where the URL does, not at + the next space of the surrounding message. A record with no arguments has its message masked + instead, whatever it is: a logged `httpx.URL` carries the query too. Each part fails closed: one + that cannot be masked is replaced by `REDACTED` rather than logged as it was. + + An attached traceback is formatted and masked into `exc_text`, which `logging.Formatter` uses as + is (httpx quotes the request URL in a status error's message). `exc_info` itself is left alone, + since a handler may need the exception object, so a formatter that reads `exc_info` rather than + `exc_text` still sees the raw traceback. + """ + if isinstance(record.args, Mapping): + mapping = cast("Mapping[Any, Any]", record.args) + record.args = {key: REDACTED if _is_secret_key(key) else _redact_arg(item) for key, item in mapping.items()} + elif isinstance(record.args, tuple) and record.args: + args = tuple(_redact_arg(arg) for arg in record.args) + if isinstance(record.msg, str): + args = _redact_header_lines(record.msg, args) + if _is_hpack(record.name): + args = _redact_header_blocks(record.msg, args) + record.args = args + elif isinstance(record.msg, str): + record.msg = _redact_text(record.msg) + else: + record.msg = _redact_arg(record.msg) + if record.exc_info and not record.exc_text: + try: + record.exc_text = redact(logging.Formatter().formatException(record.exc_info)) + except Exception: + record.exc_text = REDACTED + if isinstance(record.stack_info, str): + record.stack_info = _redact_text(record.stack_info) + + +def _blank_record(record: logging.LogRecord) -> None: + """Replaces everything a record could print with `REDACTED`, for one that could not be masked. + + The traceback goes too, `exc_info` as well as `exc_text`, since a handler that formats the + exception itself would otherwise print it raw. + """ + try: + had_traceback = bool(record.exc_info or record.exc_text) + record.msg = REDACTED + record.args = () + record.exc_info = None + record.exc_text = REDACTED if had_traceback else None + record.stack_info = None + except Exception: + return + + +def _install_record_factory() -> None: + """Wraps the process's log record factory so this SDK's records are masked as they are made. + + A record factory rather than a logger filter, because a logger's filters see only the records + logged on that logger itself: every runtime module logs on a child (`._base_client`), + and httpx, httpcore and websockets on children of their own. Masking at creation also comes + before every filter and handler, whoever configured them. The factory already installed is + wrapped, not replaced, so another library's factory keeps working, and records of every other + logger pass through untouched. A factory installed later that does not chain to this one turns + the masking off. + """ + previous = logging.getLogRecordFactory() + + def factory(*args: Any, **kwargs: Any) -> logging.LogRecord: + """The previous factory's record, masked when it belongs to one of `_REDACTED_LOGGERS`. + + Nothing here may raise: the factory runs inside every `log.debug(...)` in the process, so + a failure would break the caller's code rather than its logging. + """ + record = previous(*args, **kwargs) + try: + if _redacts(record.name): + _redact_record(record) + except Exception: + # Every part of `_redact_record` already fails closed; this is the last line, for a + # record a part of which cannot even be read (a mapping whose `.items()` raises). + _blank_record(record) + return record + + logging.setLogRecordFactory(factory) + + +_install_record_factory() logger: logging.Logger = logging.getLogger("tembo") httpx_logger: logging.Logger = logging.getLogger("httpx") diff --git a/src/tembo/_version.py b/src/tembo/_version.py index 75be0064..2423cff5 100644 --- a/src/tembo/_version.py +++ b/src/tembo/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Scalar. See README.md for details. __title__ = "temboSdk" -__version__ = "0.7.0" # x-release-please-version +__version__ = "0.8.0" # x-release-please-version diff --git a/src/tembo/resources/integrations/integrations.py b/src/tembo/resources/integrations/integrations.py index f53b3ad1..8cf63c88 100644 --- a/src/tembo/resources/integrations/integrations.py +++ b/src/tembo/resources/integrations/integrations.py @@ -482,7 +482,7 @@ def update( self, integration_id: str, *, - atlassian_service_account: Optional[integration_update_params.Variant4AtlassianServiceAccount], + gitlab: integration_update_params.Variant4Gitlab, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -491,7 +491,21 @@ def update( timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> IntegrationUpdateResponse: ... - @required_args(["settings"], ["aws"], ["postgres"], ["atlassian_service_account"]) + @overload + def update( + self, + integration_id: str, + *, + atlassian_service_account: Optional[integration_update_params.Variant5AtlassianServiceAccount], + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> IntegrationUpdateResponse: ... + + @required_args(["settings"], ["aws"], ["postgres"], ["gitlab"], ["atlassian_service_account"]) def update( self, integration_id: str, @@ -499,7 +513,8 @@ def update( settings: integration_update_params.VariantSettings | Omit = omit, aws: integration_update_params.Variant2Aws | Omit = omit, postgres: integration_update_params.Variant3Postgres | Omit = omit, - atlassian_service_account: Optional[integration_update_params.Variant4AtlassianServiceAccount] | Omit = omit, + gitlab: integration_update_params.Variant4Gitlab | Omit = omit, + atlassian_service_account: Optional[integration_update_params.Variant5AtlassianServiceAccount] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -508,13 +523,14 @@ def update( timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> IntegrationUpdateResponse: """ - Update integration settings such as the default repository, Slack channel restrictions, automatic solving, or Sentry environments; replace AWS or PostgreSQL configuration; set or remove Atlassian service-account credentials. + Update integration settings such as the default repository, Slack channel restrictions, automatic solving, or Sentry environments; replace AWS or PostgreSQL configuration; rotate GitLab service account access tokens; set or remove Atlassian service-account credentials. Args: integration_id: Path parameter. settings: Update integration settings, preserving unspecified settings. aws: Replace AWS environment and assumed-role configuration. postgres: Replace PostgreSQL connection configuration. + gitlab: Replace GitLab credentials with a service account access token, keeping repositories and projects connected. atlassian_service_account: Set Atlassian service-account OAuth credentials for triggers, or null to remove them. extra_headers: Send extra headers with the request. extra_query: Send extra query parameters with the request. @@ -541,6 +557,7 @@ def update( "settings": settings, "aws": aws, "postgres": postgres, + "gitlab": gitlab, "atlassian_service_account": atlassian_service_account, }, integration_update_params.IntegrationUpdateParams, @@ -1384,7 +1401,21 @@ async def update( self, integration_id: str, *, - atlassian_service_account: Optional[integration_update_params.Variant4AtlassianServiceAccount], + gitlab: integration_update_params.Variant4Gitlab, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> IntegrationUpdateResponse: ... + + @overload + async def update( + self, + integration_id: str, + *, + atlassian_service_account: Optional[integration_update_params.Variant5AtlassianServiceAccount], # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -1393,7 +1424,7 @@ async def update( timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> IntegrationUpdateResponse: ... - @required_args(["settings"], ["aws"], ["postgres"], ["atlassian_service_account"]) + @required_args(["settings"], ["aws"], ["postgres"], ["gitlab"], ["atlassian_service_account"]) async def update( self, integration_id: str, @@ -1401,7 +1432,8 @@ async def update( settings: integration_update_params.VariantSettings | Omit = omit, aws: integration_update_params.Variant2Aws | Omit = omit, postgres: integration_update_params.Variant3Postgres | Omit = omit, - atlassian_service_account: Optional[integration_update_params.Variant4AtlassianServiceAccount] | Omit = omit, + gitlab: integration_update_params.Variant4Gitlab | Omit = omit, + atlassian_service_account: Optional[integration_update_params.Variant5AtlassianServiceAccount] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -1410,13 +1442,14 @@ async def update( timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> IntegrationUpdateResponse: """ - Update integration settings such as the default repository, Slack channel restrictions, automatic solving, or Sentry environments; replace AWS or PostgreSQL configuration; set or remove Atlassian service-account credentials. + Update integration settings such as the default repository, Slack channel restrictions, automatic solving, or Sentry environments; replace AWS or PostgreSQL configuration; rotate GitLab service account access tokens; set or remove Atlassian service-account credentials. Args: integration_id: Path parameter. settings: Update integration settings, preserving unspecified settings. aws: Replace AWS environment and assumed-role configuration. postgres: Replace PostgreSQL connection configuration. + gitlab: Replace GitLab credentials with a service account access token, keeping repositories and projects connected. atlassian_service_account: Set Atlassian service-account OAuth credentials for triggers, or null to remove them. extra_headers: Send extra headers with the request. extra_query: Send extra query parameters with the request. @@ -1443,6 +1476,7 @@ async def update( "settings": settings, "aws": aws, "postgres": postgres, + "gitlab": gitlab, "atlassian_service_account": atlassian_service_account, }, integration_update_params.IntegrationUpdateParams, diff --git a/src/tembo/resources/live/ws.py b/src/tembo/resources/live/ws.py index b7713475..5e174765 100644 --- a/src/tembo/resources/live/ws.py +++ b/src/tembo/resources/live/ws.py @@ -67,6 +67,10 @@ def __init__( self._max_retries = max_retries self._initial_delay = initial_delay self._max_delay = max_delay + # Attempts spent since the connection last stayed up, carried across drops so a server that + # accepts and drops at once uses up max_retries instead of being redialed forever. + self._spent_attempts = 0 + self._reconnected_at: float | None = None self._extra_query = extra_query self._extra_headers = extra_headers self._intentionally_closed = False @@ -148,7 +152,12 @@ def _reconnect(self, exc: Exception) -> bool: return False self._is_reconnecting = True - for attempt in range(1, self._max_retries + 1): + # Only a reconnected socket that stayed up for the longest backoff (and at least a second) + # starts the count again at 1. + if self._reconnected_at is None or time.monotonic() - self._reconnected_at >= max(self._max_delay, 1.0): + self._spent_attempts = 0 + for attempt in range(self._spent_attempts + 1, self._max_retries + 1): + self._spent_attempts = attempt base_delay = min(self._initial_delay * (2 ** (attempt - 1)), self._max_delay) jitter = 0.75 + random.random() * 0.25 delay = base_delay * jitter @@ -187,6 +196,7 @@ def _reconnect(self, exc: Exception) -> bool: try: self._connection = self._make_ws(self._extra_query, self._extra_headers) log.info("Reconnected to WebSocket API") + self._reconnected_at = time.monotonic() self._flush_send_queue() self._is_reconnecting = False return True @@ -268,6 +278,10 @@ def __init__( self._max_retries = max_retries self._initial_delay = initial_delay self._max_delay = max_delay + # Attempts spent since the connection last stayed up, carried across drops so a server that + # accepts and drops at once uses up max_retries instead of being redialed forever. + self._spent_attempts = 0 + self._reconnected_at: float | None = None self._extra_query = extra_query self._extra_headers = extra_headers self._intentionally_closed = False @@ -351,7 +365,12 @@ async def _reconnect(self, exc: Exception) -> bool: return False self._is_reconnecting = True - for attempt in range(1, self._max_retries + 1): + # Only a reconnected socket that stayed up for the longest backoff (and at least a second) + # starts the count again at 1. + if self._reconnected_at is None or time.monotonic() - self._reconnected_at >= max(self._max_delay, 1.0): + self._spent_attempts = 0 + for attempt in range(self._spent_attempts + 1, self._max_retries + 1): + self._spent_attempts = attempt base_delay = min(self._initial_delay * (2 ** (attempt - 1)), self._max_delay) jitter = 0.75 + random.random() * 0.25 delay = base_delay * jitter @@ -390,6 +409,7 @@ async def _reconnect(self, exc: Exception) -> bool: try: self._connection = await self._make_ws(self._extra_query, self._extra_headers) log.info("Reconnected to WebSocket API") + self._reconnected_at = time.monotonic() await self._flush_send_queue() self._is_reconnecting = False return True @@ -728,7 +748,15 @@ async def _connect_ws(self, extra_query: Query, extra_headers: Headers) -> Async if self.__websocket_connection_options: log.debug("Connection options: %s", self.__websocket_connection_options) - return await connect( + # websockets 13.1+ follows a handshake redirect itself and resends `additional_headers`, the + # credentials, to whatever host `Location` names; the sync client never follows one. A 3xx + # is reported as the `InvalidStatus` it raises instead, so both clients refuse it alike. + class _ConnectWithoutRedirects(connect): + def process_redirect(self, exc: Exception) -> Exception | str: + """Reports the redirect as the handshake error instead of following it.""" + return exc + + return await _ConnectWithoutRedirects( str(url), user_agent_header=self.__client.user_agent, additional_headers=headers, diff --git a/src/tembo/resources/models.py b/src/tembo/resources/models.py index 26bb8d06..07d737f2 100644 --- a/src/tembo/resources/models.py +++ b/src/tembo/resources/models.py @@ -37,52 +37,7 @@ def with_streaming_response(self) -> ModelsResourceWithStreamingResponse: def list( self, *, - cursor: Literal[ - "claude-opus-5-5", - "claude-fable-5-1", - "claude-fable-5", - "claude-opus-5", - "claude-sonnet-5-5", - "claude-haiku-5-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-opus-4-5", - "claude-4-5-haiku", - "gpt-6.1-sol", - "gpt-6.1-terra", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5-pro", - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - "gpt-5.3-codex-spark", - "gpt-5.2-codex", - "gpt-5.2", - "glm-5p3", - "glm-5p3-flash", - "composer-1.5", - "composer-2", - "composer-2-fast", - "composer-2.5", - "gemini-3.1-pro", - "gemini-3.5-flash", - "grok-4.7", - "grok", - "kimi-k3", - "minimax-m2p7", - "minimax-m3", - "deepseek-v4-pro", - "deepseek-v4p1-flash", - ] - | Omit = omit, + cursor: str | Omit = omit, limit: int | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -126,51 +81,7 @@ def list( def update( self, - model_name: Literal[ - "claude-opus-5-5", - "claude-fable-5-1", - "claude-fable-5", - "claude-opus-5", - "claude-sonnet-5-5", - "claude-haiku-5-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-opus-4-5", - "claude-4-5-haiku", - "gpt-6.1-sol", - "gpt-6.1-terra", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5-pro", - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - "gpt-5.3-codex-spark", - "gpt-5.2-codex", - "gpt-5.2", - "glm-5p3", - "glm-5p3-flash", - "composer-1.5", - "composer-2", - "composer-2-fast", - "composer-2.5", - "gemini-3.1-pro", - "gemini-3.5-flash", - "grok-4.7", - "grok", - "kimi-k3", - "minimax-m2p7", - "minimax-m3", - "deepseek-v4-pro", - "deepseek-v4p1-flash", - ], + model_name: str, *, enabled: bool | Omit = omit, inference_provider: Optional[ @@ -202,7 +113,7 @@ def update( Example: ```python model = client.models.update( - model_name="claude-opus-5-5", + model_name="modelName", ) ``` """ @@ -236,52 +147,7 @@ def with_streaming_response(self) -> AsyncModelsResourceWithStreamingResponse: async def list( self, *, - cursor: Literal[ - "claude-opus-5-5", - "claude-fable-5-1", - "claude-fable-5", - "claude-opus-5", - "claude-sonnet-5-5", - "claude-haiku-5-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-opus-4-5", - "claude-4-5-haiku", - "gpt-6.1-sol", - "gpt-6.1-terra", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5-pro", - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - "gpt-5.3-codex-spark", - "gpt-5.2-codex", - "gpt-5.2", - "glm-5p3", - "glm-5p3-flash", - "composer-1.5", - "composer-2", - "composer-2-fast", - "composer-2.5", - "gemini-3.1-pro", - "gemini-3.5-flash", - "grok-4.7", - "grok", - "kimi-k3", - "minimax-m2p7", - "minimax-m3", - "deepseek-v4-pro", - "deepseek-v4p1-flash", - ] - | Omit = omit, + cursor: str | Omit = omit, limit: int | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -327,51 +193,7 @@ async def list( async def update( self, - model_name: Literal[ - "claude-opus-5-5", - "claude-fable-5-1", - "claude-fable-5", - "claude-opus-5", - "claude-sonnet-5-5", - "claude-haiku-5-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-opus-4-5", - "claude-4-5-haiku", - "gpt-6.1-sol", - "gpt-6.1-terra", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5-pro", - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - "gpt-5.3-codex-spark", - "gpt-5.2-codex", - "gpt-5.2", - "glm-5p3", - "glm-5p3-flash", - "composer-1.5", - "composer-2", - "composer-2-fast", - "composer-2.5", - "gemini-3.1-pro", - "gemini-3.5-flash", - "grok-4.7", - "grok", - "kimi-k3", - "minimax-m2p7", - "minimax-m3", - "deepseek-v4-pro", - "deepseek-v4p1-flash", - ], + model_name: str, *, enabled: bool | Omit = omit, inference_provider: Optional[ @@ -403,7 +225,7 @@ async def update( Example: ```python model = await client.models.update( - model_name="claude-opus-5-5", + model_name="modelName", ) ``` """ diff --git a/src/tembo/resources/skills/skills.py b/src/tembo/resources/skills/skills.py index 09b56a66..8bd22fbe 100644 --- a/src/tembo/resources/skills/skills.py +++ b/src/tembo/resources/skills/skills.py @@ -153,7 +153,7 @@ def create( skill = client.skills.create( description="", name="x", - files=[{"content": "", "filename": "x"}], + files=[{"content": "", "encoding": "utf8", "filename": "x"}], ) ``` """ @@ -427,7 +427,7 @@ async def create( skill = await client.skills.create( description="", name="x", - files=[{"content": "", "filename": "x"}], + files=[{"content": "", "encoding": "utf8", "filename": "x"}], ) ``` """ diff --git a/src/tembo/types/integration_update_params.py b/src/tembo/types/integration_update_params.py index a60d1fa3..4e9dee80 100644 --- a/src/tembo/types/integration_update_params.py +++ b/src/tembo/types/integration_update_params.py @@ -17,23 +17,35 @@ "Variant3", "Variant3Postgres", "Variant4", - "Variant4AtlassianServiceAccount", + "Variant4Gitlab", + "Variant5", + "Variant5AtlassianServiceAccount", ] -class Variant4AtlassianServiceAccount(TypedDict, total=False): +class Variant5AtlassianServiceAccount(TypedDict, total=False): client_id: Required[Annotated[str, PropertyInfo(alias="clientId")]] client_secret: Required[Annotated[str, PropertyInfo(alias="clientSecret")]] -class Variant4(TypedDict, total=False): +class Variant5(TypedDict, total=False): atlassian_service_account: Required[ - Annotated[Optional[Variant4AtlassianServiceAccount], PropertyInfo(alias="atlassianServiceAccount")] + Annotated[Optional[Variant5AtlassianServiceAccount], PropertyInfo(alias="atlassianServiceAccount")] ] """Set Atlassian service-account OAuth credentials for triggers, or null to remove them.""" +class Variant4Gitlab(TypedDict, total=False): + access_token: Required[Annotated[str, PropertyInfo(alias="accessToken")]] + """GitLab access token used to connect the integration.""" + + +class Variant4(TypedDict, total=False): + gitlab: Required[Variant4Gitlab] + """Replace GitLab credentials with a service account access token, keeping repositories and projects connected.""" + + class Variant3Postgres(TypedDict, total=False): ca_certificate: Annotated[str, PropertyInfo(alias="caCertificate")] """Optional CA certificate for the PostgreSQL TLS connection.""" @@ -78,4 +90,4 @@ class Variant(TypedDict, total=False): """Update integration settings, preserving unspecified settings.""" -IntegrationUpdateParams: TypeAlias = Union[Variant, Variant2, Variant3, Variant4] +IntegrationUpdateParams: TypeAlias = Union[Variant, Variant2, Variant3, Variant4, Variant5] diff --git a/src/tembo/types/model_list_params.py b/src/tembo/types/model_list_params.py index 8dcd9a6b..d682fa16 100644 --- a/src/tembo/types/model_list_params.py +++ b/src/tembo/types/model_list_params.py @@ -2,57 +2,13 @@ from __future__ import annotations -from typing_extensions import Literal, TypedDict +from typing_extensions import TypedDict __all__ = ["ModelListParams"] class ModelListParams(TypedDict, total=False): - cursor: Literal[ - "claude-opus-5-5", - "claude-fable-5-1", - "claude-fable-5", - "claude-opus-5", - "claude-sonnet-5-5", - "claude-haiku-5-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-opus-4-5", - "claude-4-5-haiku", - "gpt-6.1-sol", - "gpt-6.1-terra", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5-pro", - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - "gpt-5.3-codex-spark", - "gpt-5.2-codex", - "gpt-5.2", - "glm-5p3", - "glm-5p3-flash", - "composer-1.5", - "composer-2", - "composer-2-fast", - "composer-2.5", - "gemini-3.1-pro", - "gemini-3.5-flash", - "grok-4.7", - "grok", - "kimi-k3", - "minimax-m2p7", - "minimax-m3", - "deepseek-v4-pro", - "deepseek-v4p1-flash", - ] + cursor: str """Continuation cursor from the previous response nextCursor. Omit for the first page; null nextCursor means there are no more pages.""" limit: int diff --git a/src/tembo/types/model_list_response.py b/src/tembo/types/model_list_response.py index c0380532..37c8c661 100644 --- a/src/tembo/types/model_list_response.py +++ b/src/tembo/types/model_list_response.py @@ -182,51 +182,7 @@ class ItemInferenceProviderAccess(BaseModel): class Item(BaseModel): - name: Literal[ - "claude-opus-5-5", - "claude-fable-5-1", - "claude-fable-5", - "claude-opus-5", - "claude-sonnet-5-5", - "claude-haiku-5-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-opus-4-5", - "claude-4-5-haiku", - "gpt-6.1-sol", - "gpt-6.1-terra", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5-pro", - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - "gpt-5.3-codex-spark", - "gpt-5.2-codex", - "gpt-5.2", - "glm-5p3", - "glm-5p3-flash", - "composer-1.5", - "composer-2", - "composer-2-fast", - "composer-2.5", - "gemini-3.1-pro", - "gemini-3.5-flash", - "grok-4.7", - "grok", - "kimi-k3", - "minimax-m2p7", - "minimax-m3", - "deepseek-v4-pro", - "deepseek-v4p1-flash", - ] + name: str label: str @@ -280,50 +236,4 @@ class Item(BaseModel): class ModelListResponse(BaseModel): items: List[Item] - next_cursor: Optional[ - Literal[ - "claude-opus-5-5", - "claude-fable-5-1", - "claude-fable-5", - "claude-opus-5", - "claude-sonnet-5-5", - "claude-haiku-5-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-opus-4-5", - "claude-4-5-haiku", - "gpt-6.1-sol", - "gpt-6.1-terra", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5-pro", - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - "gpt-5.3-codex-spark", - "gpt-5.2-codex", - "gpt-5.2", - "glm-5p3", - "glm-5p3-flash", - "composer-1.5", - "composer-2", - "composer-2-fast", - "composer-2.5", - "gemini-3.1-pro", - "gemini-3.5-flash", - "grok-4.7", - "grok", - "kimi-k3", - "minimax-m2p7", - "minimax-m3", - "deepseek-v4-pro", - "deepseek-v4p1-flash", - ] - ] = FieldInfo(alias="nextCursor", default=None) + next_cursor: Optional[str] = FieldInfo(alias="nextCursor", default=None) diff --git a/src/tembo/types/model_update_response.py b/src/tembo/types/model_update_response.py index 1d478656..cdd23639 100644 --- a/src/tembo/types/model_update_response.py +++ b/src/tembo/types/model_update_response.py @@ -181,51 +181,7 @@ class InferenceProviderAccess(BaseModel): class ModelUpdateResponse(BaseModel): - name: Literal[ - "claude-opus-5-5", - "claude-fable-5-1", - "claude-fable-5", - "claude-opus-5", - "claude-sonnet-5-5", - "claude-haiku-5-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-opus-4-5", - "claude-4-5-haiku", - "gpt-6.1-sol", - "gpt-6.1-terra", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5-pro", - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - "gpt-5.3-codex-spark", - "gpt-5.2-codex", - "gpt-5.2", - "glm-5p3", - "glm-5p3-flash", - "composer-1.5", - "composer-2", - "composer-2-fast", - "composer-2.5", - "gemini-3.1-pro", - "gemini-3.5-flash", - "grok-4.7", - "grok", - "kimi-k3", - "minimax-m2p7", - "minimax-m3", - "deepseek-v4-pro", - "deepseek-v4p1-flash", - ] + name: str label: str diff --git a/src/tembo/types/runtime_list_response.py b/src/tembo/types/runtime_list_response.py index 56b022a3..7efb2f4c 100644 --- a/src/tembo/types/runtime_list_response.py +++ b/src/tembo/types/runtime_list_response.py @@ -47,51 +47,7 @@ class ItemCompatibleModelAvailability2(BaseModel): class ItemCompatibleModel(BaseModel): - name: Literal[ - "claude-opus-5-5", - "claude-fable-5-1", - "claude-fable-5", - "claude-opus-5", - "claude-sonnet-5-5", - "claude-haiku-5-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-opus-4-5", - "claude-4-5-haiku", - "gpt-6.1-sol", - "gpt-6.1-terra", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5-pro", - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - "gpt-5.3-codex-spark", - "gpt-5.2-codex", - "gpt-5.2", - "glm-5p3", - "glm-5p3-flash", - "composer-1.5", - "composer-2", - "composer-2-fast", - "composer-2.5", - "gemini-3.1-pro", - "gemini-3.5-flash", - "grok-4.7", - "grok", - "kimi-k3", - "minimax-m2p7", - "minimax-m3", - "deepseek-v4-pro", - "deepseek-v4p1-flash", - ] + name: str label: str @@ -140,51 +96,7 @@ class Item(BaseModel): display_name: str = FieldInfo(alias="displayName") - default_model_name: Literal[ - "claude-opus-5-5", - "claude-fable-5-1", - "claude-fable-5", - "claude-opus-5", - "claude-sonnet-5-5", - "claude-haiku-5-5", - "claude-opus-4-8", - "claude-opus-4-7", - "claude-opus-4-6", - "claude-opus-4-5", - "claude-4-5-haiku", - "gpt-6.1-sol", - "gpt-6.1-terra", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.5-pro", - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.3-codex", - "gpt-5.3-codex-spark", - "gpt-5.2-codex", - "gpt-5.2", - "glm-5p3", - "glm-5p3-flash", - "composer-1.5", - "composer-2", - "composer-2-fast", - "composer-2.5", - "gemini-3.1-pro", - "gemini-3.5-flash", - "grok-4.7", - "grok", - "kimi-k3", - "minimax-m2p7", - "minimax-m3", - "deepseek-v4-pro", - "deepseek-v4p1-flash", - ] = FieldInfo(alias="defaultModelName") + default_model_name: str = FieldInfo(alias="defaultModelName") requires_api_key: bool = FieldInfo(alias="requiresApiKey") diff --git a/src/tembo/types/skill_create_params.py b/src/tembo/types/skill_create_params.py index 967b95df..997e6560 100644 --- a/src/tembo/types/skill_create_params.py +++ b/src/tembo/types/skill_create_params.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import Iterable -from typing_extensions import Required, TypedDict +from typing_extensions import Literal, Required, TypedDict __all__ = ["SkillCreateParams", "File"] @@ -21,7 +21,10 @@ class SkillCreateParams(TypedDict, total=False): class File(TypedDict, total=False): content: Required[str] - """Complete text content of the skill file.""" + """Complete file content: plain text, or base64 when encoding is base64.""" + + encoding: Literal["utf8", "base64"] + """utf8 for text files; base64 for binary files such as images.""" filename: Required[str] """Relative file path within the skill, such as SKILL.md.""" diff --git a/src/tembo/types/skill_create_response.py b/src/tembo/types/skill_create_response.py index 00bc6ce2..262c0e2f 100644 --- a/src/tembo/types/skill_create_response.py +++ b/src/tembo/types/skill_create_response.py @@ -4,6 +4,7 @@ from typing import List from datetime import datetime +from typing_extensions import Literal from pydantic import Field as FieldInfo @@ -15,6 +16,8 @@ class File(BaseModel): content: str + encoding: Literal["utf8", "base64"] + filename: str id: str diff --git a/src/tembo/types/skill_retrieve_response.py b/src/tembo/types/skill_retrieve_response.py index 1bbde6e1..1d90efdc 100644 --- a/src/tembo/types/skill_retrieve_response.py +++ b/src/tembo/types/skill_retrieve_response.py @@ -4,6 +4,7 @@ from typing import List from datetime import datetime +from typing_extensions import Literal from pydantic import Field as FieldInfo @@ -15,6 +16,8 @@ class File(BaseModel): content: str + encoding: Literal["utf8", "base64"] + filename: str id: str diff --git a/src/tembo/types/skill_update_params.py b/src/tembo/types/skill_update_params.py index 95570a48..5b7cbecc 100644 --- a/src/tembo/types/skill_update_params.py +++ b/src/tembo/types/skill_update_params.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import Iterable -from typing_extensions import Required, TypedDict +from typing_extensions import Literal, Required, TypedDict __all__ = ["SkillUpdateParams", "File"] @@ -21,7 +21,10 @@ class SkillUpdateParams(TypedDict, total=False): class File(TypedDict, total=False): content: Required[str] - """Complete text content of the skill file.""" + """Complete file content: plain text, or base64 when encoding is base64.""" + + encoding: Literal["utf8", "base64"] + """utf8 for text files; base64 for binary files such as images.""" filename: Required[str] """Relative file path within the skill, such as SKILL.md.""" diff --git a/src/tembo/types/skill_update_response.py b/src/tembo/types/skill_update_response.py index 76afa060..e7113ef5 100644 --- a/src/tembo/types/skill_update_response.py +++ b/src/tembo/types/skill_update_response.py @@ -4,6 +4,7 @@ from typing import List from datetime import datetime +from typing_extensions import Literal from pydantic import Field as FieldInfo @@ -15,6 +16,8 @@ class File(BaseModel): content: str + encoding: Literal["utf8", "base64"] + filename: str id: str diff --git a/src/tembo/types/users/setting_retrieve_response.py b/src/tembo/types/users/setting_retrieve_response.py index 5b334ea8..ab0d8060 100644 --- a/src/tembo/types/users/setting_retrieve_response.py +++ b/src/tembo/types/users/setting_retrieve_response.py @@ -137,6 +137,8 @@ class AppSidebarPreferences(BaseModel): show_archived_sessions: Optional[bool] = FieldInfo(alias="showArchivedSessions", default=None) + show_activity: Optional[bool] = FieldInfo(alias="showActivity", default=None) + width: Optional[float] = None right_sidebar: Optional[Dict[str, AppSidebarPreferencesRightSidebar]] = FieldInfo( diff --git a/src/tembo/types/users/setting_update_params.py b/src/tembo/types/users/setting_update_params.py index 0d2c56eb..90dcc727 100644 --- a/src/tembo/types/users/setting_update_params.py +++ b/src/tembo/types/users/setting_update_params.py @@ -197,6 +197,8 @@ class AppSidebarPreferences(TypedDict, total=False): show_archived_sessions: Annotated[bool, PropertyInfo(alias="showArchivedSessions")] + show_activity: Annotated[bool, PropertyInfo(alias="showActivity")] + width: float right_sidebar: Annotated[Dict[str, AppSidebarPreferencesRightSidebar], PropertyInfo(alias="rightSidebar")] diff --git a/src/tembo/types/users/setting_update_response.py b/src/tembo/types/users/setting_update_response.py index aa1c6e49..8428d40d 100644 --- a/src/tembo/types/users/setting_update_response.py +++ b/src/tembo/types/users/setting_update_response.py @@ -137,6 +137,8 @@ class AppSidebarPreferences(BaseModel): show_archived_sessions: Optional[bool] = FieldInfo(alias="showArchivedSessions", default=None) + show_activity: Optional[bool] = FieldInfo(alias="showActivity", default=None) + width: Optional[float] = None right_sidebar: Optional[Dict[str, AppSidebarPreferencesRightSidebar]] = FieldInfo( diff --git a/tests/smoke-test.py b/tests/smoke-test.py index 1b2a3718..8f42ed7b 100644 --- a/tests/smoke-test.py +++ b/tests/smoke-test.py @@ -112,7 +112,7 @@ def _smoke_case_9() -> None: skill = client.skills.create( description="", name="x", - files=[{"content": "", "filename": "x"}], + files=[{"content": "", "encoding": "utf8", "filename": "x"}], ) @@ -133,7 +133,7 @@ def _smoke_case_12() -> None: skill_id="7c9e6679-7425-40de-944b-e07fc1f90ae7", description="", name="x", - files=[{"content": "", "filename": "x"}], + files=[{"content": "", "encoding": "utf8", "filename": "x"}], ) @@ -625,20 +625,20 @@ def _smoke_case_71() -> None: def _smoke_case_72() -> None: model = client.models.list( - cursor="claude-opus-5-5", + cursor="cursor", limit=50, ) def _smoke_case_73() -> None: model = client.models.update( - model_name="claude-opus-5-5", + model_name="modelName", ) def _smoke_case_74() -> None: model = client.models.update( - model_name="claude-opus-5-5", + model_name="modelName", enabled=False, inference_provider="TemboManagedInference", ) @@ -701,6 +701,7 @@ def _smoke_case_81() -> None: "session_sort_by": "createdAt", "session_organization": "list", "show_archived_sessions": False, + "show_activity": False, "width": 0, "right_sidebar": {}, },