From 397b4418e8e4e947885007eb2334f4c6ecc3b186 Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 17 Sep 2026 09:40:02 -0400 Subject: [PATCH 01/12] Move CI provisioning from v1 workspaces to v2 clusters DEFAULT_MANAGEMENT_VERSION is v2 and the test suite already deploys clusters, but CI still drove v1 workspace groups and tore down through a hardcoded /v1/workspaces URL, emitting a DeprecationWarning on every run. resources/create_test_cluster.py now makes one create_cluster() call. wait_on_active covers ACTIVE, the endpoint and the firewall, replacing both polling loops and the bare sleep, and closing a gap: the script never waited for the firewall, which the API applies outside the state machine. The shared "Python Client Testing" group -- never deleted by CI -- is gone with the flat cluster model. Fixes a live leak: --expires was parsed and never passed to the API, so CI clusters had no expiry and a failed shutdown job leaked a billable deployment indefinitely. It now reaches expires_at=. POST /v2/clusters generates the admin password and ignores what is sent; PATCH ignores it too (docs/management-api-audit.md item 9), so create-then-PATCH is not available and secrets.CLUSTER_PASSWORD can no longer be the password of the cluster CI just made. The generated value is read off the create response and propagated as a cluster-password job output. Job outputs are not secrets, so it is ::add-mask::-ed where it is emitted and again in every consuming job -- the mask does not cross job boundaries, and omitting the re-mask would leak a live credential. The generated password is drawn from the full printable set (one observed value: {:D}TK*[F3Ll}Ups2pNv), so a percent-encoded cluster-password-url output is emitted alongside it for the SINGLESTOREDB_URL and CIBW_ENVIRONMENT call sites; the raw form reaches drop_db.py through the environment rather than being interpolated into a shell word. Verified that the encoded form round-trips through the SDK's own URL parser. Cluster names are cleaned to [a-z0-9]([a-z0-9-]*[a-z0-9])? and truncated to 32 characters -- CI passes a workflow name that can overrun the limit. Regions are matched to a Region object, since v2 regions have no ID, and the pattern is tried against both the display and provider region names. A project is required by the API and does not auto-resolve in a multi-project org, so --project was added, defaulting to the STANDARD-edition project and pinnable with the CLUSTER_PROJECT variable. drop_test_cluster.py now takes a cluster ID, matching what the create script emits; its old contract said workspace-id but slugified the argument into a name. The workflow teardown stays curl, so the shutdown job needs no install, with the URL moved to DELETE /v2/clusters/{id}. Also makes the deprecated v1 suite a nightly gate rather than a per-PR cost: code-check.yml and coverage.yml deselect management_v1, and coverage.yml gains a job that runs it. Verified the two selections partition the suite exactly, 905 + 65 of 970. Two docs gaps closed alongside: Cluster.update()'s admin_password lacked the warning create_cluster() carries, an asymmetry that invites the create-then-PATCH dead end; and build_docs.py rewrote workspace.Stage but not cluster.Stage, which the v2 shim now re-exports. Co-Authored-By: Claude Opus 5 --- .github/workflows/code-check.yml | 6 +- .github/workflows/coverage.yml | 48 +++++- .github/workflows/publish.yml | 47 +++++- .github/workflows/smoke-test.yml | 47 +++++- resources/build_docs.py | 5 + resources/create_test_cluster.py | 198 +++++++++++++++---------- resources/drop_test_cluster.py | 40 +---- singlestoredb/management/v2/cluster.py | 13 +- 8 files changed, 281 insertions(+), 123 deletions(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index 5ef487bc7..35c2d535d 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -137,8 +137,12 @@ jobs: - name: Run MySQL protocol tests (with management API) if: steps.check-changes.outputs.changes-detected == 'true' + # -m 'not management_v1' keeps the v2 management coverage while dropping + # the deprecated v1 suite, which coverage.yml runs nightly instead. The + # -m 'not management' steps below need no second term: they already + # exclude everything v1 deploys. run: | - pytest -v --cov=singlestoredb --pyargs singlestoredb.tests + pytest -v -m 'not management_v1' --cov=singlestoredb --pyargs singlestoredb.tests env: COVERAGE_FILE: "coverage-mysql.cov" SINGLESTOREDB_URL: "root:root@127.0.0.1:3307" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6c9546fa9..ef07ecbf0 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -36,8 +36,11 @@ jobs: pip install -e ".[dev]" - name: Run MySQL protocol tests + # -m 'not management_v1' keeps the v2 management coverage while dropping + # the deprecated v1 suite; the management-v1-tests job below is where + # that runs. run: | - pytest -v --cov=singlestoredb --pyargs singlestoredb.tests + pytest -v -m 'not management_v1' --cov=singlestoredb --pyargs singlestoredb.tests env: COVERAGE_FILE: "coverage-mysql.cov" SINGLESTOREDB_URL: "root:root@127.0.0.1:3307" @@ -82,3 +85,46 @@ jobs: coverage report coverage xml coverage html + + # The deprecated v1 management API. management.version defaults to v2, so this + # is a legacy gate: it runs here nightly rather than on every PR, and it is + # what gets deleted along with management/v1/. Selects both the mocked v1 + # units and the live v1 deployments, plus test_fusion's v1 WORKSPACE grammar. + management-v1-tests: + runs-on: ubuntu-latest + environment: Base + + services: + singlestore: + image: ghcr.io/singlestore-labs/singlestoredb-dev:latest + ports: + - 3307:3306 + - 8081:8080 + - 9081:9081 + env: + SINGLESTORE_LICENSE: ${{ secrets.SINGLESTORE_LICENSE }} + ROOT_PASSWORD: "root" + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run v1 management API tests + run: | + pytest -v -m 'management_v1' --pyargs singlestoredb.tests + env: + SINGLESTOREDB_URL: "root:root@127.0.0.1:3307" + SINGLESTOREDB_PURE_PYTHON: 0 + SINGLESTORE_LICENSE: ${{ secrets.SINGLESTORE_LICENSE }} + SINGLESTOREDB_MANAGEMENT_TOKEN: ${{ secrets.CLUSTER_API_KEY }} + SINGLESTOREDB_FUSION_ENABLE_HIDDEN: "1" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d3a669c1e..eeeb663e1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -50,7 +50,7 @@ jobs: - name: Initialize database id: initialize-database run: | - python resources/create_test_cluster.py --password="${{ secrets.CLUSTER_PASSWORD }}" --token="${{ secrets.CLUSTER_API_KEY }}" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" + python resources/create_test_cluster.py --token="${{ secrets.CLUSTER_API_KEY }}" --project="${{ vars.CLUSTER_PROJECT }}" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" env: PYTHONPATH: ${{ github.workspace }} @@ -58,6 +58,11 @@ jobs: cluster-id: ${{ steps.initialize-database.outputs.cluster-id }} cluster-host: ${{ steps.initialize-database.outputs.cluster-host }} cluster-database: ${{ steps.initialize-database.outputs.cluster-database }} + # POST /v2/clusters generates the admin password and reports it only on + # the create response, so it travels as a job output rather than living + # in secrets. Job outputs are not secrets: every job below re-masks it. + cluster-password: ${{ steps.initialize-database.outputs.cluster-password }} + cluster-password-url: ${{ steps.initialize-database.outputs.cluster-password-url }} build-and-test: needs: setup-database @@ -73,6 +78,17 @@ jobs: - windows-2022 steps: + # ::add-mask:: does not cross job boundaries, so the generated admin + # password arrives here unmasked and has to be registered again before + # any step can echo it into the log. + - name: Mask cluster password + env: + CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} + CLUSTER_PASSWORD_URL: ${{ needs.setup-database.outputs.cluster-password-url }} + run: | + echo "::add-mask::$CLUSTER_PASSWORD" + echo "::add-mask::$CLUSTER_PASSWORD_URL" + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -118,7 +134,12 @@ jobs: # points --pyargs at the workspace, so that pyproject is the inifile # here. Without the plugin pytest exits on the unknown arguments. CIBW_TEST_REQUIRES: "pytest pytest-xdist" - CIBW_ENVIRONMENT: "SINGLESTOREDB_URL='mysql://${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=0'" + # cluster-password-url, not cluster-password: the generated password + # is drawn from the full printable set, and this value has to survive + # both the userinfo half of the URL and the single-quoted shell word + # cibuildwheel evaluates. Percent-encoding leaves only unreserved + # characters and %, which are inert in both. + CIBW_ENVIRONMENT: "SINGLESTOREDB_URL='mysql://${{ secrets.CLUSTER_USER }}:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=0'" PYTHONPATH: ${{ github.workspace }} # - name: Build conda @@ -228,6 +249,15 @@ jobs: runs-on: ubuntu-latest steps: + # ::add-mask:: does not cross job boundaries; see the build-and-test job. + - name: Mask cluster password + env: + CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} + CLUSTER_PASSWORD_URL: ${{ needs.setup-database.outputs.cluster-password-url }} + run: | + echo "::add-mask::$CLUSTER_PASSWORD" + echo "::add-mask::$CLUSTER_PASSWORD_URL" + - uses: actions/checkout@v3 - name: Install dependencies @@ -238,14 +268,21 @@ jobs: - name: Drop database if: ${{ always() }} + # The password reaches the script through the environment rather than + # being interpolated into the command: the generated value can contain + # any printable character, including ones the shell would act on. run: | - python resources/drop_db.py --user "${{ secrets.CLUSTER_USER }}" --password "${{ secrets.CLUSTER_PASSWORD }}" --host "${{ needs.setup-database.outputs.cluster-host }}" --port 3306 --database "${{ needs.setup-database.outputs.cluster-database }}" + python resources/drop_db.py --user "$CLUSTER_USER" --password "$CLUSTER_PASSWORD" --host "$CLUSTER_HOST" --port 3306 --database "$CLUSTER_DATABASE" env: PYTHONPATH: ${{ github.workspace }} + CLUSTER_USER: ${{ secrets.CLUSTER_USER }} + CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} + CLUSTER_HOST: ${{ needs.setup-database.outputs.cluster-host }} + CLUSTER_DATABASE: ${{ needs.setup-database.outputs.cluster-database }} - - name: Shutdown workspace + - name: Shutdown cluster if: ${{ always() }} run: | - curl -H "Accept: application/json" -H "Authorization: Bearer ${{ secrets.CLUSTER_API_KEY }}" -X DELETE "https://api.singlestore.com/v1/workspaces/${{ env.CLUSTER_ID }}" + curl -H "Accept: application/json" -H "Authorization: Bearer ${{ secrets.CLUSTER_API_KEY }}" -X DELETE "https://api.singlestore.com/v2/clusters/${{ env.CLUSTER_ID }}?force=true" env: CLUSTER_ID: ${{ needs.setup-database.outputs.cluster-id }} diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 688a2dc18..ab60501c3 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -29,7 +29,7 @@ jobs: - name: Initialize database id: initialize-database run: | - python resources/create_test_cluster.py --password="${{ secrets.CLUSTER_PASSWORD }}" --token="${{ secrets.CLUSTER_API_KEY }}" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" + python resources/create_test_cluster.py --token="${{ secrets.CLUSTER_API_KEY }}" --project="${{ vars.CLUSTER_PROJECT }}" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" env: PYTHONPATH: ${{ github.workspace }} @@ -37,6 +37,11 @@ jobs: cluster-id: ${{ steps.initialize-database.outputs.cluster-id }} cluster-host: ${{ steps.initialize-database.outputs.cluster-host }} cluster-database: ${{ steps.initialize-database.outputs.cluster-database }} + # POST /v2/clusters generates the admin password and reports it only on + # the create response, so it travels as a job output rather than living + # in secrets. Job outputs are not secrets: every job below re-masks it. + cluster-password: ${{ steps.initialize-database.outputs.cluster-password }} + cluster-password-url: ${{ steps.initialize-database.outputs.cluster-password-url }} smoke-test: @@ -100,6 +105,17 @@ jobs: buffered: 1 steps: + # ::add-mask:: does not cross job boundaries, so the generated admin + # password arrives here unmasked and has to be registered again before + # any step can echo it into the log. + - name: Mask cluster password + env: + CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} + CLUSTER_PASSWORD_URL: ${{ needs.setup-database.outputs.cluster-password-url }} + run: | + echo "::add-mask::$CLUSTER_PASSWORD" + echo "::add-mask::$CLUSTER_PASSWORD_URL" + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -118,7 +134,10 @@ jobs: run: pytest -v --pyargs singlestoredb.tests.test_basics env: PYTHONPATH: ${{ github.workspace }} - SINGLESTOREDB_URL: "${{ matrix.driver }}://${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" + # cluster-password-url, not cluster-password: the generated password + # is drawn from the full printable set and has to be percent-encoded + # to survive the userinfo half of the URL. + SINGLESTOREDB_URL: "${{ matrix.driver }}://${{ secrets.CLUSTER_USER }}:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" - name: Run tests if: ${{ matrix.driver == 'https' }} @@ -131,7 +150,7 @@ jobs: run: pytest -v -n 0 --pyargs singlestoredb.tests.test_basics env: PYTHONPATH: ${{ github.workspace }} - SINGLESTOREDB_URL: "${{ matrix.driver }}://${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}@${{ needs.setup-database.outputs.cluster-host }}:443/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" + SINGLESTOREDB_URL: "${{ matrix.driver }}://${{ secrets.CLUSTER_USER }}:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:443/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" shutdown-database: @@ -140,6 +159,15 @@ jobs: runs-on: ubuntu-latest steps: + # ::add-mask:: does not cross job boundaries; see the smoke-test job. + - name: Mask cluster password + env: + CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} + CLUSTER_PASSWORD_URL: ${{ needs.setup-database.outputs.cluster-password-url }} + run: | + echo "::add-mask::$CLUSTER_PASSWORD" + echo "::add-mask::$CLUSTER_PASSWORD_URL" + - uses: actions/checkout@v4 - name: Set up Python 3.11 @@ -156,14 +184,21 @@ jobs: - name: Drop database if: ${{ always() }} + # The password reaches the script through the environment rather than + # being interpolated into the command: the generated value can contain + # any printable character, including ones the shell would act on. run: | - python resources/drop_db.py --user "${{ secrets.CLUSTER_USER }}" --password "${{ secrets.CLUSTER_PASSWORD }}" --host "${{ needs.setup-database.outputs.cluster-host }}" --port 3306 --database "${{ needs.setup-database.outputs.cluster-database }}" + python resources/drop_db.py --user "$CLUSTER_USER" --password "$CLUSTER_PASSWORD" --host "$CLUSTER_HOST" --port 3306 --database "$CLUSTER_DATABASE" env: PYTHONPATH: ${{ github.workspace }} + CLUSTER_USER: ${{ secrets.CLUSTER_USER }} + CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} + CLUSTER_HOST: ${{ needs.setup-database.outputs.cluster-host }} + CLUSTER_DATABASE: ${{ needs.setup-database.outputs.cluster-database }} - - name: Shutdown workspace + - name: Shutdown cluster if: ${{ always() }} run: | - curl -H "Accept: application/json" -H "Authorization: Bearer ${{ secrets.CLUSTER_API_KEY }}" -X DELETE "https://api.singlestore.com/v1/workspaces/${{ env.CLUSTER_ID }}" + curl -H "Accept: application/json" -H "Authorization: Bearer ${{ secrets.CLUSTER_API_KEY }}" -X DELETE "https://api.singlestore.com/v2/clusters/${{ env.CLUSTER_ID }}?force=true" env: CLUSTER_ID: ${{ needs.setup-database.outputs.cluster-id }} diff --git a/resources/build_docs.py b/resources/build_docs.py index 628f3a1fd..5e54b184c 100755 --- a/resources/build_docs.py +++ b/resources/build_docs.py @@ -384,6 +384,11 @@ def apply_content_transformations(self, content: str, links: Dict[str, str]) -> # Change workspace.Stage to workspace.stage content = re.sub(r'>workspace\.Stage\.', r'>workspace.stage.', content) + # Change cluster.Stage to cluster.stage. Stage is re-exported from the + # v2 cluster module, so it is documented under both names for as long + # as management/workspace.py is still documented. + content = re.sub(r'>cluster\.Stage\.', r'>cluster.stage.', content) + # Fix class/method links content = re.sub( r'(]+>)?(\s*]*>\s*\s*)([\w\.]+)(\s*\s*)', diff --git a/resources/create_test_cluster.py b/resources/create_test_cluster.py index 186fadfa8..8c258eb02 100755 --- a/resources/create_test_cluster.py +++ b/resources/create_test_cluster.py @@ -2,45 +2,47 @@ # type: ignore from __future__ import annotations +import json import os import random import re -import secrets import subprocess import sys -import time import uuid from optparse import OptionParser +from urllib.parse import quote import singlestoredb as s2 # Handle command-line options -usage = 'usage: %prog [options] workspace-name' +usage = 'usage: %prog [options] cluster-name' parser = OptionParser(usage=usage) parser.add_option( '-r', '--region', default='AWS::*US East 1*', - help='region pattern or ID', -) -parser.add_option( - '-p', '--password', - default=secrets.token_urlsafe(20) + '-x&$', - help='admin password', + help='region pattern to deploy into, as provider::name ' + '(AWS::*US East 1*); * is a wildcard', ) parser.add_option( '-e', '--expires', default='4h', - help='timestamp when workspace should expire (4h)', + help='when the cluster should expire, as a timestamp or a ' + 'duration such as 4h (4h)', ) parser.add_option( '-s', '--size', default='S-00', - help='size of the workspace (S-00)', + help='size of the cluster (S-00)', ) parser.add_option( '-t', '--token', - help='API key for the workspace management API', + help='API key for the management API', +) +parser.add_option( + '--project', + help='ID or name of the project to deploy into; defaults to the ' + 'organization\'s STANDARD-edition project', ) parser.add_option( '--http-port', type='int', @@ -53,7 +55,7 @@ parser.add_option( '-o', '--output', default='env', choices=['env', 'github', 'json'], - help='report workspace information in the requested format: github, env, json', + help='report cluster information in the requested format: github, env, json', ) parser.add_option( '-d', '--database', @@ -67,111 +69,153 @@ sys.exit(1) if options.init_sql and not os.path.isfile(options.init_sql): - print('ERROR: Could not locate SQL file: {options.init_sql}', file=sys.stderr) + print(f'ERROR: Could not locate SQL file: {options.init_sql}', file=sys.stderr) sys.exit(1) -# Connect to workspace. This is still the deprecated v1 workspace-group -# grammar because the v1 test suite it sets up needs workspace groups; -# it gets ported to manage_clusters() when that suite goes. Pinned to v1 -# because manage_workspaces() otherwise follows the management.version option. -wm = s2.manage_workspaces(options.token or None, version='v1') +# Pin v2 explicitly rather than following the ambient management.version +# option: this script provisions clusters, which only exist in v2. +mgr = s2.manage_clusters(options.token or None, version='v2') + + +# Find a matching region. A v2 region is identified by the +# (provider, region_name) pair rather than by an ID, so the matched Region +# object is what gets handed to create_cluster. Candidates are shuffled to +# spread deployments across whichever regions match. +# +# The pattern is tried against both the display name and the provider region +# name -- 'US East 1' and 'us-east-1' -- so it does not matter which of the two +# a given listing puts in Region.name. +pattern = options.region.replace('*', '.*') +regions = list(mgr.regions) + + +def candidates(item): + """Return the names ``item`` can be matched by, most specific first.""" + for label in (item.name, item.region_name): + if label: + yield f'{item.provider}::{label}' if '::' in options.region else label + -# Find matching region -if '::' in options.region: - pattern = options.region.replace('*', '.*') - regions = wm.regions - for item in random.sample(regions, k=len(regions)): - region_name = '{}::{}'.format(item.provider, item.name) - if re.match(pattern, region_name): - options.region = item.id - break +region = None +for item in random.sample(regions, k=len(regions)): + if any(re.match(pattern, x) for x in candidates(item)): + region = item + break -if '::' in options.region: +if region is None: print( - 'ERROR: Could not find a region mating the pattern: ' - '{options.region}', file=sys.stderr, + 'ERROR: Could not find a region matching the pattern ' + f'{options.region}; the API reports: ' + + ', '.join(sorted(f'{x.provider}::{x.name}' for x in regions)), + file=sys.stderr, ) sys.exit(1) -# Create workspace group -wg_name = 'Python Client Testing' - -wgs = [x for x in wm.workspace_groups if x.name == wg_name] -if len(wgs) > 1: - print('ERROR: There is more than one workspace group with the specified name.') - sys.exit(1) -elif len(wgs) == 1: - wg = wgs[0] +# Choose a project. projectID is required by POST /v2/clusters and only +# auto-resolves for an organization with a single project, so pick the +# STANDARD-edition one when it was not named explicitly. +if options.project: + project_id = options.project else: - wg = wm.create_workspace_group( - wg_name, - region=options.region, - admin_password=options.password, - # firewall_ranges=requests.get('https://api.github.com/meta').json()['actions'], - firewall_ranges=['0.0.0.0/0'], - allow_all_traffic=True, - ) - -# Make sure the workspace group exists before continuing -timeout = 300 -while timeout > 0 and not [x for x in wm.workspace_groups if x.name == wg_name]: - time.sleep(10) - timeout -= 10 + projects = list(mgr.projects) + standard = [x for x in projects if x.edition == 'STANDARD'] + if not standard: + print( + 'ERROR: No STANDARD-edition project in this organization; pass ' + '--project with one of: ' + + ', '.join(f'{x.name} ({x.id}, {x.edition})' for x in projects), + file=sys.stderr, + ) + sys.exit(1) + project_id = standard[0].id + + +# A cluster name must match [a-z0-9]([a-z0-9-]*[a-z0-9])? and be 1-32 +# characters, so everything outside that alphabet becomes a hyphen, runs of +# hyphens collapse, and the result is truncated with any hyphen the cut +# exposes trimmed off again. +name = re.sub(r'[^a-z0-9]+', '-', args[0].lower()).strip('-')[:32].rstrip('-') +if not name: + print(f'ERROR: Cluster name is empty after cleaning: {args[0]}', file=sys.stderr) + sys.exit(1) -ws_name = re.sub(r'^-|-$', r'', re.sub(r'-+', r'-', re.sub(r'\s+', '-', args[0].lower()))) -ws = wg.create_workspace( - ws_name, +# wait_on_active covers ACTIVE, then the endpoint, then the firewall, so the +# cluster is actually reachable by the time this returns. +cluster = mgr.create_cluster( + name, + region=region, size=options.size, + firewall_ranges=['0.0.0.0/0'], + expires_at=options.expires, + project=project_id, wait_on_active=True, + wait_timeout=1200, ) -# Make sure the endpoint exists before continuing -timeout = 300 -while timeout > 0 and not ws.endpoint: - time.sleep(10) - ws.refresh() - timeout -= 10 - -if not ws.endpoint: - print('ERROR: Endpoint was never activated.') +# The API generates the admin password and reports it only on the create +# response -- there is no route that will hand it back later, and it is None +# after any refresh(). See item 9 of docs/management-api-audit.md: the API +# accepts an adminPassword on both POST and PATCH and ignores both, which is +# why this is read back rather than set. +password = cluster.admin_password +if not password: + print( + 'ERROR: cluster was created without a readable admin password', + file=sys.stderr, + ) sys.exit(1) - -# Extra pause for server to become available -time.sleep(10) +# The generated password is drawn from the full printable set -- one observed +# value was ``{:D}TK*[F3Ll}Ups2pNv`` -- so it cannot be dropped into the +# userinfo half of a connection URL as-is. Percent-encode everything, since +# the URL parser runs unquote_plus over the password +# (singlestoredb/connection.py:287); encoding ``+`` too is what keeps that from +# turning into a space. +password_url = quote(password, safe='') database = options.database if not database: database = 'TEMP_{}'.format(uuid.uuid4()).replace('-', '_') -host = ws.endpoint +host = cluster.endpoint if ':' in host: host, port = host.split(':', 1) port = int(port) else: port = 3306 -# Print workspace information +# Print cluster information if options.output == 'env': - print(f'CLUSTER_ID={ws.id}') + print(f'CLUSTER_ID={cluster.id}') print(f'CLUSTER_HOST={host}') print(f'CLUSTER_PORT={port}') print(f'CLUSTER_DATABASE={database}') + print(f'CLUSTER_PASSWORD={password}') + print(f'CLUSTER_PASSWORD_URL={password_url}') elif options.output == 'github': + # Register both forms with the runner before anything can log them. This + # only holds within this job; each job that consumes the outputs has to + # mask them again for itself. + print(f'::add-mask::{password}') + print(f'::add-mask::{password_url}') with open(os.environ['GITHUB_OUTPUT'], 'a') as output: - print(f'cluster-id={ws.id}', file=output) + print(f'cluster-id={cluster.id}', file=output) print(f'cluster-host={host}', file=output) print(f'cluster-port={port}', file=output) print(f'cluster-database={database}', file=output) + print(f'cluster-password={password}', file=output) + print(f'cluster-password-url={password_url}', file=output) elif options.output == 'json': print('{') - print(f' "cluster-id": "{ws.id}",') + print(f' "cluster-id": "{cluster.id}",') print(f' "cluster-host": "{host}",') - print(f' "cluster-port": {port}') - print(f' "cluster-database": {database}') + print(f' "cluster-port": {port},') + print(f' "cluster-database": "{database}",') + print(f' "cluster-password": {json.dumps(password)},') + print(f' "cluster-password-url": "{password_url}"') print('}') # Initialize the database @@ -179,7 +223,7 @@ init_db = [ os.path.join(os.path.dirname(__file__), 'init_db.py'), '--host', str(host), '--port', str(port), - '--user', 'admin', '--password', options.password, + '--user', 'admin', '--password', password, '--database', database, ] diff --git a/resources/drop_test_cluster.py b/resources/drop_test_cluster.py index 16ed7539d..6a7105dd7 100755 --- a/resources/drop_test_cluster.py +++ b/resources/drop_test_cluster.py @@ -2,7 +2,6 @@ # type: ignore from __future__ import annotations -import re import sys from optparse import OptionParser @@ -10,11 +9,11 @@ # Handle command-line options -usage = 'usage: %prog [options] workspace-id' +usage = 'usage: %prog [options] cluster-id' parser = OptionParser(usage=usage) parser.add_option( '-t', '--token', - help='API key for the workspace management API', + help='API key for the management API', ) (options, args) = parser.parse_args() @@ -23,33 +22,10 @@ sys.exit(1) -# Connect to workspace. This is still the deprecated v1 workspace-group -# grammar because the v1 test suite it sets up needs workspace groups; -# it gets ported to manage_clusters() when that suite goes. Pinned to v1 -# because manage_workspaces() otherwise follows the management.version option. -wm = s2.manage_workspaces(options.token or None, version='v1') +# Pin v2 explicitly rather than following the ambient management.version +# option: clusters only exist in v2. +mgr = s2.manage_clusters(options.token or None, version='v2') -wg_name = 'Python Client Testing' - -wgs = [x for x in wm.workspace_groups if x.name == wg_name] -if len(wgs) > 1: - print('ERROR: There is more than one workspace group with the specified name.') - sys.exit(1) -elif len(wgs) == 0: - print('ERROR: There is no workspace group with the specified name.') - sys.exit(1) -wg = wgs[0] - -ws_name = re.sub(r'^-|-$', r'', re.sub(r'-+', r'-', re.sub(r'\s+', '-', args[0].lower()))) - -wss = [x for x in wg.workspaces if x.name == ws_name] -if len(wss) > 1: - print('ERROR: There is more than one workspace with the specified name.') - sys.exit(1) -elif len(wss) == 0: - print('ERROR: There is no workspace with the specified name.') - sys.exit(1) -ws = wss[0] - -# Terminate workspace -ws.terminate() +# force=True so a cluster with connections still open goes away; this only +# ever runs against clusters this repo's CI created. +mgr.get_cluster(args[0]).terminate(force=True, wait_on_terminated=True) diff --git a/singlestoredb/management/v2/cluster.py b/singlestoredb/management/v2/cluster.py index 511b7cace..9c8765cf1 100644 --- a/singlestoredb/management/v2/cluster.py +++ b/singlestoredb/management/v2/cluster.py @@ -611,7 +611,18 @@ def update( allow_all_traffic : bool, optional Allow all traffic to the cluster admin_password : str, optional - Admin password for the cluster + Admin password for the cluster. + + .. warning:: This is ignored, exactly as it is on + ``POST /v2/clusters``. ``PATCH /v2/clusters/{id}`` accepts the + field and does not honor it: a live probe found the patched value + refused with ``1045: Access denied`` while the password the + original create generated kept working. So the admin password + cannot be set after the fact either -- the only value that + authenticates is the generated one + :attr:`Cluster.admin_password` carried on the create response. + See item 9 of ``docs/management-api-audit.md``. The field is + still sent in case the API starts honoring it. expires_at : str, optional Timestamp of when the cluster will expire. Expiration time can be specified as a timestamp or a duration. From 25b8117a7ea22d9be8a4885cac17ad4e392d237e Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 17 Sep 2026 09:59:10 -0400 Subject: [PATCH 02/12] Parse Go time.Time strings in management to_datetime The v2 clusters API returns expiresAt in Go's time.Time.String() format -- '2026-09-17 14:42:41.445984 +0000 UTC' -- while every other timestamp is RFC 3339. to_datetime split the string on '.' to pad the fractional seconds, produced '445984 +0000 UTC' as the fraction, and datetime_fromisoformat then returned None. Cluster.expires_at read as None on a cluster that had an expiry set, and SHOW CLUSTERS printed a blank expiry column. Normalize both shapes before parsing: pad or truncate the fraction to six digits, keep a numeric offset, and drop the trailing zone abbreviation and Go's monotonic-clock reading. An offset-aware result is converted to UTC and returned naive, matching what the RFC 3339 path already produced. Co-Authored-By: Claude Opus 5 --- singlestoredb/management/utils.py | 96 ++++++++++++++++---- singlestoredb/tests/test_management_utils.py | 72 +++++++++++++++ 2 files changed, 152 insertions(+), 16 deletions(-) diff --git a/singlestoredb/management/utils.py b/singlestoredb/management/utils.py index bfdcc8658..e00844dd3 100644 --- a/singlestoredb/management/utils.py +++ b/singlestoredb/management/utils.py @@ -408,6 +408,80 @@ def enable_http_tracing() -> None: requests_log.propagate = True +#: A Go ``time.Time`` rendered by its ``String()`` method: +#: ``2026-09-17 14:42:41.445984 +0000 UTC``. ``GET /v2/clusters/{id}`` reports +#: ``expiresAt`` in this shape while every other timestamp it returns is +#: RFC 3339, and the trailing zone name is not ISO 8601, so the whole value +#: fails to parse and the expiration silently reads as unset. The zone name and +#: the monotonic-clock reading Go appends to some values are both optional. +_GO_DATETIME_RE = re.compile( + r'^(?P\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?)' + r'(?:\s*(?P[+-]\d{2}:?\d{2}))?' + r'(?:\s+(?P[A-Za-z]\S*))?' + r'(?:\s+m=\S+)?$', +) + + +def _normalize_datetime(obj: str) -> str: + """ + Return ``obj`` as something :func:`converters.datetime_fromisoformat` reads. + + Handles the two shapes the management API returns -- RFC 3339 and the Go + ``time.Time.String()`` form -- by reducing both to a bare ISO 8601 + timestamp plus an optional numeric offset. Fractional seconds are padded to + microseconds, since Go trims trailing zeros. + + Parameters + ---------- + obj : str + Timestamp as reported by the API + + Returns + ------- + str + + """ + match = _GO_DATETIME_RE.match(obj.strip()) + if match is None: + # Not a shape this recognizes; hand it over untouched so the converter + # gets its usual chance to make sense of it. + return obj.replace('Z', '') + + stamp = match.group('stamp') + + # Fix datetimes with truncated zeros + if '.' in stamp: + stamp, micros = stamp.split('.', 1) + micros = micros[:6] + '0' * (6 - len(micros)) + stamp = stamp + '.' + micros + + return stamp + (match.group('offset') or '') + + +def _as_naive_utc(obj: datetime.datetime) -> datetime.datetime: + """ + Return ``obj`` as a naive UTC datetime. + + An RFC 3339 timestamp loses its ``Z`` before it is parsed, so it arrives + here naive and already meaning UTC. A value carrying a numeric offset is + shifted onto UTC and stripped, so both shapes end up on the one convention + -- otherwise two timestamps read off the same object could not be compared. + + Parameters + ---------- + obj : datetime.datetime + Parsed timestamp, with or without a timezone + + Returns + ------- + datetime.datetime + + """ + if obj.tzinfo is None: + return obj + return obj.astimezone(datetime.timezone.utc).replace(tzinfo=None) + + def to_datetime( obj: Optional[Union[str, datetime.datetime]], ) -> Optional[datetime.datetime]: @@ -418,18 +492,14 @@ def to_datetime( return obj if obj == '0001-01-01T00:00:00Z': return None - obj = obj.replace('Z', '') - # Fix datetimes with truncated zeros - if '.' in obj: - obj, micros = obj.split('.', 1) - micros = micros + '0' * (6 - len(micros)) - obj = obj + '.' + micros - out = converters.datetime_fromisoformat(obj) + out = converters.datetime_fromisoformat(_normalize_datetime(obj)) if isinstance(out, str): return None if isinstance(out, datetime.date) and not isinstance(out, datetime.datetime): return datetime.datetime(out.year, out.month, out.day) - return out + if out is None: + return None + return _as_naive_utc(out) def to_datetime_strict( @@ -442,20 +512,14 @@ def to_datetime_strict( return obj if obj == '0001-01-01T00:00:00Z': raise ValueError('not possible to convert 0001-01-01T00:00:00Z to datetime') - obj = obj.replace('Z', '') - # Fix datetimes with truncated zeros - if '.' in obj: - obj, micros = obj.split('.', 1) - micros = micros + '0' * (6 - len(micros)) - obj = obj + '.' + micros - out = converters.datetime_fromisoformat(obj) + out = converters.datetime_fromisoformat(_normalize_datetime(obj)) if not out: raise TypeError('not possible to convert None to datetime') if isinstance(out, str): raise ValueError('value cannot be str') if isinstance(out, datetime.date) and not isinstance(out, datetime.datetime): return datetime.datetime(out.year, out.month, out.day) - return out + return _as_naive_utc(out) def from_datetime( diff --git a/singlestoredb/tests/test_management_utils.py b/singlestoredb/tests/test_management_utils.py index b7b25f1e5..d2531bb9a 100644 --- a/singlestoredb/tests/test_management_utils.py +++ b/singlestoredb/tests/test_management_utils.py @@ -18,6 +18,8 @@ from singlestoredb.exceptions import ManagementError from singlestoredb.management.utils import normalize_remote_path +from singlestoredb.management.utils import to_datetime +from singlestoredb.management.utils import to_datetime_strict from singlestoredb.tests.utils import counting_file_space from singlestoredb.tests.utils import counting_stage @@ -1815,5 +1817,75 @@ def test_a_since_that_is_not_a_date_is_rejected(self): self.mod.parse_since('last tuesday') +class TestToDatetime(unittest.TestCase): + """ + ``to_datetime`` has to read both timestamp shapes the API returns. + + Most fields come back as RFC 3339, but ``GET /v2/clusters/{id}`` reports + ``expiresAt`` as a Go ``time.Time.String()`` rendering -- verified live: + ``2026-09-17 14:42:41.445984 +0000 UTC`` against a ``createdAt`` of + ``2026-09-17T13:42:41.493848Z`` on the same cluster. The trailing zone name + is not ISO 8601, and parsing it used to fail into ``None``, which reads as + "this cluster never expires". + """ + + def test_rfc_3339(self): + out = to_datetime('2026-09-17T13:42:41.493848Z') + self.assertEqual(out, datetime.datetime(2026, 9, 17, 13, 42, 41, 493848)) + + def test_go_time_string(self): + out = to_datetime('2026-09-17 14:42:41.445984 +0000 UTC') + self.assertEqual(out, datetime.datetime(2026, 9, 17, 14, 42, 41, 445984)) + + def test_go_time_string_with_truncated_fraction(self): + # Go trims trailing zeros, so the fraction is not always 6 digits. + out = to_datetime('2026-09-17 14:42:41.4 +0000 UTC') + self.assertEqual(out, datetime.datetime(2026, 9, 17, 14, 42, 41, 400000)) + + def test_go_time_string_with_monotonic_reading(self): + out = to_datetime( + '2026-09-17 14:42:41.445984 +0000 UTC m=+0.000000001', + ) + self.assertEqual(out, datetime.datetime(2026, 9, 17, 14, 42, 41, 445984)) + + def test_offset_is_applied_and_dropped(self): + # Shifted onto UTC and left naive, matching the RFC 3339 values, so two + # timestamps read off one object can be compared. + out = to_datetime('2026-09-17 09:42:41 -0500 EST') + self.assertEqual(out, datetime.datetime(2026, 9, 17, 14, 42, 41)) + self.assertIsNone(out.tzinfo) + + def test_both_shapes_subtract(self): + created = to_datetime('2026-09-17T13:42:41.493848Z') + expires = to_datetime('2026-09-17 14:42:41.445984 +0000 UTC') + self.assertAlmostEqual( + (expires - created).total_seconds(), 3600, delta=1, + ) + + def test_date_only(self): + out = to_datetime('2026-09-17') + self.assertEqual(out, datetime.datetime(2026, 9, 17)) + + def test_zero_sentinel_and_unparseable_are_none(self): + self.assertIsNone(to_datetime('0001-01-01T00:00:00Z')) + self.assertIsNone(to_datetime(None)) + self.assertIsNone(to_datetime('')) + self.assertIsNone(to_datetime('not a date')) + + def test_datetime_passes_through(self): + given = datetime.datetime(2026, 9, 17, 13, 42, 41) + self.assertIs(to_datetime(given), given) + + def test_strict_reads_the_go_shape_too(self): + out = to_datetime_strict('2026-09-17 14:42:41.445984 +0000 UTC') + self.assertEqual(out, datetime.datetime(2026, 9, 17, 14, 42, 41, 445984)) + + def test_strict_still_raises_on_nothing(self): + with self.assertRaises(TypeError): + to_datetime_strict(None) + with self.assertRaises(ValueError): + to_datetime_strict('0001-01-01T00:00:00Z') + + if __name__ == '__main__': unittest.main() From 9d3abf3bf997bd3ab85bc8d072f516988fafb798 Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 17 Sep 2026 10:03:37 -0400 Subject: [PATCH 03/12] Fix the CI cluster user and project in the workflows A v2 cluster has exactly one user, admin, and no route creates another, so secrets.CLUSTER_USER could only ever hold that one value. Drop the secret and name admin directly. Name the deployment project in the workflow too, rather than reading it from vars.CLUSTER_PROJECT: the target is then visible next to the create call and an unset repository variable cannot quietly change where CI deploys. create_cluster resolves a project name against GET /v2/projects and raises with the org's project list if it matches none, so a renamed project fails loudly at setup. Co-Authored-By: Claude Opus 5 --- .github/workflows/publish.yml | 11 +++++++---- .github/workflows/smoke-test.yml | 13 ++++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index eeeb663e1..f39dc79ec 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -49,8 +49,12 @@ jobs: - name: Initialize database id: initialize-database + # A new cluster has exactly one user, admin, which is why that name is + # fixed everywhere below. The project is named here rather than read + # from a repo variable so the deployment target is visible in the + # workflow and does not depend on repository settings. run: | - python resources/create_test_cluster.py --token="${{ secrets.CLUSTER_API_KEY }}" --project="${{ vars.CLUSTER_PROJECT }}" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" + python resources/create_test_cluster.py --token="${{ secrets.CLUSTER_API_KEY }}" --project="Standard Project" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" env: PYTHONPATH: ${{ github.workspace }} @@ -139,7 +143,7 @@ jobs: # both the userinfo half of the URL and the single-quoted shell word # cibuildwheel evaluates. Percent-encoding leaves only unreserved # characters and %, which are inert in both. - CIBW_ENVIRONMENT: "SINGLESTOREDB_URL='mysql://${{ secrets.CLUSTER_USER }}:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=0'" + CIBW_ENVIRONMENT: "SINGLESTOREDB_URL='mysql://admin:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=0'" PYTHONPATH: ${{ github.workspace }} # - name: Build conda @@ -272,10 +276,9 @@ jobs: # being interpolated into the command: the generated value can contain # any printable character, including ones the shell would act on. run: | - python resources/drop_db.py --user "$CLUSTER_USER" --password "$CLUSTER_PASSWORD" --host "$CLUSTER_HOST" --port 3306 --database "$CLUSTER_DATABASE" + python resources/drop_db.py --user admin --password "$CLUSTER_PASSWORD" --host "$CLUSTER_HOST" --port 3306 --database "$CLUSTER_DATABASE" env: PYTHONPATH: ${{ github.workspace }} - CLUSTER_USER: ${{ secrets.CLUSTER_USER }} CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} CLUSTER_HOST: ${{ needs.setup-database.outputs.cluster-host }} CLUSTER_DATABASE: ${{ needs.setup-database.outputs.cluster-database }} diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index ab60501c3..50a35b5e3 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -28,8 +28,12 @@ jobs: - name: Initialize database id: initialize-database + # A new cluster has exactly one user, admin, which is why that name is + # fixed everywhere below. The project is named here rather than read + # from a repo variable so the deployment target is visible in the + # workflow and does not depend on repository settings. run: | - python resources/create_test_cluster.py --token="${{ secrets.CLUSTER_API_KEY }}" --project="${{ vars.CLUSTER_PROJECT }}" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" + python resources/create_test_cluster.py --token="${{ secrets.CLUSTER_API_KEY }}" --project="Standard Project" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" env: PYTHONPATH: ${{ github.workspace }} @@ -137,7 +141,7 @@ jobs: # cluster-password-url, not cluster-password: the generated password # is drawn from the full printable set and has to be percent-encoded # to survive the userinfo half of the URL. - SINGLESTOREDB_URL: "${{ matrix.driver }}://${{ secrets.CLUSTER_USER }}:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" + SINGLESTOREDB_URL: "${{ matrix.driver }}://admin:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" - name: Run tests if: ${{ matrix.driver == 'https' }} @@ -150,7 +154,7 @@ jobs: run: pytest -v -n 0 --pyargs singlestoredb.tests.test_basics env: PYTHONPATH: ${{ github.workspace }} - SINGLESTOREDB_URL: "${{ matrix.driver }}://${{ secrets.CLUSTER_USER }}:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:443/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" + SINGLESTOREDB_URL: "${{ matrix.driver }}://admin:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:443/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" shutdown-database: @@ -188,10 +192,9 @@ jobs: # being interpolated into the command: the generated value can contain # any printable character, including ones the shell would act on. run: | - python resources/drop_db.py --user "$CLUSTER_USER" --password "$CLUSTER_PASSWORD" --host "$CLUSTER_HOST" --port 3306 --database "$CLUSTER_DATABASE" + python resources/drop_db.py --user admin --password "$CLUSTER_PASSWORD" --host "$CLUSTER_HOST" --port 3306 --database "$CLUSTER_DATABASE" env: PYTHONPATH: ${{ github.workspace }} - CLUSTER_USER: ${{ secrets.CLUSTER_USER }} CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} CLUSTER_HOST: ${{ needs.setup-database.outputs.cluster-host }} CLUSTER_DATABASE: ${{ needs.setup-database.outputs.cluster-database }} From 6c22959dd6c1b21cc2216f39a42a4f69df973b9c Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 17 Sep 2026 10:10:10 -0400 Subject: [PATCH 04/12] Add actionlint to pre-commit and fix what it reports The workflow files had no linter, so nothing checked expression syntax, needs.*.outputs.* references or the shell inside run: blocks. actionlint covers all three and ships as a pip wrapper, so the hook needs no Go toolchain or Docker. It reported 17 pre-existing problems, all fixed here so the hook lands green: - actions/checkout@v3, actions/setup-python@v4 and docker/setup-qemu-action@v2 run on node runtimes GitHub is retiring. Bumped to v4, v5 and v3, which is what the newer workflows already pin. - publish.yml named a step after ${{ matrix.python-version }} in a job whose matrix defines only os, so the name rendered with nothing after it. That job pins 3.10 as the host interpreter for cibuildwheel, so the reference was never going to resolve; dropped it. - code-check.yml left $GITHUB_OUTPUT unquoted on six redirects. Quoted. The one remaining sed is prefixing every line, which parameter expansion cannot do, so SC2001 is suppressed in place with a reason. Co-Authored-By: Claude Opus 5 --- .github/workflows/code-check.yml | 16 +++++++++------- .github/workflows/coverage.yml | 4 ++-- .github/workflows/pre-commit.yml | 4 ++-- .github/workflows/publish.yml | 16 +++++++++------- .pre-commit-config.yaml | 4 ++++ 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index 35c2d535d..0195ae88f 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -34,7 +34,7 @@ jobs: fetch-depth: 2 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" cache: "pip" @@ -78,8 +78,8 @@ jobs: COMMIT_MSG=$(git log -1 --format='%s' HEAD) if [[ "$COMMIT_MSG" =~ ^Prepare\ for\ v[0-9]+\.[0-9]+\.[0-9]+\ release$ ]]; then echo "🚀 Release preparation commit detected: $COMMIT_MSG" - echo "changes-detected=true" >> $GITHUB_OUTPUT - echo "changed-directories=release" >> $GITHUB_OUTPUT + echo "changes-detected=true" >> "$GITHUB_OUTPUT" + echo "changed-directories=release" >> "$GITHUB_OUTPUT" echo "" echo "🎯 RESULT: Full test suite will run for release preparation" exit 0 @@ -98,6 +98,8 @@ jobs: if [ -n "$CHANGED_FILES" ]; then echo "✅ Changes detected in: $DIR" echo "Files changed:" + # shellcheck disable=SC2001 # prefixing every line, which + # ${var//search/replace} cannot do echo "$CHANGED_FILES" | sed 's/^/ - /' CHANGES_FOUND=true if [ -z "$CHANGED_DIRS" ]; then @@ -115,13 +117,13 @@ jobs: # Set outputs if [ "$CHANGES_FOUND" = true ]; then - echo "changes-detected=true" >> $GITHUB_OUTPUT - echo "changed-directories=$CHANGED_DIRS" >> $GITHUB_OUTPUT + echo "changes-detected=true" >> "$GITHUB_OUTPUT" + echo "changed-directories=$CHANGED_DIRS" >> "$GITHUB_OUTPUT" echo "" echo "🎯 RESULT: Changes detected in monitored directories" else - echo "changes-detected=false" >> $GITHUB_OUTPUT - echo "changed-directories=" >> $GITHUB_OUTPUT + echo "changes-detected=false" >> "$GITHUB_OUTPUT" + echo "changed-directories=" >> "$GITHUB_OUTPUT" echo "" echo "🎯 RESULT: No changes in monitored directories" fi diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ef07ecbf0..023bdb44f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" cache: "pip" @@ -109,7 +109,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" cache: "pip" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index a9217a93f..6b3d87393 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -16,10 +16,10 @@ jobs: - "3.13" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f39dc79ec..fa88d195b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install dependencies run: | @@ -93,10 +93,12 @@ jobs: echo "::add-mask::$CLUSTER_PASSWORD" echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + # This job's matrix varies only over os; cibuildwheel supplies its own + # interpreters, so 3.10 here is just the host Python that drives it. + - name: Set up Python + uses: actions/setup-python@v5 with: python-version: "3.10" cache: "pip" @@ -120,7 +122,7 @@ jobs: - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: all @@ -200,7 +202,7 @@ jobs: url: https://pypi.org/p/singlestoredb steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download Linux wheels and sdist uses: actions/download-artifact@v4 @@ -262,7 +264,7 @@ jobs: echo "::add-mask::$CLUSTER_PASSWORD" echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install dependencies run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d4c60017..b190dd101 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,3 +41,7 @@ repos: hooks: - id: mypy additional_dependencies: [types-requests] +- repo: https://github.com/Mateusz-Grzelinski/actionlint-py + rev: v1.7.7.23 + hooks: + - id: actionlint From 8828326375d0898fc34df663bf990b954bb404f5 Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 17 Sep 2026 10:19:06 -0400 Subject: [PATCH 05/12] Move every action pin off the Node.js 20 runtime The previous commit bumped checkout to v4 and setup-python to v5 to satisfy actionlint, but both of those majors still declare node20, so the runners kept reporting them as forced onto node24. actionlint 1.7.7 does not know about that deprecation, so it had nothing to say. Pin the lowest major of each action that declares node24, taken from action.yml at the tag rather than from the release notes: checkout v5, setup-python v6, upload-artifact v6, setup-qemu-action v4, and download-artifact v7 -- v5 and v6 of download-artifact are still node20, so it is the one that has to skip further ahead. cibuildwheel and gh-action-pypi-publish are composite actions and never had a node runtime to move. Choosing the lowest node24 major rather than the newest keeps the behavioural change to a minimum; there is no other reason to jump to checkout v7 today. Co-Authored-By: Claude Opus 5 --- .github/workflows/code-check.yml | 4 ++-- .github/workflows/coverage.yml | 8 ++++---- .github/workflows/fusion-docs.yml | 4 ++-- .github/workflows/pre-commit.yml | 4 ++-- .github/workflows/publish.yml | 22 +++++++++++----------- .github/workflows/smoke-test.yml | 12 ++++++------ 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index 0195ae88f..75114ef36 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -29,12 +29,12 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 2 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" cache: "pip" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 023bdb44f..e2bd2a4a1 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -22,10 +22,10 @@ jobs: ROOT_PASSWORD: "root" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" cache: "pip" @@ -106,10 +106,10 @@ jobs: ROOT_PASSWORD: "root" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" cache: "pip" diff --git a/.github/workflows/fusion-docs.yml b/.github/workflows/fusion-docs.yml index 75a74ffa6..3a40c15c4 100644 --- a/.github/workflows/fusion-docs.yml +++ b/.github/workflows/fusion-docs.yml @@ -14,10 +14,10 @@ jobs: actions: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.11 cache: "pip" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 6b3d87393..10181d541 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -16,10 +16,10 @@ jobs: - "3.13" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index fa88d195b..3a1f5b583 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Install dependencies run: | @@ -93,12 +93,12 @@ jobs: echo "::add-mask::$CLUSTER_PASSWORD" echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 # This job's matrix varies only over os; cibuildwheel supplies its own # interpreters, so 3.10 here is just the host Python that drives it. - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" cache: "pip" @@ -122,7 +122,7 @@ jobs: - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 with: platforms: all @@ -174,14 +174,14 @@ jobs: mv ./wheelhouse/*.whl ./dist/. - name: Archive source dist and wheel - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: artifacts-${{ runner.os }} path: dist retention-days: 2 # - name: Archive conda -# uses: actions/upload-artifact@v4 +# uses: actions/upload-artifact@v6 # with: # name: conda-${{ matrix.os }} # path: ./conda-bld @@ -202,22 +202,22 @@ jobs: url: https://pypi.org/p/singlestoredb steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Download Linux wheels and sdist - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7 with: name: artifacts-Linux path: dist - name: Download Windows wheels and sdist - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7 with: name: artifacts-Windows path: dist - name: Download Mac wheels and sdist - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7 with: name: artifacts-macOS path: dist @@ -264,7 +264,7 @@ jobs: echo "::add-mask::$CLUSTER_PASSWORD" echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Install dependencies run: | diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 50a35b5e3..34e0f0391 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -12,10 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.11 cache: "pip" @@ -120,10 +120,10 @@ jobs: echo "::add-mask::$CLUSTER_PASSWORD" echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: "pip" @@ -172,10 +172,10 @@ jobs: echo "::add-mask::$CLUSTER_PASSWORD" echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.11 cache: "pip" From 0f16a6d6a6d1ae998f5fa25a50b58e28998d54fe Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 17 Sep 2026 10:26:14 -0400 Subject: [PATCH 06/12] Pin every action at its newest major Follows the node24 move rather than stopping at the lowest major that cleared it: checkout v7, setup-python v7, upload-artifact v7, download-artifact v8, setup-qemu-action v4. One deprecation cycle instead of two. Checked the breaking changes in the majors this skips over: - setup-python dropped its default Python version, so a step that names neither python-version nor python-version-file now fails. All nine call sites name python-version, so none are affected. - download-artifact v5 changed the path layout for single downloads by ID. publish.yml downloads by name, so it is untouched. - download-artifact v8 stopped unzipping unconditionally, checking Content-Type first, and now errors on a hash mismatch. The artifacts here are ordinary zipped directory uploads, so both apply harmlessly. cibuildwheel and gh-action-pypi-publish stay where they are: both are composite actions, so neither was part of the node problem, and moving cibuildwheel two majors is a build-behaviour change that does not belong in this PR. The artifact pins are the ones with no coverage here -- publish.yml runs only on a tag or a release, so they are first exercised by the next release build. Co-Authored-By: Claude Opus 5 --- .github/workflows/code-check.yml | 4 ++-- .github/workflows/coverage.yml | 8 ++++---- .github/workflows/fusion-docs.yml | 4 ++-- .github/workflows/pre-commit.yml | 4 ++-- .github/workflows/publish.yml | 20 ++++++++++---------- .github/workflows/smoke-test.yml | 12 ++++++------ 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index 75114ef36..36b44259a 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -29,12 +29,12 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: fetch-depth: 2 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.10" cache: "pip" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index e2bd2a4a1..182a4247c 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -22,10 +22,10 @@ jobs: ROOT_PASSWORD: "root" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.10" cache: "pip" @@ -106,10 +106,10 @@ jobs: ROOT_PASSWORD: "root" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.10" cache: "pip" diff --git a/.github/workflows/fusion-docs.yml b/.github/workflows/fusion-docs.yml index 3a40c15c4..bfbb35d01 100644 --- a/.github/workflows/fusion-docs.yml +++ b/.github/workflows/fusion-docs.yml @@ -14,10 +14,10 @@ jobs: actions: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Set up Python 3.11 - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: 3.11 cache: "pip" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 10181d541..aa6145076 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -16,10 +16,10 @@ jobs: - "3.13" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3a1f5b583..5ee454e0f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Install dependencies run: | @@ -93,12 +93,12 @@ jobs: echo "::add-mask::$CLUSTER_PASSWORD" echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 # This job's matrix varies only over os; cibuildwheel supplies its own # interpreters, so 3.10 here is just the host Python that drives it. - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.10" cache: "pip" @@ -174,14 +174,14 @@ jobs: mv ./wheelhouse/*.whl ./dist/. - name: Archive source dist and wheel - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: artifacts-${{ runner.os }} path: dist retention-days: 2 # - name: Archive conda -# uses: actions/upload-artifact@v6 +# uses: actions/upload-artifact@v7 # with: # name: conda-${{ matrix.os }} # path: ./conda-bld @@ -202,22 +202,22 @@ jobs: url: https://pypi.org/p/singlestoredb steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Download Linux wheels and sdist - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: name: artifacts-Linux path: dist - name: Download Windows wheels and sdist - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: name: artifacts-Windows path: dist - name: Download Mac wheels and sdist - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: name: artifacts-macOS path: dist @@ -264,7 +264,7 @@ jobs: echo "::add-mask::$CLUSTER_PASSWORD" echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Install dependencies run: | diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 34e0f0391..4764f1458 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -12,10 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Set up Python 3.11 - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: 3.11 cache: "pip" @@ -120,10 +120,10 @@ jobs: echo "::add-mask::$CLUSTER_PASSWORD" echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} cache: "pip" @@ -172,10 +172,10 @@ jobs: echo "::add-mask::$CLUSTER_PASSWORD" echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Set up Python 3.11 - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: 3.11 cache: "pip" From 7a0f3ba56afc5424c8c9fb9e3228a7c82d2d0017 Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 17 Sep 2026 10:31:30 -0400 Subject: [PATCH 07/12] Emit the timezone offset with a colon in to_datetime Go renders the offset without a separator -- the expiresAt values come back as '+0000' -- and datetime.fromisoformat only accepts that spelling on Python 3.11 and later. On 3.9 and 3.10 it raised, the converter returned the string unchanged, and to_datetime turned that into None: exactly the silent unset expiration the Go-format handling was added to fix, just on the interpreters the previous commit did not cover. Normalize the offset to +00:00. Verified that every shape the normalizer emits parses on 3.8, 3.10 and 3.11. The new test asserts on the normalized string rather than on a parsed datetime. The six existing tests were correct and still passed on 3.11, which is how this reached CI; a string comparison fails the same way on every version. Co-Authored-By: Claude Opus 5 --- singlestoredb/management/utils.py | 9 ++++++++- singlestoredb/tests/test_management_utils.py | 21 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/singlestoredb/management/utils.py b/singlestoredb/management/utils.py index e00844dd3..781452f13 100644 --- a/singlestoredb/management/utils.py +++ b/singlestoredb/management/utils.py @@ -455,7 +455,14 @@ def _normalize_datetime(obj: str) -> str: micros = micros[:6] + '0' * (6 - len(micros)) stamp = stamp + '.' + micros - return stamp + (match.group('offset') or '') + # Go writes the offset without a separator (+0000). Only Python 3.11 and + # later accept that spelling; 3.9 and 3.10 want +00:00, so always emit the + # colon. + offset = match.group('offset') or '' + if offset and ':' not in offset: + offset = offset[:3] + ':' + offset[3:] + + return stamp + offset def _as_naive_utc(obj: datetime.datetime) -> datetime.datetime: diff --git a/singlestoredb/tests/test_management_utils.py b/singlestoredb/tests/test_management_utils.py index d2531bb9a..ba4f6e745 100644 --- a/singlestoredb/tests/test_management_utils.py +++ b/singlestoredb/tests/test_management_utils.py @@ -17,6 +17,7 @@ from unittest.mock import patch from singlestoredb.exceptions import ManagementError +from singlestoredb.management.utils import _normalize_datetime from singlestoredb.management.utils import normalize_remote_path from singlestoredb.management.utils import to_datetime from singlestoredb.management.utils import to_datetime_strict @@ -1837,6 +1838,26 @@ def test_go_time_string(self): out = to_datetime('2026-09-17 14:42:41.445984 +0000 UTC') self.assertEqual(out, datetime.datetime(2026, 9, 17, 14, 42, 41, 445984)) + def test_offset_is_normalized_to_include_a_colon(self): + # Go writes +0000; datetime.fromisoformat only accepts that spelling on + # 3.11 and later, so the normalizer has to insert the colon itself. This + # asserts on the normalized string rather than on a parsed result + # because the parsed result is only wrong on 3.9 and 3.10, which would + # leave the failure invisible to anyone testing on a newer interpreter. + self.assertEqual( + _normalize_datetime('2026-09-17 14:42:41.445984 +0000 UTC'), + '2026-09-17 14:42:41.445984+00:00', + ) + self.assertEqual( + _normalize_datetime('2026-09-17 09:42:41 +0530 IST'), + '2026-09-17 09:42:41+05:30', + ) + # An offset that already carries a colon is left as it is. + self.assertEqual( + _normalize_datetime('2026-09-17 09:42:41 +05:30 IST'), + '2026-09-17 09:42:41+05:30', + ) + def test_go_time_string_with_truncated_fraction(self): # Go trims trailing zeros, so the fraction is not always 6 digits. out = to_datetime('2026-09-17 14:42:41.4 +0000 UTC') From 855a593cc664328803d55ee6001ea37076c70b14 Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 17 Sep 2026 11:10:57 -0400 Subject: [PATCH 08/12] Reset the admin password instead of passing it between jobs POST /v2/clusters generates its own admin password and ignores any that is sent, so create_test_cluster.py was reading the generated one back off the create response and reporting it as a job output. That cannot work: the value has to be masked, and the runner drops any output whose value matches a mask -- "Skip output 'cluster-password' since it may contain secret" -- so the test jobs received an empty password and failed with "1045: Access denied for user 'admin'@... (using password: NO)". Take a --password instead and hand it to the new cluster over SQL once it is active, so every job reads the credential from secrets.CLUSTER_PASSWORD and nothing crosses a job boundary. ALTER USER is the statement that works; SET PASSWORD wants a 41-digit hash and rejects a literal. The percent-encoded variant and the per-job ::add-mask:: steps both go away with the output. Co-Authored-By: Claude Opus 5 --- .github/workflows/publish.yml | 50 +++++++---------------- .github/workflows/smoke-test.yml | 50 ++++++++--------------- resources/create_test_cluster.py | 70 +++++++++++++++++++------------- 3 files changed, 72 insertions(+), 98 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5ee454e0f..30a4265a7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -53,8 +53,15 @@ jobs: # fixed everywhere below. The project is named here rather than read # from a repo variable so the deployment target is visible in the # workflow and does not depend on repository settings. + # + # POST /v2/clusters generates its own admin password and ignores any + # that is sent, so the script resets it to CLUSTER_PASSWORD over SQL + # once the cluster is up. That keeps the credential a secret the runner + # masks everywhere, instead of a job output: the runner refuses to write + # an output whose value is masked, so a generated password could not + # reach these jobs at all. run: | - python resources/create_test_cluster.py --token="${{ secrets.CLUSTER_API_KEY }}" --project="Standard Project" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" + python resources/create_test_cluster.py --password="${{ secrets.CLUSTER_PASSWORD }}" --token="${{ secrets.CLUSTER_API_KEY }}" --project="Standard Project" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" env: PYTHONPATH: ${{ github.workspace }} @@ -62,11 +69,6 @@ jobs: cluster-id: ${{ steps.initialize-database.outputs.cluster-id }} cluster-host: ${{ steps.initialize-database.outputs.cluster-host }} cluster-database: ${{ steps.initialize-database.outputs.cluster-database }} - # POST /v2/clusters generates the admin password and reports it only on - # the create response, so it travels as a job output rather than living - # in secrets. Job outputs are not secrets: every job below re-masks it. - cluster-password: ${{ steps.initialize-database.outputs.cluster-password }} - cluster-password-url: ${{ steps.initialize-database.outputs.cluster-password-url }} build-and-test: needs: setup-database @@ -82,17 +84,6 @@ jobs: - windows-2022 steps: - # ::add-mask:: does not cross job boundaries, so the generated admin - # password arrives here unmasked and has to be registered again before - # any step can echo it into the log. - - name: Mask cluster password - env: - CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} - CLUSTER_PASSWORD_URL: ${{ needs.setup-database.outputs.cluster-password-url }} - run: | - echo "::add-mask::$CLUSTER_PASSWORD" - echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v7 # This job's matrix varies only over os; cibuildwheel supplies its own @@ -140,12 +131,10 @@ jobs: # points --pyargs at the workspace, so that pyproject is the inifile # here. Without the plugin pytest exits on the unknown arguments. CIBW_TEST_REQUIRES: "pytest pytest-xdist" - # cluster-password-url, not cluster-password: the generated password - # is drawn from the full printable set, and this value has to survive - # both the userinfo half of the URL and the single-quoted shell word - # cibuildwheel evaluates. Percent-encoding leaves only unreserved - # characters and %, which are inert in both. - CIBW_ENVIRONMENT: "SINGLESTOREDB_URL='mysql://admin:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=0'" + # CLUSTER_PASSWORD has to survive both the userinfo half of the URL + # and the single-quoted shell word cibuildwheel evaluates, so keep the + # secret alphanumeric: no ':', '@', '/', '%' or quote characters. + CIBW_ENVIRONMENT: "SINGLESTOREDB_URL='mysql://admin:${{ secrets.CLUSTER_PASSWORD }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=0'" PYTHONPATH: ${{ github.workspace }} # - name: Build conda @@ -255,15 +244,6 @@ jobs: runs-on: ubuntu-latest steps: - # ::add-mask:: does not cross job boundaries; see the build-and-test job. - - name: Mask cluster password - env: - CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} - CLUSTER_PASSWORD_URL: ${{ needs.setup-database.outputs.cluster-password-url }} - run: | - echo "::add-mask::$CLUSTER_PASSWORD" - echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v7 - name: Install dependencies @@ -275,13 +255,13 @@ jobs: - name: Drop database if: ${{ always() }} # The password reaches the script through the environment rather than - # being interpolated into the command: the generated value can contain - # any printable character, including ones the shell would act on. + # being interpolated into the command, so the shell never sees its + # characters. run: | python resources/drop_db.py --user admin --password "$CLUSTER_PASSWORD" --host "$CLUSTER_HOST" --port 3306 --database "$CLUSTER_DATABASE" env: PYTHONPATH: ${{ github.workspace }} - CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} + CLUSTER_PASSWORD: ${{ secrets.CLUSTER_PASSWORD }} CLUSTER_HOST: ${{ needs.setup-database.outputs.cluster-host }} CLUSTER_DATABASE: ${{ needs.setup-database.outputs.cluster-database }} diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 4764f1458..4253a1bb4 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -32,8 +32,15 @@ jobs: # fixed everywhere below. The project is named here rather than read # from a repo variable so the deployment target is visible in the # workflow and does not depend on repository settings. + # + # POST /v2/clusters generates its own admin password and ignores any + # that is sent, so the script resets it to CLUSTER_PASSWORD over SQL + # once the cluster is up. That keeps the credential a secret the runner + # masks everywhere, instead of a job output: the runner refuses to write + # an output whose value is masked, so a generated password could not + # reach these jobs at all. run: | - python resources/create_test_cluster.py --token="${{ secrets.CLUSTER_API_KEY }}" --project="Standard Project" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" + python resources/create_test_cluster.py --password="${{ secrets.CLUSTER_PASSWORD }}" --token="${{ secrets.CLUSTER_API_KEY }}" --project="Standard Project" --init-sql singlestoredb/tests/test.sql --output=github --expires=2h "python - $GITHUB_WORKFLOW - $GITHUB_RUN_NUMBER" env: PYTHONPATH: ${{ github.workspace }} @@ -41,11 +48,6 @@ jobs: cluster-id: ${{ steps.initialize-database.outputs.cluster-id }} cluster-host: ${{ steps.initialize-database.outputs.cluster-host }} cluster-database: ${{ steps.initialize-database.outputs.cluster-database }} - # POST /v2/clusters generates the admin password and reports it only on - # the create response, so it travels as a job output rather than living - # in secrets. Job outputs are not secrets: every job below re-masks it. - cluster-password: ${{ steps.initialize-database.outputs.cluster-password }} - cluster-password-url: ${{ steps.initialize-database.outputs.cluster-password-url }} smoke-test: @@ -109,17 +111,6 @@ jobs: buffered: 1 steps: - # ::add-mask:: does not cross job boundaries, so the generated admin - # password arrives here unmasked and has to be registered again before - # any step can echo it into the log. - - name: Mask cluster password - env: - CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} - CLUSTER_PASSWORD_URL: ${{ needs.setup-database.outputs.cluster-password-url }} - run: | - echo "::add-mask::$CLUSTER_PASSWORD" - echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} @@ -138,10 +129,10 @@ jobs: run: pytest -v --pyargs singlestoredb.tests.test_basics env: PYTHONPATH: ${{ github.workspace }} - # cluster-password-url, not cluster-password: the generated password - # is drawn from the full printable set and has to be percent-encoded - # to survive the userinfo half of the URL. - SINGLESTOREDB_URL: "${{ matrix.driver }}://admin:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" + # CLUSTER_PASSWORD goes into the userinfo half of a URL here, so it + # has to be free of characters that would need percent-encoding -- + # ':', '@', '/', '%' and the like. Keep the secret alphanumeric. + SINGLESTOREDB_URL: "${{ matrix.driver }}://admin:${{ secrets.CLUSTER_PASSWORD }}@${{ needs.setup-database.outputs.cluster-host }}:3306/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" - name: Run tests if: ${{ matrix.driver == 'https' }} @@ -154,7 +145,7 @@ jobs: run: pytest -v -n 0 --pyargs singlestoredb.tests.test_basics env: PYTHONPATH: ${{ github.workspace }} - SINGLESTOREDB_URL: "${{ matrix.driver }}://admin:${{ needs.setup-database.outputs.cluster-password-url }}@${{ needs.setup-database.outputs.cluster-host }}:443/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" + SINGLESTOREDB_URL: "${{ matrix.driver }}://admin:${{ secrets.CLUSTER_PASSWORD }}@${{ needs.setup-database.outputs.cluster-host }}:443/${{ needs.setup-database.outputs.cluster-database }}?pure_python=${{ matrix.pure-python }}&buffered=${{ matrix.buffered }}" shutdown-database: @@ -163,15 +154,6 @@ jobs: runs-on: ubuntu-latest steps: - # ::add-mask:: does not cross job boundaries; see the smoke-test job. - - name: Mask cluster password - env: - CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} - CLUSTER_PASSWORD_URL: ${{ needs.setup-database.outputs.cluster-password-url }} - run: | - echo "::add-mask::$CLUSTER_PASSWORD" - echo "::add-mask::$CLUSTER_PASSWORD_URL" - - uses: actions/checkout@v7 - name: Set up Python 3.11 @@ -189,13 +171,13 @@ jobs: - name: Drop database if: ${{ always() }} # The password reaches the script through the environment rather than - # being interpolated into the command: the generated value can contain - # any printable character, including ones the shell would act on. + # being interpolated into the command, so the shell never sees its + # characters. run: | python resources/drop_db.py --user admin --password "$CLUSTER_PASSWORD" --host "$CLUSTER_HOST" --port 3306 --database "$CLUSTER_DATABASE" env: PYTHONPATH: ${{ github.workspace }} - CLUSTER_PASSWORD: ${{ needs.setup-database.outputs.cluster-password }} + CLUSTER_PASSWORD: ${{ secrets.CLUSTER_PASSWORD }} CLUSTER_HOST: ${{ needs.setup-database.outputs.cluster-host }} CLUSTER_DATABASE: ${{ needs.setup-database.outputs.cluster-database }} diff --git a/resources/create_test_cluster.py b/resources/create_test_cluster.py index 8c258eb02..7c0b5040d 100755 --- a/resources/create_test_cluster.py +++ b/resources/create_test_cluster.py @@ -2,7 +2,6 @@ # type: ignore from __future__ import annotations -import json import os import random import re @@ -10,7 +9,6 @@ import sys import uuid from optparse import OptionParser -from urllib.parse import quote import singlestoredb as s2 @@ -35,6 +33,12 @@ default='S-00', help='size of the cluster (S-00)', ) +parser.add_option( + '-p', '--password', + help='password to give the admin user once the cluster is up; required, ' + 'because the password the API generates cannot be handed to another ' + 'CI job (see below)', +) parser.add_option( '-t', '--token', help='API key for the management API', @@ -68,6 +72,10 @@ parser.print_help() sys.exit(1) +if not options.password: + print('ERROR: --password is required', file=sys.stderr) + sys.exit(1) + if options.init_sql and not os.path.isfile(options.init_sql): print(f'ERROR: Could not locate SQL file: {options.init_sql}', file=sys.stderr) sys.exit(1) @@ -160,26 +168,14 @@ def candidates(item): # after any refresh(). See item 9 of docs/management-api-audit.md: the API # accepts an adminPassword on both POST and PATCH and ignores both, which is # why this is read back rather than set. -password = cluster.admin_password -if not password: +generated = cluster.admin_password +if not generated: print( 'ERROR: cluster was created without a readable admin password', file=sys.stderr, ) sys.exit(1) -# The generated password is drawn from the full printable set -- one observed -# value was ``{:D}TK*[F3Ll}Ups2pNv`` -- so it cannot be dropped into the -# userinfo half of a connection URL as-is. Percent-encode everything, since -# the URL parser runs unquote_plus over the password -# (singlestoredb/connection.py:287); encoding ``+`` too is what keeps that from -# turning into a space. -password_url = quote(password, safe='') - -database = options.database -if not database: - database = 'TEMP_{}'.format(uuid.uuid4()).replace('-', '_') - host = cluster.endpoint if ':' in host: host, port = host.split(':', 1) @@ -187,35 +183,51 @@ def candidates(item): else: port = 3306 -# Print cluster information +# Trade the generated password for the caller's, because the generated one +# cannot leave this process. A caller running under GitHub Actions has to mask +# it, and the runner drops any output whose value matches a mask -- "Skip output +# 'cluster-password' since it may contain secret" -- so masking it and passing +# it to another job are mutually exclusive. The password the caller already +# holds has neither problem. +# +# ALTER USER is the statement that works: SET PASSWORD wants a pre-hashed value +# and rejects a literal with '1372: Password hash should be a 41-digit +# hexadecimal number'. Verified against a live S-00 cluster, including that the +# control plane leaves the new password alone afterwards. +password = options.password +escaped = password.replace('\\', '\\\\').replace("'", "\\'") + +with s2.connect( + host=host, port=port, user='admin', + password=generated, connect_timeout=30, +) as conn: + with conn.cursor() as cur: + cur.execute(f"ALTER USER 'admin'@'%' IDENTIFIED BY '{escaped}'") + +database = options.database +if not database: + database = 'TEMP_{}'.format(uuid.uuid4()).replace('-', '_') + +# Print cluster information. No password is reported: the caller passed it in, +# so it already knows it, and under GitHub Actions it is a secret the runner +# masks on its own. if options.output == 'env': print(f'CLUSTER_ID={cluster.id}') print(f'CLUSTER_HOST={host}') print(f'CLUSTER_PORT={port}') print(f'CLUSTER_DATABASE={database}') - print(f'CLUSTER_PASSWORD={password}') - print(f'CLUSTER_PASSWORD_URL={password_url}') elif options.output == 'github': - # Register both forms with the runner before anything can log them. This - # only holds within this job; each job that consumes the outputs has to - # mask them again for itself. - print(f'::add-mask::{password}') - print(f'::add-mask::{password_url}') with open(os.environ['GITHUB_OUTPUT'], 'a') as output: print(f'cluster-id={cluster.id}', file=output) print(f'cluster-host={host}', file=output) print(f'cluster-port={port}', file=output) print(f'cluster-database={database}', file=output) - print(f'cluster-password={password}', file=output) - print(f'cluster-password-url={password_url}', file=output) elif options.output == 'json': print('{') print(f' "cluster-id": "{cluster.id}",') print(f' "cluster-host": "{host}",') print(f' "cluster-port": {port},') - print(f' "cluster-database": "{database}",') - print(f' "cluster-password": {json.dumps(password)},') - print(f' "cluster-password-url": "{password_url}"') + print(f' "cluster-database": "{database}"') print('}') # Initialize the database From 51f6b148b9424ae9fdc1b58c43cc7ccaf14c3af7 Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 17 Sep 2026 12:38:41 -0400 Subject: [PATCH 09/12] Give the change detector the ref it compares against code-check.yml checks singlestoredb/management and singlestoredb/fusion for changes and picks between a run that includes the v2 management tests and one that excludes them. It has always picked the second. The checkout was shallow, fetch-depth: 2, which creates no origin/main, so every diff against it died -- "fatal: bad revision 'origin/main'" appears twice in each run log -- and the `|| true` turned that into an empty file list, which reads as "nothing changed". The step that runs -m 'not management_v1' was unreachable and the -m 'not management' one always won, so the 37 live v2 management tests never ran on a pull request; only the nightly coverage.yml covered them. Fetch the full history so origin/main exists, which also repairs the branch-push path that probed origin/main and origin/master and fell through to HEAD~1. Drop the `|| true` as well: a git failure means the comparison did not happen, and swallowing it silently downgrades the run rather than reporting the breakage. Expect this job to get slower on any PR touching those directories -- the tests it now selects deploy real clusters. Co-Authored-By: Claude Opus 5 --- .github/workflows/code-check.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index 36b44259a..318778f29 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -31,7 +31,12 @@ jobs: - name: Checkout code uses: actions/checkout@v7 with: - fetch-depth: 2 + # Full history, because the change detector below diffs against + # origin/main. A shallow clone does not create that ref -- with + # fetch-depth: 2 every diff died on `fatal: bad revision + # 'origin/main'`, which the detector read as "nothing changed", so + # the management step never ran on a PR. + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v7 @@ -94,7 +99,10 @@ jobs: for DIR in $MONITORED_DIRS; do if [ -d "$DIR" ]; then - CHANGED_FILES=$(git diff --name-only $BASE_COMMIT HEAD -- "$DIR" || true) + # No `|| true` here: a git failure means the comparison did not + # happen, and swallowing it silently downgrades the run to the + # no-management path instead of reporting the breakage. + CHANGED_FILES=$(git diff --name-only "$BASE_COMMIT" HEAD -- "$DIR") if [ -n "$CHANGED_FILES" ]; then echo "✅ Changes detected in: $DIR" echo "Files changed:" From 46610d72eb455648aa55ed5c0c0b01b7f24b3efe Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Fri, 18 Sep 2026 09:48:26 -0400 Subject: [PATCH 10/12] Pad the fraction on RFC 3339 timestamps too A job's createdAt came back as '2026-09-18T12:39:20.43888Z'. The zone group in _GO_DATETIME_RE demanded whitespace ahead of it, so a bare Z never matched and the value fell through to the escape hatch, which strips the Z and skips the fractional-second padding. Only Python 3.11 and later read a fraction that is neither 3 nor 6 digits, so on 3.10 the converter handed back the string and to_datetime_strict raised ValueError, taking down TestJobsFusion.test_run_wait_drop_job in CI. Recognize Z as an offset so RFC 3339 goes down the same path as the Go shape and gets its fraction padded, and spell the offset out as +00:00 for the same reason the numeric ones grew a colon: nothing before 3.11 parses the short form. _as_naive_utc shifts it back off, so parsed results are unchanged. Verified on a real 3.10 that every shape the normalizer emits parses, and that the old output for this value does not. The tests assert on the normalized string, not on a parsed datetime, so they fail on 3.11 as well -- the same blind spot that let the offset bug reach CI. Co-Authored-By: Claude Opus 5 --- singlestoredb/management/utils.py | 24 +++++++++++++------- singlestoredb/tests/test_management_utils.py | 21 +++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/singlestoredb/management/utils.py b/singlestoredb/management/utils.py index 781452f13..52aca29fb 100644 --- a/singlestoredb/management/utils.py +++ b/singlestoredb/management/utils.py @@ -414,9 +414,13 @@ def enable_http_tracing() -> None: #: RFC 3339, and the trailing zone name is not ISO 8601, so the whole value #: fails to parse and the expiration silently reads as unset. The zone name and #: the monotonic-clock reading Go appends to some values are both optional. +#: An RFC 3339 ``Z`` counts as an offset here so that shape goes down the same +#: path: its fraction needs the same padding, and until it matched, a value like +#: ``...20.43888Z`` reached the converter with five digits, which only 3.11 and +#: later parse. _GO_DATETIME_RE = re.compile( r'^(?P\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?)' - r'(?:\s*(?P[+-]\d{2}:?\d{2}))?' + r'(?:\s*(?P[Zz]|[+-]\d{2}:?\d{2}))?' r'(?:\s+(?P[A-Za-z]\S*))?' r'(?:\s+m=\S+)?$', ) @@ -429,7 +433,8 @@ def _normalize_datetime(obj: str) -> str: Handles the two shapes the management API returns -- RFC 3339 and the Go ``time.Time.String()`` form -- by reducing both to a bare ISO 8601 timestamp plus an optional numeric offset. Fractional seconds are padded to - microseconds, since Go trims trailing zeros. + microseconds -- Go trims trailing zeros, and ``datetime.fromisoformat`` + accepts only 3 or 6 digits before Python 3.11. Parameters ---------- @@ -457,9 +462,11 @@ def _normalize_datetime(obj: str) -> str: # Go writes the offset without a separator (+0000). Only Python 3.11 and # later accept that spelling; 3.9 and 3.10 want +00:00, so always emit the - # colon. + # colon. Z is spelled out for the same reason: nothing before 3.11 reads it. offset = match.group('offset') or '' - if offset and ':' not in offset: + if offset in ('Z', 'z'): + offset = '+00:00' + elif offset and ':' not in offset: offset = offset[:3] + ':' + offset[3:] return stamp + offset @@ -469,10 +476,11 @@ def _as_naive_utc(obj: datetime.datetime) -> datetime.datetime: """ Return ``obj`` as a naive UTC datetime. - An RFC 3339 timestamp loses its ``Z`` before it is parsed, so it arrives - here naive and already meaning UTC. A value carrying a numeric offset is - shifted onto UTC and stripped, so both shapes end up on the one convention - -- otherwise two timestamps read off the same object could not be compared. + A value carrying an offset -- which is every recognized shape, since an + RFC 3339 ``Z`` is normalized to ``+00:00`` -- is shifted onto UTC and + stripped. A value that arrives naive is already meaning UTC and is left + alone. Both end up on the one convention -- otherwise two timestamps read + off the same object could not be compared. Parameters ---------- diff --git a/singlestoredb/tests/test_management_utils.py b/singlestoredb/tests/test_management_utils.py index ba4f6e745..b2662c252 100644 --- a/singlestoredb/tests/test_management_utils.py +++ b/singlestoredb/tests/test_management_utils.py @@ -1858,6 +1858,27 @@ def test_offset_is_normalized_to_include_a_colon(self): '2026-09-17 09:42:41+05:30', ) + def test_rfc_3339_fraction_is_padded(self): + # The API trims trailing zeros here too: a job's createdAt came back as + # '2026-09-18T12:39:20.43888Z'. Only 3.11 and later read a fraction that + # is neither 3 nor 6 digits, so before Z was recognized as an offset this + # value skipped the padding and to_datetime_strict raised on 3.10. + self.assertEqual( + _normalize_datetime('2026-09-18T12:39:20.43888Z'), + '2026-09-18T12:39:20.438880+00:00', + ) + self.assertEqual( + to_datetime_strict('2026-09-18T12:39:20.43888Z'), + datetime.datetime(2026, 9, 18, 12, 39, 20, 438880), + ) + + def test_rfc_3339_nanoseconds_are_truncated(self): + # Nine digits does not fit a datetime; the extra ones are dropped. + self.assertEqual( + _normalize_datetime('2026-09-18T12:39:20.438880123Z'), + '2026-09-18T12:39:20.438880+00:00', + ) + def test_go_time_string_with_truncated_fraction(self): # Go trims trailing zeros, so the fraction is not always 6 digits. out = to_datetime('2026-09-17 14:42:41.4 +0000 UTC') From 1378da4d23fd388fe7892fd869080f69f3acd753 Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Fri, 18 Sep 2026 13:57:31 -0400 Subject: [PATCH 11/12] Smoke-test on Python 3.14 The matrix stopped at 3.13 while 3.14 has been final since October 2025, so the newest interpreter the package claims to support -- requires-python is >=3.9 with no ceiling -- went untested. Crossed with the driver axis this adds two jobs, mysql and https. 3.15 is left out on purpose: it is at rc2 today with GA planned for 2026-10-01, and setup-python will not resolve a bare "3.15" until then. The condition for adding it is recorded above the matrix. Not touched: the include: block still pins macOS and Windows to 3.11, so 3.14 is covered on Linux only, and publish.yml builds one abi3 wheel from cp39, which needs no change for a new minor. Co-Authored-By: Claude Opus 5 --- .github/workflows/smoke-test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 4253a1bb4..7e3bbf01f 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -59,12 +59,18 @@ jobs: matrix: os: - ubuntu-24.04 + # Every version from the floor in pyproject.toml (requires-python + # >=3.9) up to the newest final release. 3.15 is deliberately absent: + # as of 2026-09-18 it is at rc2 with GA planned for 2026-10-01, and + # setup-python needs allow-prereleases plus an explicit "3.15.0-rc.2" + # to install it at all. Add a bare "3.15" once it ships. python-version: - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" + - "3.14" driver: - mysql - https From 4e49cfa58e7b07048907ef5ec4f17df43bde1348 Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Fri, 18 Sep 2026 14:18:37 -0400 Subject: [PATCH 12/12] Report the cluster ID before anything else can fail Two findings from review. resources/create_test_cluster.py reported the cluster ID after the password reset and the SQL load, both of which can fail against a cluster that is already running and already billing. A CI teardown job would then have an empty cluster-id output and send its DELETE to /v2/clusters/, leaking the cluster. Everything the reporting block prints is known as soon as create_cluster() returns, so it now runs there. The two workflow shutdown steps also refuse to issue a DELETE with an empty ID, and --fail-with-body makes a refused one fail the step. to_datetime read Go's zero time as year 1 whenever it arrived in the Go shape -- '0001-01-01 00:00:00 +0000 UTC' -- because only the RFC 3339 spelling was compared against. That reports an expiry on a resource that does not expire. Both helpers now test the parsed value for January 1 of year 1, which covers every spelling including the offset, zone name and monotonic reading, and the check runs before the UTC shift, which can fall below MINYEAR on a year-1 value. Co-Authored-By: Claude Opus 5 --- .github/workflows/publish.yml | 10 ++- .github/workflows/smoke-test.yml | 10 ++- resources/create_test_cluster.py | 71 +++++++++++--------- singlestoredb/management/utils.py | 42 ++++++++++-- singlestoredb/tests/test_management_utils.py | 17 +++++ 5 files changed, 109 insertions(+), 41 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 30a4265a7..be006ab2d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -267,7 +267,15 @@ jobs: - name: Shutdown cluster if: ${{ always() }} + # An empty ID would send the DELETE to /v2/clusters/ and leave a live + # cluster behind, so say so loudly instead: at that point the ID has to + # be recovered by hand. --fail-with-body is what makes a refused DELETE + # fail this step rather than printing the error and exiting 0. run: | - curl -H "Accept: application/json" -H "Authorization: Bearer ${{ secrets.CLUSTER_API_KEY }}" -X DELETE "https://api.singlestore.com/v2/clusters/${{ env.CLUSTER_ID }}?force=true" + if [ -z "$CLUSTER_ID" ]; then + echo "::error::No cluster ID from setup-database; the cluster (if any) must be terminated by hand" + exit 1 + fi + curl --fail-with-body -H "Accept: application/json" -H "Authorization: Bearer ${{ secrets.CLUSTER_API_KEY }}" -X DELETE "https://api.singlestore.com/v2/clusters/$CLUSTER_ID?force=true" env: CLUSTER_ID: ${{ needs.setup-database.outputs.cluster-id }} diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 7e3bbf01f..f99b820e3 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -189,7 +189,15 @@ jobs: - name: Shutdown cluster if: ${{ always() }} + # An empty ID would send the DELETE to /v2/clusters/ and leave a live + # cluster behind, so say so loudly instead: at that point the ID has to + # be recovered by hand. --fail-with-body is what makes a refused DELETE + # fail this step rather than printing the error and exiting 0. run: | - curl -H "Accept: application/json" -H "Authorization: Bearer ${{ secrets.CLUSTER_API_KEY }}" -X DELETE "https://api.singlestore.com/v2/clusters/${{ env.CLUSTER_ID }}?force=true" + if [ -z "$CLUSTER_ID" ]; then + echo "::error::No cluster ID from setup-database; the cluster (if any) must be terminated by hand" + exit 1 + fi + curl --fail-with-body -H "Accept: application/json" -H "Authorization: Bearer ${{ secrets.CLUSTER_API_KEY }}" -X DELETE "https://api.singlestore.com/v2/clusters/$CLUSTER_ID?force=true" env: CLUSTER_ID: ${{ needs.setup-database.outputs.cluster-id }} diff --git a/resources/create_test_cluster.py b/resources/create_test_cluster.py index 7c0b5040d..28be11c38 100755 --- a/resources/create_test_cluster.py +++ b/resources/create_test_cluster.py @@ -163,6 +163,44 @@ def candidates(item): wait_timeout=1200, ) +host = cluster.endpoint +if ':' in host: + host, port = host.split(':', 1) + port = int(port) +else: + port = 3306 + +database = options.database +if not database: + database = 'TEMP_{}'.format(uuid.uuid4()).replace('-', '_') + +# Report before touching the cluster any further. Everything below can fail +# against a cluster that already exists and is already billing, and the caller's +# only handle on it is the ID reported here -- a CI teardown job with an empty +# cluster-id output would issue its DELETE against /v2/clusters/ and leak the +# cluster it was meant to remove. +# +# No password is reported: the caller passed it in, so it already knows it, and +# under GitHub Actions it is a secret the runner masks on its own. +if options.output == 'env': + print(f'CLUSTER_ID={cluster.id}') + print(f'CLUSTER_HOST={host}') + print(f'CLUSTER_PORT={port}') + print(f'CLUSTER_DATABASE={database}') +elif options.output == 'github': + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + print(f'cluster-id={cluster.id}', file=output) + print(f'cluster-host={host}', file=output) + print(f'cluster-port={port}', file=output) + print(f'cluster-database={database}', file=output) +elif options.output == 'json': + print('{') + print(f' "cluster-id": "{cluster.id}",') + print(f' "cluster-host": "{host}",') + print(f' "cluster-port": {port},') + print(f' "cluster-database": "{database}"') + print('}') + # The API generates the admin password and reports it only on the create # response -- there is no route that will hand it back later, and it is None # after any refresh(). See item 9 of docs/management-api-audit.md: the API @@ -176,13 +214,6 @@ def candidates(item): ) sys.exit(1) -host = cluster.endpoint -if ':' in host: - host, port = host.split(':', 1) - port = int(port) -else: - port = 3306 - # Trade the generated password for the caller's, because the generated one # cannot leave this process. A caller running under GitHub Actions has to mask # it, and the runner drops any output whose value matches a mask -- "Skip output @@ -204,32 +235,6 @@ def candidates(item): with conn.cursor() as cur: cur.execute(f"ALTER USER 'admin'@'%' IDENTIFIED BY '{escaped}'") -database = options.database -if not database: - database = 'TEMP_{}'.format(uuid.uuid4()).replace('-', '_') - -# Print cluster information. No password is reported: the caller passed it in, -# so it already knows it, and under GitHub Actions it is a secret the runner -# masks on its own. -if options.output == 'env': - print(f'CLUSTER_ID={cluster.id}') - print(f'CLUSTER_HOST={host}') - print(f'CLUSTER_PORT={port}') - print(f'CLUSTER_DATABASE={database}') -elif options.output == 'github': - with open(os.environ['GITHUB_OUTPUT'], 'a') as output: - print(f'cluster-id={cluster.id}', file=output) - print(f'cluster-host={host}', file=output) - print(f'cluster-port={port}', file=output) - print(f'cluster-database={database}', file=output) -elif options.output == 'json': - print('{') - print(f' "cluster-id": "{cluster.id}",') - print(f' "cluster-host": "{host}",') - print(f' "cluster-port": {port},') - print(f' "cluster-database": "{database}"') - print('}') - # Initialize the database if options.init_sql: init_db = [ diff --git a/singlestoredb/management/utils.py b/singlestoredb/management/utils.py index 52aca29fb..ba6553de4 100644 --- a/singlestoredb/management/utils.py +++ b/singlestoredb/management/utils.py @@ -472,6 +472,31 @@ def _normalize_datetime(obj: str) -> str: return stamp + offset +def _is_go_zero_time(obj: Union[datetime.date, datetime.datetime]) -> bool: + """ + Return whether ``obj`` is Go's zero time, which means "unset". + + A Go ``time.Time`` that was never assigned renders as January 1 of year 1, + and the API returns that for a field it has no value for -- most visibly an + ``expiresAt`` on a resource that does not expire. It arrives spelled either + way the two timestamp shapes allow: ``0001-01-01T00:00:00Z`` and + ``0001-01-01 00:00:00 +0000 UTC``. Testing the parsed value rather than the + string covers both, along with any offset or monotonic reading that comes + with them. + + Parameters + ---------- + obj : datetime.date or datetime.datetime + Parsed timestamp + + Returns + ------- + bool + + """ + return (obj.year, obj.month, obj.day) == (1, 1, 1) + + def _as_naive_utc(obj: datetime.datetime) -> datetime.datetime: """ Return ``obj`` as a naive UTC datetime. @@ -505,15 +530,18 @@ def to_datetime( return None if isinstance(obj, datetime.datetime): return obj - if obj == '0001-01-01T00:00:00Z': - return None out = converters.datetime_fromisoformat(_normalize_datetime(obj)) if isinstance(out, str): return None - if isinstance(out, datetime.date) and not isinstance(out, datetime.datetime): - return datetime.datetime(out.year, out.month, out.day) if out is None: return None + # Before _as_naive_utc: shifting an aware year-1 value onto UTC can carry it + # below datetime.MINYEAR, which raises rather than returning the None this + # value means. + if _is_go_zero_time(out): + return None + if isinstance(out, datetime.date) and not isinstance(out, datetime.datetime): + return datetime.datetime(out.year, out.month, out.day) return _as_naive_utc(out) @@ -525,13 +553,15 @@ def to_datetime_strict( raise TypeError('not possible to convert None to datetime') if isinstance(obj, datetime.datetime): return obj - if obj == '0001-01-01T00:00:00Z': - raise ValueError('not possible to convert 0001-01-01T00:00:00Z to datetime') out = converters.datetime_fromisoformat(_normalize_datetime(obj)) if not out: raise TypeError('not possible to convert None to datetime') if isinstance(out, str): raise ValueError('value cannot be str') + # See to_datetime: checked here rather than after the UTC shift, which can + # raise on a year-1 value. + if _is_go_zero_time(out): + raise ValueError(f'not possible to convert {obj} to datetime') if isinstance(out, datetime.date) and not isinstance(out, datetime.datetime): return datetime.datetime(out.year, out.month, out.day) return _as_naive_utc(out) diff --git a/singlestoredb/tests/test_management_utils.py b/singlestoredb/tests/test_management_utils.py index b2662c252..685c352b1 100644 --- a/singlestoredb/tests/test_management_utils.py +++ b/singlestoredb/tests/test_management_utils.py @@ -1914,6 +1914,19 @@ def test_zero_sentinel_and_unparseable_are_none(self): self.assertIsNone(to_datetime('')) self.assertIsNone(to_datetime('not a date')) + def test_the_go_spelling_of_the_zero_sentinel_is_none_too(self): + # Go's zero time means "unset" -- an expiresAt on a resource that does + # not expire -- and arrives in whichever shape the field uses. Reading + # the Go spelling as a real timestamp reported year 1 as an expiry. + self.assertIsNone(to_datetime('0001-01-01 00:00:00 +0000 UTC')) + # Recognized from the parsed value, so the trimmings Go may add do not + # each need their own literal. + self.assertIsNone( + to_datetime('0001-01-01 00:00:00 +0000 UTC m=+0.000000001'), + ) + self.assertIsNone(to_datetime('0001-01-01 00:00:00 +0000 GMT')) + self.assertIsNone(to_datetime('0001-01-01')) + def test_datetime_passes_through(self): given = datetime.datetime(2026, 9, 17, 13, 42, 41) self.assertIs(to_datetime(given), given) @@ -1928,6 +1941,10 @@ def test_strict_still_raises_on_nothing(self): with self.assertRaises(ValueError): to_datetime_strict('0001-01-01T00:00:00Z') + def test_strict_raises_on_the_go_spelling_of_the_sentinel(self): + with self.assertRaises(ValueError): + to_datetime_strict('0001-01-01 00:00:00 +0000 UTC') + if __name__ == '__main__': unittest.main()