diff --git a/Tiltfile b/Tiltfile index d69e1aac1e..60df93b8ad 100644 --- a/Tiltfile +++ b/Tiltfile @@ -55,6 +55,12 @@ local(["bash", "infra/space/kind/scripts/check-object-store-transition.sh", KIND # "Upgrading from the node-stack layout" in infra/space/kind/README.md. local(["bash", "infra/space/kind/scripts/check-node-layout.sh", KIND_CLUSTER], quiet = True) +# A cluster whose Postgres data directory an earlier space-postgres chart +# initialized (PostgreSQL 16) cannot run this chart's PostgreSQL 17; refuse it +# with the database reset rather than crash-loop the pod. See "Recovering from +# an unbundled migration version" in infra/space/kind/README.md. +local(["bash", "infra/space/kind/scripts/check-postgres-data.sh", KIND_CLUSTER], quiet = True) + # Select one metrics sink per process. The datadog and otel modes also select # the matching trace exporter; off disables both. # These local selectors do not activate any deployed environment. diff --git a/ci/mirrored-images.txt b/ci/mirrored-images.txt index 22b1591c24..87321fe5d8 100644 --- a/ci/mirrored-images.txt +++ b/ci/mirrored-images.txt @@ -59,7 +59,7 @@ kindest/node:v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8 # Images the Kind charts run (infra/space/kind/charts/*/values.yaml), which # scripts/tests/lib/bootstrap-kind-stack.sh loads into the cluster. -postgres:16-alpine sha256:1a66d744c1b459e13b05a8fca341da84cb63383e99ce262210efee5a319d4551 ghcr.io/perplexityai/space-ci-mirror:postgres-16-alpine +postgres:17 sha256:3cec7eb015ba8adb28139fa5c83b8489cdf0e666e53dfdf20f598ae0cc8739e3 ghcr.io/perplexityai/space-ci-mirror:postgres-17 redis:7-alpine sha256:ca0acbb137c1dc3339c8b147a58fd6f42775d4599327b50e7b116c23de501af2 ghcr.io/perplexityai/space-ci-mirror:redis-7-alpine # Bump versitygw to the first release that contains versity/versitygw#2415 # (see the space-object-store chart's values); Content-MD5 stays. diff --git a/infra/space/kind/README.md b/infra/space/kind/README.md index a6a4b7f2fb..ec7cf04eb2 100644 --- a/infra/space/kind/README.md +++ b/infra/space/kind/README.md @@ -868,6 +868,22 @@ tilt trigger space-seed-publish Deleting the publish Job is required because the database reset removes the template-release tags that resolve `seed:latest`. +The same reset applies to a cluster whose database was created by an earlier +revision of the `space-postgres` chart (PostgreSQL 16, superuser connection +string): PostgreSQL 17 cannot start on that data directory, and the role +setup the pod's init script provides runs only when it initializes an empty +one. `bootstrap.sh`, the smoke suites' `scripts/tests/lib/bootstrap-kind-stack.sh` +(under `SKIP_BOOTSTRAP=1` too) and the `Tiltfile` run +`infra/space/kind/scripts/check-postgres-data.sh` before they apply anything +or query the database. It reads the data directory's `PG_VERSION` on the node +that holds the volume, so it answers the same whether the old server is still +running or the new one is crash-looping, and on a mismatch it prints the reset +commands and stops without changing anything (the scripts exit 1; Tilt +reports a Tiltfile load error). Without Tilt, rerun `bootstrap.sh` (or the +smoke suite) after the three `kubectl delete` commands above; a running Tilt +reloads with `tilt trigger "(Tiltfile)"`. Recreating the cluster has the same +effect. + ### Diagnosing iteration disk pressure Every build loads its images into each Kind node's containerd store, which @@ -977,7 +993,7 @@ kind cluster: space-local (kind-cluster.yaml.tmpl — 6 nodes) ├── worker2 [pool=sandbox, space-enabled=true] │ └── identical to worker ├── worker3 [pool=infra, space.pplx.net/infra=true] -│ ├── space-postgres (StatefulSet) +│ ├── space-postgres (StatefulSet; PostgreSQL 17 serving TLS only, non-superuser master user) │ ├── space-object-store (versitygw S3 gateway + bucket-init Job) │ ├── space-api (Deployment, 2–3 replicas by HPA; SPACE_K8S_CONTROL_PLANE=true) │ ├── space-admin-api (Deployment; the same binary on the dedicated admin surface) @@ -1378,7 +1394,13 @@ asks of the hosting deployment. | One DaemonSet per node service, systemd watchdogs via k8s probes | One DaemonSet per node service and role from the Kind charts (`charts/space--local`), probed on each service's health socket; runtime-supervisor is the same host systemd unit node-manager's init container installs | | Node image (packer) provides `/space/cache`, the cgroup subtree, loop devices, device permissions, SPACE accounts and the `/space` skeleton before any pod starts | Sandbox workers and builders boot the SPACE Kind node image, which bakes in the same as systemd units that run before kubelet on every node start (see [The node layer](#the-node-layer)) | | `/space/cache` is btrfs on instance-store NVMe | btrfs on a sparse loopback image at `/var/lib/space-cache.img` on the kind node (40 GiB, same code paths, less I/O headroom), mounted on the node's `/space/cache` by `space-cache.service`, so it outlives pod restarts like the NVMe does | -| Postgres on RDS, secrets via ExternalSecrets / AWS Secrets Manager | In-cluster `space-postgres` StatefulSet, static Kubernetes Secrets | +| Postgres on RDS, secrets via ExternalSecrets / AWS Secrets Manager | In-cluster `space-postgres` StatefulSet, one pod on a 1 GiB volume, static Kubernetes Secrets | +| PostgreSQL 17 | The same major version (`postgres:17`; CI takes it from the image mirror at a pinned digest) | +| Every client connects with `sslmode=require` | The same `sslmode=require` on every connection: the services and the migrate Job through the `space-postgres` Secret's connection string, and the harness's `psql` helpers over loopback TLS inside the pod. The certificate is self-signed per pod start, which `require` accepts because it does not verify the server. Kind's server also refuses unencrypted TCP, so no Kind client runs in plaintext and one that sets `sslmode=disable` fails. A client that omits `sslmode` still connects over TLS, because libpq and sqlx default to `prefer`, so Kind cannot catch a connection string that loses `sslmode=require` | +| Bootstrap and migrations run as the managed service's master user, which is not a superuser but a member of `rds_superuser` | The same branch of `bootstrap-roles`: every connection string names `space_admin`, which has `CREATEDB CREATEROLE` and `rds_superuser` WITH ADMIN OPTION and owns the database. A stock server has none of that, so the pod's init script creates it, including the `rds_superuser` role itself and its `SET` privilege on `log_statement`, which a managed service grants that role and `bootstrap-roles` needs before writing passwords. The image's superuser remains, used only by the image's init and the harness's direct-state `psql` | +| Per-service pools: the api tier far above the binary's default of 25, internal-api somewhat above it, every other service on the default; no connection pooler | `POSTGRES_POOL_MAX_CONNECTIONS` 100 for space-api and space-admin-api, 30 for internal-api, unset elsewhere; no pooler. `max_connections` (1400) holds every replica's pools at their HPA maxima plus a rollout surge, the migrate Job and headroom, so a service runs out of its own pool before the server refuses it. The pod's memory limit (8 GiB) is sized for that many backends rather than for the suites' usual load | +| Server logs lock waits; the application roles' `lock_timeout` is off (`SPACE_DB_APP_LOCK_TIMEOUT` `0`) | `log_lock_waits = on`; `SPACE_DB_APP_LOCK_TIMEOUT` is unset, which leaves `lock_timeout` off the same way. Other server parameters are the image's defaults rather than a managed parameter group | +| The migration Job fails fast: one attempt, no restarts, its deadline passed to the migrator | The same `space-db-migrate` Job shape (`backoffLimit: 0`, `restartPolicy: Never`, `activeDeadlineSeconds` = `SPACE_DB_MIGRATE_JOB_DEADLINE_SECONDS`), bootstrap-roles → migrate → bootstrap-roles. `bootstrap.sh` stops on a failed Job with its logs; rerunning it replaces the Job | | S3 for snapshots + template artifacts | In-cluster versitygw (real S3 protocol, sigv4, path-style) | | `space-credmgr-authority` Deployment whose platform KEK comes from an external intermediate-key broker, with TLS from its own certificate (`credmgr_authority_tls.tf`; the direct KMS backend is gone, `credmgr_authority_kms.tf`) | two replicas on the same broker path against the in-repo broker fixture (`space-kind-key-broker`), which derives keys instead of protecting them; TLS from a self-signed certificate `internal-rpc-tls.sh` issues per cluster | | Internal RPC CA and listener leaves issued by Terraform and synced by External Secrets (`internal_rpc_tls.tf`) | the same Secrets issued per cluster by `scripts/internal-rpc-tls.sh`; the ingress activation leaf names `space-api` rather than `space-ingress-activation` | diff --git a/infra/space/kind/charts/space-api-local/templates/deployment.yaml b/infra/space/kind/charts/space-api-local/templates/deployment.yaml index 41d8e67003..6989fb45ca 100644 --- a/infra/space/kind/charts/space-api-local/templates/deployment.yaml +++ b/infra/space/kind/charts/space-api-local/templates/deployment.yaml @@ -148,6 +148,8 @@ spec: {{- include "space-cluster.dbPassword" (dict "ctx" . "role" "USER" "value" .Values.postgres.managedUserPassword) | nindent 14 }} - name: SPACE_DB_CREDS_PASSWORD {{- include "space-cluster.dbPassword" (dict "ctx" . "role" "CREDS" "value" .Values.postgres.managedCredentialsPassword) | nindent 14 }} + - name: POSTGRES_POOL_MAX_CONNECTIONS + value: {{ .Values.postgres.poolMaxConnections | quote }} - name: SPACE_INTERNAL_TOKEN valueFrom: secretKeyRef: diff --git a/infra/space/kind/charts/space-api-local/templates/migrate-job.yaml b/infra/space/kind/charts/space-api-local/templates/migrate-job.yaml index 92b4e25d49..43dbda9835 100644 --- a/infra/space/kind/charts/space-api-local/templates/migrate-job.yaml +++ b/infra/space/kind/charts/space-api-local/templates/migrate-job.yaml @@ -6,6 +6,12 @@ # runner can turn a retried lock race into a failed run # (crates/libs/db/migrator/src/migration_retry.rs). Tilt makes space-api wait # for this Job, and bootstrap.sh waits for it before applying the API. +# +# It fails fast, as a deployment's migration Job does: one attempt, no pod +# restarts, inside a deadline the migrator sizes its lock-retry window against +# (--job-deadline-seconds, crates/libs/db/migrator/src/lib.rs). A failed run +# stays Failed for an operator to read rather than being retried into a +# different state; bootstrap.sh and Tilt replace the Job to run it again. apiVersion: batch/v1 kind: Job metadata: @@ -15,14 +21,15 @@ metadata: app.kubernetes.io/name: space-db-migrate app.kubernetes.io/instance: {{ .Release.Name }} spec: - backoffLimit: 6 + backoffLimit: 0 + activeDeadlineSeconds: {{ .Values.migrate.jobDeadlineSeconds }} template: metadata: labels: app.kubernetes.io/name: space-db-migrate app.kubernetes.io/instance: {{ .Release.Name }} spec: - restartPolicy: OnFailure + restartPolicy: Never {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -33,8 +40,9 @@ spec: # pre-run creates the managed logins (space_user, space_creds, …) # the services connect as, and the post-run grants them on any # tables the migrations just created. The kind Postgres connection - # string authenticates as the image-created superuser, which can - # create roles. + # string authenticates as the master user, which, as on a managed + # database, is not a superuser but can create roles (space-postgres + # chart, configmap.yaml). - name: bootstrap-roles image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} @@ -60,6 +68,10 @@ spec: secretKeyRef: name: {{ .Values.postgres.secretName | default "space-postgres" }} key: POSTGRES_CONNECTION_STRING + # Kubernetes does not expose activeDeadlineSeconds to the + # container, so the Job passes its own. + - name: SPACE_DB_MIGRATE_JOB_DEADLINE_SECONDS + value: {{ .Values.migrate.jobDeadlineSeconds | quote }} containers: - name: bootstrap-roles-grants image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/infra/space/kind/charts/space-api-local/values.yaml b/infra/space/kind/charts/space-api-local/values.yaml index 3f31d6c324..adb0eca720 100644 --- a/infra/space/kind/charts/space-api-local/values.yaml +++ b/infra/space/kind/charts/space-api-local/values.yaml @@ -84,6 +84,10 @@ clusterId: "kind-local" # space-db-migrator + space-api-svc). migrate: enabled: true + # activeDeadlineSeconds, also passed to the migrator. The migrator's own + # default for an unstated deadline (MIGRATION_JOB_ACTIVE_DEADLINE, + # crates/libs/db/migrator/src/migration_retry.rs). + jobDeadlineSeconds: 1800 postgres: # Secret holding POSTGRES_CONNECTION_STRING. @@ -99,6 +103,12 @@ postgres: bootstrapSuperadminPassword: space-dev-superadmin bootstrapReadOnlyPassword: space-dev-ro bootstrapMigratorPassword: space-dev-migrator + # POSTGRES_POOL_MAX_CONNECTIONS for space-api and space-admin-api, each of + # whose two pools (space_user and space_creds) it sizes + # (crates/libs/common/svc/src/sql/mod.rs). The api tier runs a pool far + # above the binary's default of 25, as deployed. The space-postgres chart's + # max_connections is sized to hold every replica's pools. + poolMaxConnections: 100 # K8s control-plane substrate. SPACE_K8S_CONTROL_PLANE=true runs the # api-server on the KubernetesSubstrate: it discovers nodes through the Pod diff --git a/infra/space/kind/charts/space-internal-api-local/templates/deployment.yaml b/infra/space/kind/charts/space-internal-api-local/templates/deployment.yaml index 86a3b32446..10131fb09d 100644 --- a/infra/space/kind/charts/space-internal-api-local/templates/deployment.yaml +++ b/infra/space/kind/charts/space-internal-api-local/templates/deployment.yaml @@ -133,6 +133,8 @@ spec: key: {{ .Values.postgres.secretKey }} - name: SPACE_DB_USER_PASSWORD {{- include "space-cluster.dbPassword" (dict "ctx" . "role" "USER" "value" .Values.postgres.managedUserPassword) | nindent 14 }} + - name: POSTGRES_POOL_MAX_CONNECTIONS + value: {{ .Values.postgres.poolMaxConnections | quote }} - name: SPACE_INTERNAL_TOKEN valueFrom: secretKeyRef: diff --git a/infra/space/kind/charts/space-internal-api-local/values.yaml b/infra/space/kind/charts/space-internal-api-local/values.yaml index 239bb6ca0d..4a6f72322c 100644 --- a/infra/space/kind/charts/space-internal-api-local/values.yaml +++ b/infra/space/kind/charts/space-internal-api-local/values.yaml @@ -59,8 +59,13 @@ postgres: secretName: space-postgres secretKey: POSTGRES_CONNECTION_STRING # Dev-default space_user password; the role is provisioned by the api - # chart's bootstrap-roles initContainer with the same value. + # chart's space-db-migrate Job (bootstrap-roles) with the same value. managedUserPassword: space-dev-user + # POSTGRES_POOL_MAX_CONNECTIONS for the general space_user pool + # (crates/libs/common/svc/src/sql/mod.rs); the template-visibility sideline + # pool keeps its own fixed cap. Somewhat above the binary's default of 25, + # as deployed. The space-postgres chart's max_connections holds it. + poolMaxConnections: 30 nodeSelector: space.pplx.net/infra: "true" diff --git a/infra/space/kind/charts/space-postgres/Chart.yaml b/infra/space/kind/charts/space-postgres/Chart.yaml index 98c3ab8be3..94fec3f0e0 100644 --- a/infra/space/kind/charts/space-postgres/Chart.yaml +++ b/infra/space/kind/charts/space-postgres/Chart.yaml @@ -3,4 +3,4 @@ name: space-postgres description: Single-pod Postgres for the local KinD SPACE stack type: application version: 0.1.0 -appVersion: "16" +appVersion: "17" diff --git a/infra/space/kind/charts/space-postgres/templates/configmap.yaml b/infra/space/kind/charts/space-postgres/templates/configmap.yaml new file mode 100644 index 0000000000..4418f83466 --- /dev/null +++ b/infra/space/kind/charts/space-postgres/templates/configmap.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-postgres + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: space-postgres + app.kubernetes.io/instance: {{ .Release.Name }} +data: + # Every TCP client must encrypt: an unencrypted connection matches no line + # and is refused, so no Kind client runs in plaintext and one that sets + # sslmode=disable fails. A client that omits sslmode still connects, over + # TLS: libpq and sqlx both default to prefer. The Unix socket serves only + # the image's init scripts; the in-pod harness and probes connect over + # loopback TLS. + pg_hba.conf: | + # TYPE DATABASE USER ADDRESS METHOD + local all all trust + hostssl all all 127.0.0.1/32 trust + hostssl all all ::1/128 trust + hostssl all all all scram-sha-256 + # Run once by the image's entrypoint when it initializes an empty data + # directory, as the image superuser. It provides what a managed service gives + # its master user, which bootstrap-roles' managed-service branch relies on + # (crates/libs/db/migrator/src/bootstrap_roles.rs): + # - rds_superuser exists, so converge_superadmin_attributes grants it to + # space_superadmin instead of making that role a real superuser. The master + # holds it WITH ADMIN OPTION, which that GRANT needs. + # - CREATEROLE, for ensure_roles and converge_memberships, and CREATEDB, which + # a non-superuser needs to give space_superadmin CREATEDB. + # - SET on log_statement for rds_superuser: prepare_session mutes statement + # logging before it writes role passwords and fails without it. A managed + # service lets its rds_superuser set that parameter; stock PostgreSQL needs + # the grant. + # - Ownership of the database, so the master owns schema public + # (pg_database_owner) and can converge its privileges + # (converge_schema_privileges), as a managed service's master owns the + # database it creates. + 10-managed-service-master.sh: | + #!/usr/bin/env bash + set -Eeuo pipefail + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" \ + --set=master_user="$SPACE_DB_MASTER_USER" \ + --set=master_password="$SPACE_DB_MASTER_PASSWORD" \ + --set=database="$POSTGRES_DB" <<'SQL' + CREATE ROLE rds_superuser NOLOGIN; + GRANT SET ON PARAMETER log_statement TO rds_superuser; + CREATE ROLE :"master_user" WITH LOGIN CREATEDB CREATEROLE PASSWORD :'master_password'; + GRANT rds_superuser TO :"master_user" WITH ADMIN OPTION; + ALTER DATABASE :"database" OWNER TO :"master_user"; + SQL diff --git a/infra/space/kind/charts/space-postgres/templates/secret.yaml b/infra/space/kind/charts/space-postgres/templates/secret.yaml index 42893aea03..00f47a0f02 100644 --- a/infra/space/kind/charts/space-postgres/templates/secret.yaml +++ b/infra/space/kind/charts/space-postgres/templates/secret.yaml @@ -11,4 +11,8 @@ stringData: POSTGRES_USER: {{ .Values.credentials.user | quote }} POSTGRES_PASSWORD: {{ .Values.credentials.password | quote }} POSTGRES_DB: {{ .Values.credentials.database | quote }} - POSTGRES_CONNECTION_STRING: "postgres://{{ .Values.credentials.user }}:{{ .Values.credentials.password }}@{{ .Values.service.name }}:{{ .Values.service.port }}/{{ .Values.credentials.database }}?sslmode=disable" + SPACE_DB_MASTER_USER: {{ .Values.credentials.masterUser | quote }} + SPACE_DB_MASTER_PASSWORD: {{ .Values.credentials.masterPassword | quote }} + # The master user over TLS, as deployed: sslmode=require encrypts without + # verifying the server, so the pod's self-signed certificate is enough. + POSTGRES_CONNECTION_STRING: "postgres://{{ .Values.credentials.masterUser }}:{{ .Values.credentials.masterPassword }}@{{ .Values.service.name }}:{{ .Values.service.port }}/{{ .Values.credentials.database }}?sslmode=require" diff --git a/infra/space/kind/charts/space-postgres/templates/statefulset.yaml b/infra/space/kind/charts/space-postgres/templates/statefulset.yaml index 17b3f6d311..643423c3e8 100644 --- a/infra/space/kind/charts/space-postgres/templates/statefulset.yaml +++ b/infra/space/kind/charts/space-postgres/templates/statefulset.yaml @@ -18,6 +18,9 @@ spec: metadata: annotations: ad.datadoghq.com/tags: '{"team":"{{ .Values.datadogTeam }}"}' + # The init script and pg_hba.conf live in the ConfigMap; roll the pod + # when they change. + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} labels: app.kubernetes.io/name: space-postgres app.kubernetes.io/instance: {{ .Release.Name }} @@ -28,10 +31,48 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} terminationGracePeriodSeconds: 30 + initContainers: + # A self-signed server certificate per pod start. Clients connect with + # sslmode=require, which encrypts without verifying the server, so no + # cluster CA is involved. The server refuses a key file that others + # can read or that neither root nor the database user owns, so the + # key is mode 0600 and chowned to the image's postgres account by + # name: its uid differs between the Debian and alpine images. The + # openssl CLI comes from the Debian image; the alpine images do not + # ship it. + - name: tls + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -ec + - | + umask 077 + openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \ + -subj "/CN={{ .Values.service.name }}" \ + -addext "subjectAltName=DNS:{{ .Values.service.name }},DNS:{{ .Values.service.name }}.{{ .Release.Namespace }}.svc,DNS:localhost,IP:127.0.0.1" \ + -keyout /tls/tls.key -out /tls/tls.crt + chown postgres:postgres /tls/tls.key /tls/tls.crt + volumeMounts: + - name: tls + mountPath: /tls containers: - name: postgres image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - -c + - ssl=on + - -c + - ssl_cert_file=/var/run/space-postgres-tls/tls.crt + - -c + - ssl_key_file=/var/run/space-postgres-tls/tls.key + - -c + - hba_file=/etc/space-postgres/pg_hba.conf + - -c + - max_connections={{ .Values.server.maxConnections }} + - -c + - log_lock_waits={{ .Values.server.logLockWaits }} env: - name: POSTGRES_USER valueFrom: @@ -48,6 +89,17 @@ spec: secretKeyRef: name: space-postgres key: POSTGRES_DB + # Read by the managed-service init script (configmap.yaml). + - name: SPACE_DB_MASTER_USER + valueFrom: + secretKeyRef: + name: space-postgres + key: SPACE_DB_MASTER_USER + - name: SPACE_DB_MASTER_PASSWORD + valueFrom: + secretKeyRef: + name: space-postgres + key: SPACE_DB_MASTER_PASSWORD - name: PGDATA value: /var/lib/postgresql/data/pgdata ports: @@ -67,10 +119,32 @@ spec: volumeMounts: - name: data mountPath: /var/lib/postgresql/data + - name: tls + mountPath: /var/run/space-postgres-tls + readOnly: true + - name: config + mountPath: /etc/space-postgres/pg_hba.conf + subPath: pg_hba.conf + readOnly: true + - name: config + mountPath: /docker-entrypoint-initdb.d/10-managed-service-master.sh + subPath: 10-managed-service-master.sh + readOnly: true {{- with .Values.resources }} resources: {{- toYaml . | nindent 12 }} {{- end }} + volumes: + - name: tls + emptyDir: + medium: Memory + sizeLimit: 1Mi + - name: config + configMap: + name: {{ .Release.Name }}-postgres + # Executable, so the image's entrypoint runs the init script in + # its own shell rather than sourcing it into the entrypoint's. + defaultMode: 0555 volumeClaimTemplates: - metadata: name: data diff --git a/infra/space/kind/charts/space-postgres/values.yaml b/infra/space/kind/charts/space-postgres/values.yaml index 2028c8f708..fe1bb7af23 100644 --- a/infra/space/kind/charts/space-postgres/values.yaml +++ b/infra/space/kind/charts/space-postgres/values.yaml @@ -1,6 +1,9 @@ +# PostgreSQL 17, the major version deployments run and SPACE is built and +# tested against (infra/space/aws/reference/addons/variables.tf). CI takes this +# tag from the GHCR mirror at the digest ci/mirrored-images.txt pins. image: repository: postgres - tag: "16-alpine" + tag: "17" pullPolicy: IfNotPresent datadogTeam: team-space @@ -10,14 +13,54 @@ service: port: 5432 # Dev-only credentials. Override in CI or anything beyond local laptops. -# NOTE: do NOT use "space" anywhere here. Tilt redacts secret VALUES from -# command lines (including docker build-args), so a `space` substring inside -# this secret poisons every `docker build … space-rust-build-… …` invocation -# and the build fails with "pull access denied" on the redacted image name. +# NOTE: do NOT use "space" as any whole value here. Tilt redacts secret VALUES +# from command lines (including docker build-args), so a value of `space` +# poisons every `docker build … space-rust-build-… …` invocation and the build +# fails with "pull access denied" on the redacted image name. credentials: + # The image's superuser, which initdb creates. Only the image's init and the + # harness's direct-state psql helpers (kind_stack_psql) use it; no SPACE + # process connects as it. user: spaceuser password: spacepassword database: spacedb + # The login every SPACE connection string names, as a managed database's + # master user: not a superuser, but CREATEDB CREATEROLE and a member of + # rds_superuser (templates/configmap.yaml creates both). bootstrap-roles + # takes its managed-service branch for this session + # (converge_superadmin_attributes and converge_role_attributes in + # crates/libs/db/migrator/src/bootstrap_roles.rs), and the services replace + # it with their managed identity (database_connect_options, + # crates/libs/common/svc/src/sql/mod.rs). Deployments name it space_admin, + # which bootstrap-roles never manages (ensure_no_session_collision). + masterUser: space_admin + masterPassword: masterpassword + +# Server settings, passed to postgres as `-c` flags (templates/statefulset.yaml). +server: + # Every client pool's maximum must fit, so that a service is limited by its + # own pool, as deployed, and never by the server refusing a connection. + # Per process, from the pools each one opens and POSTGRES_POOL_MAX_CONNECTIONS + # (default 25, crates/libs/common/svc/src/sql/mod.rs) in its chart: + # space-api 2 pools x 100 (space_user + space_creds, + # crates/libs/api/src/run.rs) x 4 pods (HPA maximum 3, + # plus 1 rolling-update surge) 800 + # space-admin-api 2 x 100 x 1 (Recreate) 200 + # internal-api 30 + 3 (template-visibility sideline pool, + # crates/libs/internal/api/server/src/run.rs) x 1 33 + # cluster-controller (2 x 25 + 1 node-drain listener, + # node_drain/store.rs) x 3 (2 replicas + 1 surge) 153 + # credmgr-authority 25 (space_creds) x 3 (2 replicas + 1 surge) 75 + # space-db-migrate one connection per step (lib.rs, bootstrap_roles.rs) 1 + # ----- + # 1262 + # The rest is headroom for operator and harness psql sessions and the + # superuser reserve (superuser_reserved_connections, 3). A pool-size or + # replica change in those charts must keep this sum below the setting. + maxConnections: 1400 + # Logs each lock wait longer than deadlock_timeout (1 s), as the deployed + # server does, so contention shows up in Kind's logs. + logLockWaits: "on" nodeSelector: space.pplx.net/infra: "true" @@ -28,7 +71,15 @@ storage: resources: requests: cpu: 50m - memory: 128Mi + memory: 256Mi + # Sized for maxConnections backends rather than for the few dozen the suites + # usually hold open: each backend keeps its own memory (catalog caches, + # prepared statements, and work_mem for each sort or hash it runs), so pools + # that grow toward their maxima must not OOM-kill the database. Against this + # image and the migrated schema, every TLS backend that has read each table + # holds about 3.3 MiB, 5.3 GiB for the pod at maxConnections; 4 GiB was + # OOM-killed at about 1,280. The rest is headroom for prepared statements and + # work_mem. The request stays small, so the limit reserves nothing. limits: cpu: 500m - memory: 512Mi + memory: 8Gi diff --git a/infra/space/kind/scripts/bootstrap-prod-seed.sh b/infra/space/kind/scripts/bootstrap-prod-seed.sh index c6d57a6da1..5015591ba8 100755 --- a/infra/space/kind/scripts/bootstrap-prod-seed.sh +++ b/infra/space/kind/scripts/bootstrap-prod-seed.sh @@ -17,7 +17,7 @@ # # Required env (the bootstrap-templates.sh half): # POSTGRES_CONNECTION_STRING — control-plane DB. Defaults to a forwarded -# postgresql://spaceuser:spacepassword@127.0.0.1:5433/spacedb +# postgresql://spaceuser:spacepassword@127.0.0.1:5433/spacedb?sslmode=require # so callers don't need to know the secret. # AWS_* — sandbox creds (read-only suffices); load # with `set -a; source infra/space/kind/private.env`. @@ -58,7 +58,7 @@ aws sts get-caller-identity >/dev/null 2>&1 \ # ── Postgres port-forward (idempotent) ──────────────────────────────────── PG_PF_PORT="${PG_PF_PORT:-5433}" ensure_pg_forwarded() { - if /bin/psql "postgresql://spaceuser:spacepassword@127.0.0.1:${PG_PF_PORT}/spacedb" \ + if /bin/psql "postgresql://spaceuser:spacepassword@127.0.0.1:${PG_PF_PORT}/spacedb?sslmode=require" \ -c 'SELECT 1' >/dev/null 2>&1; then log "postgres reachable at 127.0.0.1:${PG_PF_PORT}" return 0 @@ -70,7 +70,7 @@ ensure_pg_forwarded() { local i for i in 1 2 3 4 5 6 7 8 9 10; do sleep 1 - if /bin/psql "postgresql://spaceuser:spacepassword@127.0.0.1:${PG_PF_PORT}/spacedb" \ + if /bin/psql "postgresql://spaceuser:spacepassword@127.0.0.1:${PG_PF_PORT}/spacedb?sslmode=require" \ -c 'SELECT 1' >/dev/null 2>&1; then log "postgres reachable after ${i}s" return 0 @@ -82,7 +82,7 @@ ensure_pg_forwarded() { # ── Step 1: control-plane registration via the shared bootstrap script ──── register_control_plane() { log "registering templates in control-plane Postgres" - PATH=/bin:$PATH POSTGRES_CONNECTION_STRING="postgresql://spaceuser:spacepassword@127.0.0.1:${PG_PF_PORT}/spacedb" \ + PATH=/bin:$PATH POSTGRES_CONNECTION_STRING="postgresql://spaceuser:spacepassword@127.0.0.1:${PG_PF_PORT}/spacedb?sslmode=require" \ bash "$REPO_ROOT/scripts/bootstrap-templates.sh" } @@ -112,7 +112,7 @@ stage_artifacts_on_workers() { # Pull manifest + metadata fresh from S3, derive the hash + storage prefix # to avoid relying on whatever the prior step persisted. - hash="$(/bin/psql "postgresql://spaceuser:spacepassword@127.0.0.1:${PG_PF_PORT}/spacedb" \ + hash="$(/bin/psql "postgresql://spaceuser:spacepassword@127.0.0.1:${PG_PF_PORT}/spacedb?sslmode=require" \ -tA -v ON_ERROR_STOP=1 -v name="$name" <<'SQL' SELECT tr.hash FROM template_release_tags t @@ -121,7 +121,7 @@ WHERE t.template_name = :'name' AND t.tag = 'latest'; SQL )" [[ -n "$hash" ]] || die "no 'latest' release for $name in control-plane" - storage_uri="$(/bin/psql "postgresql://spaceuser:spacepassword@127.0.0.1:${PG_PF_PORT}/spacedb" \ + storage_uri="$(/bin/psql "postgresql://spaceuser:spacepassword@127.0.0.1:${PG_PF_PORT}/spacedb?sslmode=require" \ -tA -v ON_ERROR_STOP=1 -v name="$name" -v hash="$hash" <<'SQL' SELECT l.storage_connection_string FROM template_release_storage_locations l diff --git a/infra/space/kind/scripts/bootstrap.sh b/infra/space/kind/scripts/bootstrap.sh index 5123f03751..571f8954ad 100755 --- a/infra/space/kind/scripts/bootstrap.sh +++ b/infra/space/kind/scripts/bootstrap.sh @@ -197,6 +197,10 @@ if kind get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER"; then # service, and its sandboxes, in one pod per node; stop rather than run both # layouts on a node. bash "$KIND_DIR/scripts/check-node-layout.sh" "$KIND_CLUSTER" || exit 1 + # A database an earlier space-postgres chart initialized (PostgreSQL 16) + # cannot start under this chart's PostgreSQL 17; stop with the reset before + # the chart is applied rather than wait out its rollout. + bash "$KIND_DIR/scripts/check-postgres-data.sh" "$KIND_CLUSTER" || exit 1 else # The sandbox workers and builders boot the SPACE Kind node image, which # bakes in the node layer a production node image provides (btrfs @@ -387,9 +391,26 @@ kubectl --context "$KCTX" -n "$NAMESPACE" delete job space-db-migrate --ignore-n helm template space "$KIND_DIR/charts/space-api-local" --namespace "$NAMESPACE" \ ${TELEMETRY_HELM_ARGS[@]+"${TELEMETRY_HELM_ARGS[@]}"} --show-only templates/migrate-job.yaml \ | kubectl --context "$KCTX" apply -n "$NAMESPACE" --validate=false -f - +# The Job makes one attempt (backoffLimit 0), so a failure is final: stop on +# it with the Job's logs instead of waiting out the timeout for a completion +# that cannot come. Rerunning bootstrap.sh replaces the Job above. log "waiting for the space-db-migrate Job (control-plane migrations)" -kubectl --context "$KCTX" -n "$NAMESPACE" wait --for=condition=complete \ - job/space-db-migrate --timeout="${TIMEOUT}s" >/dev/null +migrate_deadline=$((SECONDS + TIMEOUT)) +while :; do + migrate_conditions="$(kubectl --context "$KCTX" -n "$NAMESPACE" get job space-db-migrate \ + -o jsonpath='{range .status.conditions[?(@.status=="True")]}{.type}{" "}{end}')" \ + || migrate_conditions="" + case " $migrate_conditions " in + *" Complete "*) break ;; + *" Failed "*) + kubectl --context "$KCTX" -n "$NAMESPACE" logs job/space-db-migrate \ + --all-containers --prefix --tail=50 >&2 || true + die "the space-db-migrate Job failed; its logs are above" + ;; + esac + (( SECONDS < migrate_deadline )) || die "the space-db-migrate Job did not finish within ${TIMEOUT}s" + sleep 2 +done # api-server. The main container is gated on /readyz, which only turns 200 # after the K8s pod reflector's initial LIST completes (empty list is fine at diff --git a/infra/space/kind/scripts/check-postgres-data.sh b/infra/space/kind/scripts/check-postgres-data.sh new file mode 100755 index 0000000000..ac4aee6c9c --- /dev/null +++ b/infra/space/kind/scripts/check-postgres-data.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# check-postgres-data.sh — refuse a kind cluster whose Postgres data directory +# was initialized by another PostgreSQL major version than the one the +# `space-postgres` chart in this checkout runs. +# +# Earlier revisions of the chart ran PostgreSQL 16 without TLS and with a +# superuser connection string. PostgreSQL 17 cannot start on a 16 data +# directory, so re-applying the chart crash-loops the pod and the bring-up +# waits out its rollout timeout; and until it is re-applied the old server +# keeps running without TLS, so the harness's sslmode=require psql fails. The +# role setup the pod's init script provides (templates/configmap.yaml) runs +# only when the image initializes an empty data directory, so the database is +# reset rather than upgraded in place. Every bring-up path stops here instead. +# See "Recovering from an unbundled migration version" in +# infra/space/kind/README.md. +# +# The version is the data directory's own PG_VERSION file, read on the node +# that holds the volume rather than from the server, so the answer is the same +# whether the old server is running, the new one is crash-looping on the old +# data, or no pod runs at all. +# +# Read-only. Exits 0 when the cluster, the volume or its PG_VERSION does not +# exist yet, or the data directory matches the chart's major version; 1 (with +# the remedy on stderr) otherwise, or when the volume cannot be read. +# +# Usage: +# bash infra/space/kind/scripts/check-postgres-data.sh [cluster] +# +# The cluster defaults to $SPACE_KIND_CLUSTER, then space-local. +set -euo pipefail + +KIND_CLUSTER="${1:-${SPACE_KIND_CLUSTER:-space-local}}" +NAMESPACE=space +CHART="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/charts/space-postgres" +# The StatefulSet's volumeClaimTemplate (`data`) and PGDATA, a subdirectory of +# that volume (templates/statefulset.yaml). +PVC=data-space-postgres-0 +PGDATA_SUBDIR=pgdata + +kind get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER" || exit 0 + +want="$(awk -F'"' '/^appVersion:/ { print $2 }' "$CHART/Chart.yaml")" +[[ -n "$want" ]] || { printf 'ERROR: no appVersion in %s/Chart.yaml\n' "$CHART" >&2; exit 1; } + +cannot_read() { + printf 'ERROR: cannot read the space-postgres data directory version in kind cluster %s:\n%s\n' \ + "$KIND_CLUSTER" "$1" >&2 + exit 1 +} + +# No claim yet, or one not yet bound: the chart will initialize an empty one. +if ! pv="$(kubectl --context "kind-$KIND_CLUSTER" -n "$NAMESPACE" \ + get pvc "$PVC" --ignore-not-found -o jsonpath='{.spec.volumeName}' 2>&1)"; then + cannot_read "$pv" +fi +[[ -n "$pv" ]] || exit 0 + +# Kind's default StorageClass (local-path) provisions a directory on one node +# and pins the volume to that node, whose container has the node's name. +if ! location="$(kubectl --context "kind-$KIND_CLUSTER" get pv "$pv" -o jsonpath=\ +'{.spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].values[0]} {.spec.hostPath.path}{.spec.local.path}' 2>&1)"; then + cannot_read "$location" +fi +read -r node path <<<"$location" +[[ -n "${node:-}" && -n "${path:-}" ]] \ + || cannot_read "persistent volume $pv names no node and host path ('$location')" +if ! have="$(docker exec "$node" sh -c '[ ! -e "$1" ] || cat "$1"' sh \ + "$path/$PGDATA_SUBDIR/PG_VERSION" 2>&1)"; then + cannot_read "$have" +fi +# initdb has not written it yet: nothing to be incompatible with. +[[ -n "$have" ]] || exit 0 +[[ "$have" == "$want" ]] && exit 0 + +kctl="kubectl --context kind-$KIND_CLUSTER -n $NAMESPACE" +env_prefix="" +[[ "$KIND_CLUSTER" == space-local ]] || env_prefix="SPACE_KIND_CLUSTER=$KIND_CLUSTER " +{ + printf 'ERROR: the space-postgres data directory in kind cluster %s was initialized\n' "$KIND_CLUSTER" + printf ' by PostgreSQL %s; this checkout runs PostgreSQL %s, which cannot start on\n' "$have" "$want" + printf ' it, with TLS and a non-superuser master user whose roles the pod creates\n' + printf ' only on an empty data directory. Reset the database; this deletes the\n' + printf ' control-plane state (sandbox records, templates, API keys) and keeps the\n' + printf ' cluster and its image caches:\n' + printf ' %s delete statefulset space-postgres\n' "$kctl" + printf ' %s delete pvc %s\n' "$kctl" "$PVC" + printf ' %s delete job space-db-migrate space-seed-publish --ignore-not-found\n' "$kctl" + printf ' then bring the stack up from this checkout again:\n' + printf ' %sbash infra/space/kind/scripts/bootstrap.sh\n' "$env_prefix" + printf ' (or re-run the smoke suite without SKIP_BOOTSTRAP, or "tilt up"; a Tilt\n' + printf ' that is already running reloads with: tilt trigger "(Tiltfile)").\n' + printf ' Recreating the cluster (teardown.sh, then bootstrap.sh) also works.\n' + printf ' Nothing was changed. See "Recovering from an unbundled migration\n' + printf ' version" in infra/space/kind/README.md.\n' +} >&2 +exit 1 diff --git a/scripts/tests/lib/bootstrap-kind-stack.sh b/scripts/tests/lib/bootstrap-kind-stack.sh index aa75b5240c..17be5b190d 100755 --- a/scripts/tests/lib/bootstrap-kind-stack.sh +++ b/scripts/tests/lib/bootstrap-kind-stack.sh @@ -653,10 +653,14 @@ kind_stack_psql_resolve_creds() { [[ -n "$KIND_STACK_PG_USER" && -n "$KIND_STACK_PG_DB" ]] } +# Over loopback TLS rather than the pod's Unix socket: the server refuses +# unencrypted TCP and the socket cannot encrypt, so sessions from this helper +# show up encrypted in pg_stat_ssl like every other client's. kind_stack_psql() { kind_stack_psql_resolve_creds || return 1 kubectl --context "kind-${KIND_CLUSTER}" -n "${NAMESPACE:-space}" \ exec -i sts/space-postgres -- \ + env PGHOST=127.0.0.1 PGSSLMODE=require \ psql -U "$KIND_STACK_PG_USER" -d "$KIND_STACK_PG_DB" -tA -v ON_ERROR_STOP=1 "$@" } @@ -841,6 +845,18 @@ kind_stack_assert_cluster_mounts() { rm -f "$expected" } +# Refuse a cluster whose Postgres data directory an earlier space-postgres +# chart initialized (PostgreSQL 16), before anything reads or applies it. +# Beside the mount check, and for the same reason: under SKIP_BOOTSTRAP=1 such +# a bench still runs the old server, which has no TLS, and the first +# sslmode=require `kind_stack_psql` would fail without naming the cause. +kind_stack_assert_postgres_data() { + # No cluster, nothing stale. Tested first, as the mount check does, so a + # box with no cluster never resolves the script's path. + kind get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER" || return 0 + bash "$REPO_ROOT/infra/space/kind/scripts/check-postgres-data.sh" "$KIND_CLUSTER" || exit 1 +} + # Detached application bring-up: cluster created, Tilt started in its own # session and its selected context verified, then return. Initial profile # reconciliation still completes before that context is registered. Registration @@ -857,12 +873,14 @@ kind_stack_assert_cluster_mounts() { kind_stack_start_detached() { kind_stack_assert_smoke_prereqs kind_stack_assert_cluster_mounts + kind_stack_assert_postgres_data KIND_STACK_DETACH=1 bootstrap_kind_stack__bring_up } bootstrap_kind_stack() { kind_stack_assert_smoke_prereqs kind_stack_assert_cluster_mounts + kind_stack_assert_postgres_data bootstrap_kind_stack__bring_up # Outside the bring-up, so it also runs under SKIP_BOOTSTRAP=1 — the mode # every devbox run uses once the stack is up, and therefore the mode in @@ -919,6 +937,7 @@ bootstrap_kind_stack__load_mirrored_chart_images() { kind_stack_prepare_cluster() { kind_stack_assert_smoke_prereqs kind_stack_assert_cluster_mounts + kind_stack_assert_postgres_data bootstrap_kind_stack__assert_port_owner bootstrap_kind_stack__prepare_cluster } diff --git a/scripts/tests/run-local-smokes.sh b/scripts/tests/run-local-smokes.sh index 294027277a..f8bd1e6bf1 100755 --- a/scripts/tests/run-local-smokes.sh +++ b/scripts/tests/run-local-smokes.sh @@ -186,7 +186,9 @@ seed_e2b_api_key() { local hash; hash="$(sha256_hex "$E2B_API_KEY")" log "seeding local-dev E2B api key (hash ${hash:0:12}…)" # space-postgres is a single-pod StatefulSet; address the pod directly. + # Loopback TLS, as kind_stack_psql connects: the server refuses unencrypted TCP. kubectl --context "$KCTX" -n "$NS" exec -i space-postgres-0 -- \ + env PGHOST=127.0.0.1 PGSSLMODE=require \ psql -U spaceuser -d spacedb -v ON_ERROR_STOP=1 >/dev/null < None: "-i", "space-postgres-0", "--", + # Loopback TLS, as kind_stack_psql connects: the server refuses + # unencrypted TCP. + "env", + "PGHOST=127.0.0.1", + "PGSSLMODE=require", "psql", "-U", "spaceuser", diff --git a/space_smoke/adapters.py b/space_smoke/adapters.py index 5da3135910..e2b865ab69 100644 --- a/space_smoke/adapters.py +++ b/space_smoke/adapters.py @@ -944,6 +944,11 @@ def _psql(self, sql: str, *, action: str) -> str: "-i", "space-postgres-0", "--", + # Loopback TLS, as kind_stack_psql connects: the server refuses + # unencrypted TCP. + "env", + "PGHOST=127.0.0.1", + "PGSSLMODE=require", "psql", "-U", "spaceuser",