feat(observability): deployment (OTel collector + Parca/eBPF)

.github/workflows/create-release-candidate.yml

@@ -174,7 +174,7 @@ jobs:
 
           # Export the single-node directory from the tagged source
           mkdir -p "$TEMP_DIR/_raw"
-          git archive --format=tar "$TAG_NAME" -- bin/single-node/ sql/ NOTICE licenses/LICENSE-BSD-3-Clause.txt | tar -x -C "$TEMP_DIR/_raw"
+          git archive --format=tar "$TAG_NAME" -- bin/single-node/ bin/observability/ sql/ NOTICE licenses/LICENSE-BSD-3-Clause.txt | tar -x -C "$TEMP_DIR/_raw"
 
           # Copy deployment files
           cp "$TEMP_DIR/_raw/bin/single-node/docker-compose.yml" "$BUNDLE_DIR/"
@@ -187,10 +187,19 @@ jobs:
           cp "$TEMP_DIR/_raw/licenses/LICENSE-BSD-3-Clause.txt" "$BUNDLE_DIR/licenses/"
           cp -r "$TEMP_DIR/_raw/sql" "$BUNDLE_DIR/"
 
+          # Observability configs (OTel collector + Parca) referenced by the
+          # opt-in observability compose profiles. Shipped next to the compose
+          # file so the ../observability mounts resolve in the flattened bundle.
+          cp -r "$TEMP_DIR/_raw/bin/observability" "$BUNDLE_DIR/observability"
+
           # Patch the SQL mount path for the self-contained bundle layout
           # In the repo it's ../../sql (relative to bin/single-node/), in the bundle it's ./sql
           sed -i 's|../../sql|./sql|g' "$BUNDLE_DIR/docker-compose.yml"
 
+          # Patch the observability mount paths: ../observability (relative to
+          # bin/single-node/ in the repo) becomes ./observability in the bundle.
+          sed -i 's|../observability|./observability|g' "$BUNDLE_DIR/docker-compose.yml"
+
           # Generate a release-pinned .env file with the version tag
           # Start from the source .env and ensure IMAGE_REGISTRY and IMAGE_TAG are set
           cp "$TEMP_DIR/_raw/bin/single-node/.env" "$BUNDLE_DIR/.env"

.licenserc.yaml

@@ -37,6 +37,8 @@ header:
     - '**/.env.example'
     - '**/.prettierrc'
     - '**/bun.lock'
+    # Helm's generated dependency lock file (no header; like yarn.lock/bun.lock)
+    - '**/Chart.lock'
     # Third-party code with MIT license - see LICENSE file for attribution
     - 'common/workflow-operator/src/main/scala/com/kjetland/**'
     # TypeFox monaco-languageclient derived files (MIT License)

bin/k8s/Chart.lock

@@ -0,0 +1,21 @@
+dependencies:
+- name: postgresql
+  repository: https://charts.bitnami.com/bitnami
+  version: 16.5.6
+- name: minio
+  repository: https://charts.bitnami.com/bitnami
+  version: 15.0.7
+- name: lakefs
+  repository: https://charts.lakefs.io
+  version: 1.8.1
+- name: gateway-helm
+  repository: oci://docker.io/envoyproxy
+  version: 1.6.3
+- name: lakekeeper
+  repository: https://lakekeeper.github.io/lakekeeper-charts/
+  version: 0.9.0
+- name: metrics-server
+  repository: https://kubernetes-sigs.github.io/metrics-server/
+  version: 3.12.2
+digest: sha256:031184824e3bd3e03883a3debe64459f462610e4f5bd7c147f03fd05333b169e
+generated: "2026-07-11T11:09:44.800500493-07:00"

bin/k8s/install.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# One-command install/upgrade of the Texera Helm chart.
+#
+# `helm install` on its own fails on a fresh checkout because the subchart
+# dependencies (postgresql, minio, lakefs, envoy-gateway, lakekeeper,
+# metrics-server) are declared in Chart.yaml but not vendored into charts/.
+# This wrapper fetches them first (idempotent), then upgrades-or-installs.
+#
+# Usage (run from anywhere):
+#   bin/k8s/install.sh                                  # app only
+#   bin/k8s/install.sh --set observability.enabled=true # app + observability
+#   RELEASE=texera NAMESPACE=texera bin/k8s/install.sh --set observability.enabled=true
+#
+# Any extra args are forwarded verbatim to `helm upgrade --install`, so
+# --set / -f values.override.yaml / --namespace etc. all work.
+
+set -euo pipefail
+
+CHART_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RELEASE="${RELEASE:-texera}"
+NAMESPACE="${NAMESPACE:-texera}"
+
+command -v helm >/dev/null 2>&1 || { echo "helm not found on PATH" >&2; exit 1; }
+
+# Fetch/refresh subcharts into charts/. `dependency build` uses Chart.lock when
+# present (reproducible); it falls back to `dependency update` if there is no
+# lock yet. Both are safe to re-run.
+echo "==> Resolving chart dependencies"
+if [ -f "${CHART_DIR}/Chart.lock" ]; then
+  helm dependency build "${CHART_DIR}"
+else
+  helm dependency update "${CHART_DIR}"
+fi
+
+echo "==> helm upgrade --install ${RELEASE} (namespace: ${NAMESPACE})"
+exec helm upgrade --install "${RELEASE}" "${CHART_DIR}" \
+  --namespace "${NAMESPACE}" \
+  --create-namespace \
+  "$@"

bin/k8s/templates/base/_helpers.tpl

@@ -55,6 +55,26 @@ services fall back to the in-cluster RustFS Service and its auto-generated
 {{- if .Values.storage.s3.endpoint -}}secret-access-key{{- else -}}RUSTFS_SECRET_KEY{{- end -}}
 {{- end -}}
 
+{{/*
+Observability emission env for a Scala service pod.
+
+Renders OTEL_EXPORTER_OTLP_ENDPOINT (pointing at the in-cluster OTel
+Collector Service) plus TEXERA_OTEL_ALLOWED_HOSTS. OtelInit validates the
+endpoint host against an allowlist whose default is localhost only, so the
+collector's Service name must be added explicitly or the SDK rejects it and
+emits nothing. Renders nothing unless both the observability stack and the
+collector are enabled, so the default install is unchanged. Include inside a
+container's `env:` list, e.g. `{{- include "texera.observability.env" . | nindent 12 }}`.
+*/}}
+{{- define "texera.observability.env" -}}
+{{- if and .Values.observability.enabled .Values.observability.collector.enabled }}
+- name: OTEL_EXPORTER_OTLP_ENDPOINT
+  value: "http://{{ .Release.Name }}-otel-collector:{{ .Values.observability.collector.grpcPort }}"
+- name: TEXERA_OTEL_ALLOWED_HOSTS
+  value: "{{ .Release.Name }}-otel-collector"
+{{- end }}
+{{- end -}}
+
 {{/*
 The audience the mounter's service-account tokens are bound to. Fixed rather than
 configurable: it is one half of a credential contract between access-control-service and the
@@ -83,3 +103,81 @@ append the uid to it, so a bare value would render "http://<origin>jupyter/7". *
 {{- printf "/%s" (trimAll "/" .Values.jupyterPool.basePath) -}}
 {{- end -}}
 
+{{/* Grafana datasource provisioning body. Kept as a helper so the Deployment's
+checksum/config annotation can hash just this content; hashing the template that
+holds the annotation would make it reference itself and recurse. */}}
+{{- define "texera.observability.grafanaDatasources" -}}
+apiVersion: 1
+datasources:
+  # Metrics: VictoriaMetrics speaks the Prometheus query API. Default.
+  - name: Metrics
+    type: prometheus
+    uid: metrics
+    access: proxy
+    url: http://{{ .Release.Name }}-victoriametrics:{{ .Values.observability.metrics.port }}
+    isDefault: true
+    editable: false
+{{- if .Values.observability.logs.enabled }}
+  # Logs: Loki, LogQL over OTLP-ingested logs.
+  - name: Loki
+    type: loki
+    uid: loki
+    access: proxy
+    url: http://{{ .Release.Name }}-loki:{{ .Values.observability.logs.port }}
+    editable: false
+{{- end }}
+{{- if .Values.observability.traces.enabled }}
+  # Traces: Jaeger query API / UI backend.
+  - name: Jaeger
+    type: jaeger
+    uid: jaeger
+    access: proxy
+    url: http://{{ .Release.Name }}-jaeger:{{ .Values.observability.traces.queryPort }}
+    editable: false
+{{- end }}
+{{- end -}}
+
+{{/* Loki server config body. A helper for the same reason as the Grafana one,
+and so the checksum/config annotation hashes the config instead of its template.
+Retention is enforced here: the compactor deletes chunks older than
+observability.logs.retentionPeriod, which Loki otherwise ignores. */}}
+{{- define "texera.observability.lokiConfig" -}}
+# No multi-tenancy: one implicit tenant for the in-cluster stack.
+auth_enabled: false
+server:
+  http_listen_port: {{ .Values.observability.logs.port }}
+  grpc_listen_port: 9096
+common:
+  instance_addr: 127.0.0.1
+  # Every component's data path defaults under this prefix, the PVC at /loki.
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+limits_config:
+  # Required for OTLP ingest (OTLP attributes -> structured metadata).
+  allow_structured_metadata: true
+  reject_old_samples: false
+  # Age after which log chunks are eligible for deletion by the compactor.
+  retention_period: {{ .Values.observability.logs.retentionPeriod }}
+compactor:
+  working_directory: /loki/compactor
+  # Without this the retention_period above is parsed but never enforced,
+  # so logs accumulate until the volume fills.
+  retention_enabled: true
+  delete_request_store: filesystem
+{{- end -}}

bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml

@@ -89,6 +89,7 @@ spec:
             - name: {{ .name }}
               value: "{{ .value }}"
             {{- end }}
+            {{- include "texera.observability.env" . | nindent 12 }}
           livenessProbe:
             httpGet:
               path: /api/healthcheck

bin/k8s/templates/base/config-service/config-service-deployment.yaml

@@ -59,6 +59,7 @@ spec:
             - name: {{ .name }}
               value: "{{ .value }}"
             {{- end }}
+            {{- include "texera.observability.env" . | nindent 12 }}
           livenessProbe:
             httpGet:
               path: /api/healthcheck

bin/k8s/templates/base/file-service/file-service-deployment.yaml

@@ -82,4 +82,5 @@ spec:
             {{- range .Values.texeraEnvVars }}
             - name: {{ .name }}
               value: "{{ .value }}"
-            {{- end }}
+            {{- end }}
+            {{- include "texera.observability.env" . | nindent 12 }}

bin/k8s/templates/base/notebook-migration-service/notebook-migration-service-deployment.yaml

@@ -90,6 +90,7 @@ spec:
             - name: {{ .name }}
               value: "{{ .value }}"
             {{- end }}
+            {{- include "texera.observability.env" . | nindent 12 }}
           livenessProbe:
             httpGet:
               path: /api/healthcheck

bin/k8s/templates/base/observability/grafana.yaml

@@ -0,0 +1,125 @@
+{{/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/}}
+
+{{- if and .Values.observability.enabled .Values.observability.grafana.enabled }}
+# Grafana: dashboard/UI, k8s counterpart of the docker-compose grafana
+# service. Reads the Apache-2.0 backends (Loki, VictoriaMetrics, Jaeger)
+# as pre-provisioned datasources via the ConfigMap below. Grafana is
+# AGPLv3, so it is an optional operator-run component, not part of any
+# Texera release artifact (see bin/observability/README.md). Only this
+# ASF-licensed template lives in the tree. The Service is ClusterIP (no
+# Ingress), matching the rest of the stack; expose it via your own
+# Ingress / port-forward to reach the UI.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-grafana-datasources
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Release.Name }}-grafana
+data:
+  datasources.yaml: |
+    {{- include "texera.observability.grafanaDatasources" . | nindent 4 }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-grafana
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Release.Name }}-grafana
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-grafana
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-grafana
+      annotations:
+        # Roll the pod when the provisioned datasources change.
+        checksum/config: {{ include "texera.observability.grafanaDatasources" . | sha256sum }}
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 472
+        fsGroup: 472
+      containers:
+        - name: grafana
+          image: {{ .Values.observability.grafana.image }}
+          imagePullPolicy: {{ .Values.texeraImages.pullPolicy }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          env:
+            # Admin login. Override for any environment reachable beyond
+            # an internal port-forward.
+            - name: GF_SECURITY_ADMIN_USER
+              value: {{ .Values.observability.grafana.adminUser | quote }}
+            - name: GF_SECURITY_ADMIN_PASSWORD
+              value: {{ .Values.observability.grafana.adminPassword | quote }}
+            - name: GF_AUTH_ANONYMOUS_ENABLED
+              value: "false"
+            - name: GF_USERS_ALLOW_SIGN_UP
+              value: "false"
+          ports:
+            - name: http
+              containerPort: {{ .Values.observability.grafana.port }}
+          volumeMounts:
+            - name: datasources
+              mountPath: /etc/grafana/provisioning/datasources
+              readOnly: true
+            - name: data
+              mountPath: /var/lib/grafana
+          livenessProbe:
+            httpGet:
+              path: /api/health
+              port: {{ .Values.observability.grafana.port }}
+            initialDelaySeconds: 15
+            periodSeconds: 15
+          resources:
+            {{- toYaml .Values.observability.grafana.resources | nindent 12 }}
+      volumes:
+        - name: datasources
+          configMap:
+            name: {{ .Release.Name }}-grafana-datasources
+        # Grafana's SQLite state. emptyDir: datasources are re-provisioned
+        # on every start, so no PVC is required for the reference stack.
+        - name: data
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-grafana
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Release.Name }}-grafana
+spec:
+  type: ClusterIP
+  selector:
+    app: {{ .Release.Name }}-grafana
+  ports:
+    - name: http
+      protocol: TCP
+      port: {{ .Values.observability.grafana.port }}
+      targetPort: {{ .Values.observability.grafana.port }}
+{{- end }}