From 5841c2ef5a6d5f3369fe29f5d4ba250dc63e74ea Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 9 Oct 2026 11:00:24 -0700 Subject: [PATCH] feat(observability): deployment (collector, Grafana/Loki, Parca) Add the local deployment wiring that receives, stores, and visualizes the telemetry emitted by PR 2: an OpenTelemetry Collector, per-signal backends, and a Grafana UI, for both the single-node docker-compose stack and the Kubernetes Helm chart. Infrastructure only; the application runs unchanged whether or not these services are started. This is PR 3 of the observability stack. Single-node compose (bin/single-node, bin/observability): - otel-collector as the single OTLP ingress, routing logs to Loki over OTLP HTTP, metrics to VictoriaMetrics via prometheusremotewrite, and traces to Jaeger over OTLP; the collector config caps OTLP message size and bounds memory via memory_limiter. - Grafana as the dashboard/UI, pre-provisioned (bin/observability/grafana/provisioning) with datasources for Loki, VictoriaMetrics (Prometheus-compatible query API), and Jaeger, so operators get log search, metric dashboards, and trace views without a Texera-built UI. - Loki as the log store (bin/observability/loki/config.yaml, OTLP ingest on /otlp), VictoriaMetrics as the metric store, Jaeger as the trace store, and Parca server plus eBPF agent for continuous profiling (bin/observability/parca). - A one-shot victoriametrics-init container chowns the metrics volume to VictoriaMetrics's non-root uid before it starts, so a freshly created (root-owned) named volume does not crash-loop the read-only VM container; it is idempotent on an existing volume. - Each signal is its own compose profile and is OFF by default: COMPOSE_PROFILES is empty, so observability is opt-in, and Grafana has its own observability-grafana profile. - Every observability host port, including the Grafana UI, binds to loopback (127.0.0.1); nothing is published on 0.0.0.0. Emission is gated by OTEL_SDK_DISABLED (default disabled), so nothing is emitted unless an operator opts in alongside the collector profile. Kubernetes Helm chart (bin/k8s): - Templates for the collector, Grafana, Loki, Jaeger, and VictoriaMetrics under templates/base/observability, plus values.yaml knobs and an install.sh helper. Grafana is a ClusterIP service with its datasources provisioned from a ConfigMap. - Service deployments gain the OTLP endpoint wiring. Licensing: the Collector, VictoriaMetrics, Jaeger, and Parca are Apache-2.0 and form the shipped reference stack. Grafana and Loki are AGPLv3 optional operator-run components, documented as such in bin/observability/README.md and not bundled in any Texera release artifact. Docs: bin/observability/README.md describes the stack, how to reach Grafana at http://localhost:3000 with the pre-provisioned datasources, and how to point an external Grafana or OpenSearch at the same endpoints instead. Tests: ObservabilityComposeSpec and ParcaConfigSpec are string-level smoke tests over the compose, collector, and Parca config. They pin the image versions, assert the opt-in default and loopback binding for every port, and check the collector routes logs to Loki, metrics to VictoriaMetrics, and traces to Jaeger. bin/observability/smoke_test.py is a runtime routing check meant to run against a live stack: it pushes a log, a metric, and a trace per service through the collector over OTLP and asserts each reaches its correct backend (Loki, VictoriaMetrics, Jaeger) attributed to the emitting service, plus an isolation check that one service's data is not mis-attributed to another; it exits non-zero on a routing regression. Stacked on PR 2 (obs/pr2/backend-emit). --- .../workflows/create-release-candidate.yml | 11 +- .licenserc.yaml | 2 + bin/k8s/Chart.lock | 21 ++ bin/k8s/install.sh | 56 ++++ bin/k8s/templates/base/_helpers.tpl | 98 +++++++ .../access-control-service-deployment.yaml | 1 + .../config-service-deployment.yaml | 1 + .../file-service/file-service-deployment.yaml | 3 +- ...notebook-migration-service-deployment.yaml | 1 + .../templates/base/observability/grafana.yaml | 125 +++++++++ .../templates/base/observability/jaeger.yaml | 81 ++++++ .../templates/base/observability/loki.yaml | 136 ++++++++++ .../otel-collector-configmap.yaml | 109 ++++++++ .../base/observability/otel-collector.yaml | 95 +++++++ .../base/observability/victoriametrics.yaml | 112 ++++++++ .../base/webserver/webserver-deployment.yaml | 3 +- ...workflow-compiling-service-deployment.yaml | 3 +- ...low-computing-unit-manager-deployment.yaml | 3 +- bin/k8s/values.yaml | 81 ++++++ bin/local-dev/docker-compose.override.yml | 8 + bin/observability/README.md | 131 +++++++++ bin/observability/docker-compose.yml | 39 +++ .../provisioning/datasources/datasources.yaml | 51 ++++ bin/observability/loki/config.yaml | 84 ++++++ bin/observability/otel-collector/config.yaml | 132 +++++++++ bin/observability/parca/README.md | 72 +++++ bin/observability/parca/parca-agent.env | 60 +++++ bin/observability/parca/parca.yaml | 42 +++ bin/observability/smoke_test.py | 207 +++++++++++++++ bin/single-node/.env | 65 ++++- .../docker-compose.observability.yml | 50 ++++ bin/single-node/docker-compose.override.yml | 29 ++ bin/single-node/docker-compose.yml | 250 +++++++++++++++++- bin/single-node/main.sh | 76 ++++-- .../ObservabilityComposeSpec.scala | 210 +++++++++++++++ .../observability/ParcaConfigSpec.scala | 129 +++++++++ .../ComputingUnitManagingResource.scala | 42 ++- .../ComputingUnitManagingResourceSpec.scala | 37 +++ 38 files changed, 2611 insertions(+), 45 deletions(-) create mode 100644 bin/k8s/Chart.lock create mode 100755 bin/k8s/install.sh create mode 100644 bin/k8s/templates/base/observability/grafana.yaml create mode 100644 bin/k8s/templates/base/observability/jaeger.yaml create mode 100644 bin/k8s/templates/base/observability/loki.yaml create mode 100644 bin/k8s/templates/base/observability/otel-collector-configmap.yaml create mode 100644 bin/k8s/templates/base/observability/otel-collector.yaml create mode 100644 bin/k8s/templates/base/observability/victoriametrics.yaml create mode 100644 bin/observability/README.md create mode 100644 bin/observability/docker-compose.yml create mode 100644 bin/observability/grafana/provisioning/datasources/datasources.yaml create mode 100644 bin/observability/loki/config.yaml create mode 100644 bin/observability/otel-collector/config.yaml create mode 100644 bin/observability/parca/README.md create mode 100644 bin/observability/parca/parca-agent.env create mode 100644 bin/observability/parca/parca.yaml create mode 100644 bin/observability/smoke_test.py create mode 100644 bin/single-node/docker-compose.observability.yml create mode 100644 bin/single-node/docker-compose.override.yml create mode 100644 common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala diff --git a/.github/workflows/create-release-candidate.yml b/.github/workflows/create-release-candidate.yml index 1e12d228083..d73b1df6dda 100644 --- a/.github/workflows/create-release-candidate.yml +++ b/.github/workflows/create-release-candidate.yml @@ -174,7 +174,7 @@ jobs: # Export the single-node directory from the tagged source mkdir -p "$TEMP_DIR/_raw" - git archive --format=tar "$TAG_NAME" -- bin/single-node/ sql/ NOTICE licenses/LICENSE-BSD-3-Clause.txt | tar -x -C "$TEMP_DIR/_raw" + git archive --format=tar "$TAG_NAME" -- bin/single-node/ bin/observability/ sql/ NOTICE licenses/LICENSE-BSD-3-Clause.txt | tar -x -C "$TEMP_DIR/_raw" # Copy deployment files cp "$TEMP_DIR/_raw/bin/single-node/docker-compose.yml" "$BUNDLE_DIR/" @@ -187,10 +187,19 @@ jobs: cp "$TEMP_DIR/_raw/licenses/LICENSE-BSD-3-Clause.txt" "$BUNDLE_DIR/licenses/" cp -r "$TEMP_DIR/_raw/sql" "$BUNDLE_DIR/" + # Observability configs (OTel collector + Parca) referenced by the + # opt-in observability compose profiles. Shipped next to the compose + # file so the ../observability mounts resolve in the flattened bundle. + cp -r "$TEMP_DIR/_raw/bin/observability" "$BUNDLE_DIR/observability" + # Patch the SQL mount path for the self-contained bundle layout # In the repo it's ../../sql (relative to bin/single-node/), in the bundle it's ./sql sed -i 's|../../sql|./sql|g' "$BUNDLE_DIR/docker-compose.yml" + # Patch the observability mount paths: ../observability (relative to + # bin/single-node/ in the repo) becomes ./observability in the bundle. + sed -i 's|../observability|./observability|g' "$BUNDLE_DIR/docker-compose.yml" + # Generate a release-pinned .env file with the version tag # Start from the source .env and ensure IMAGE_REGISTRY and IMAGE_TAG are set cp "$TEMP_DIR/_raw/bin/single-node/.env" "$BUNDLE_DIR/.env" diff --git a/.licenserc.yaml b/.licenserc.yaml index cbef454e403..0cebbbb3ca5 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -37,6 +37,8 @@ header: - '**/.env.example' - '**/.prettierrc' - '**/bun.lock' + # Helm's generated dependency lock file (no header; like yarn.lock/bun.lock) + - '**/Chart.lock' # Third-party code with MIT license - see LICENSE file for attribution - 'common/workflow-operator/src/main/scala/com/kjetland/**' # TypeFox monaco-languageclient derived files (MIT License) diff --git a/bin/k8s/Chart.lock b/bin/k8s/Chart.lock new file mode 100644 index 00000000000..af90dd0b29b --- /dev/null +++ b/bin/k8s/Chart.lock @@ -0,0 +1,21 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 16.5.6 +- name: minio + repository: https://charts.bitnami.com/bitnami + version: 15.0.7 +- name: lakefs + repository: https://charts.lakefs.io + version: 1.8.1 +- name: gateway-helm + repository: oci://docker.io/envoyproxy + version: 1.6.3 +- name: lakekeeper + repository: https://lakekeeper.github.io/lakekeeper-charts/ + version: 0.9.0 +- name: metrics-server + repository: https://kubernetes-sigs.github.io/metrics-server/ + version: 3.12.2 +digest: sha256:031184824e3bd3e03883a3debe64459f462610e4f5bd7c147f03fd05333b169e +generated: "2026-07-11T11:09:44.800500493-07:00" diff --git a/bin/k8s/install.sh b/bin/k8s/install.sh new file mode 100755 index 00000000000..d1ed9b55e6e --- /dev/null +++ b/bin/k8s/install.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# One-command install/upgrade of the Texera Helm chart. +# +# `helm install` on its own fails on a fresh checkout because the subchart +# dependencies (postgresql, minio, lakefs, envoy-gateway, lakekeeper, +# metrics-server) are declared in Chart.yaml but not vendored into charts/. +# This wrapper fetches them first (idempotent), then upgrades-or-installs. +# +# Usage (run from anywhere): +# bin/k8s/install.sh # app only +# bin/k8s/install.sh --set observability.enabled=true # app + observability +# RELEASE=texera NAMESPACE=texera bin/k8s/install.sh --set observability.enabled=true +# +# Any extra args are forwarded verbatim to `helm upgrade --install`, so +# --set / -f values.override.yaml / --namespace etc. all work. + +set -euo pipefail + +CHART_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RELEASE="${RELEASE:-texera}" +NAMESPACE="${NAMESPACE:-texera}" + +command -v helm >/dev/null 2>&1 || { echo "helm not found on PATH" >&2; exit 1; } + +# Fetch/refresh subcharts into charts/. `dependency build` uses Chart.lock when +# present (reproducible); it falls back to `dependency update` if there is no +# lock yet. Both are safe to re-run. +echo "==> Resolving chart dependencies" +if [ -f "${CHART_DIR}/Chart.lock" ]; then + helm dependency build "${CHART_DIR}" +else + helm dependency update "${CHART_DIR}" +fi + +echo "==> helm upgrade --install ${RELEASE} (namespace: ${NAMESPACE})" +exec helm upgrade --install "${RELEASE}" "${CHART_DIR}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + "$@" diff --git a/bin/k8s/templates/base/_helpers.tpl b/bin/k8s/templates/base/_helpers.tpl index fe25aab1b72..8f310823851 100644 --- a/bin/k8s/templates/base/_helpers.tpl +++ b/bin/k8s/templates/base/_helpers.tpl @@ -55,6 +55,26 @@ services fall back to the in-cluster RustFS Service and its auto-generated {{- if .Values.storage.s3.endpoint -}}secret-access-key{{- else -}}RUSTFS_SECRET_KEY{{- end -}} {{- end -}} +{{/* +Observability emission env for a Scala service pod. + +Renders OTEL_EXPORTER_OTLP_ENDPOINT (pointing at the in-cluster OTel +Collector Service) plus TEXERA_OTEL_ALLOWED_HOSTS. OtelInit validates the +endpoint host against an allowlist whose default is localhost only, so the +collector's Service name must be added explicitly or the SDK rejects it and +emits nothing. Renders nothing unless both the observability stack and the +collector are enabled, so the default install is unchanged. Include inside a +container's `env:` list, e.g. `{{- include "texera.observability.env" . | nindent 12 }}`. +*/}} +{{- define "texera.observability.env" -}} +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://{{ .Release.Name }}-otel-collector:{{ .Values.observability.collector.grpcPort }}" +- name: TEXERA_OTEL_ALLOWED_HOSTS + value: "{{ .Release.Name }}-otel-collector" +{{- end }} +{{- end -}} + {{/* The audience the mounter's service-account tokens are bound to. Fixed rather than configurable: it is one half of a credential contract between access-control-service and the @@ -83,3 +103,81 @@ append the uid to it, so a bare value would render "http://jupyter/7". * {{- printf "/%s" (trimAll "/" .Values.jupyterPool.basePath) -}} {{- end -}} +{{/* Grafana datasource provisioning body. Kept as a helper so the Deployment's +checksum/config annotation can hash just this content; hashing the template that +holds the annotation would make it reference itself and recurse. */}} +{{- define "texera.observability.grafanaDatasources" -}} +apiVersion: 1 +datasources: + # Metrics: VictoriaMetrics speaks the Prometheus query API. Default. + - name: Metrics + type: prometheus + uid: metrics + access: proxy + url: http://{{ .Release.Name }}-victoriametrics:{{ .Values.observability.metrics.port }} + isDefault: true + editable: false +{{- if .Values.observability.logs.enabled }} + # Logs: Loki, LogQL over OTLP-ingested logs. + - name: Loki + type: loki + uid: loki + access: proxy + url: http://{{ .Release.Name }}-loki:{{ .Values.observability.logs.port }} + editable: false +{{- end }} +{{- if .Values.observability.traces.enabled }} + # Traces: Jaeger query API / UI backend. + - name: Jaeger + type: jaeger + uid: jaeger + access: proxy + url: http://{{ .Release.Name }}-jaeger:{{ .Values.observability.traces.queryPort }} + editable: false +{{- end }} +{{- end -}} + +{{/* Loki server config body. A helper for the same reason as the Grafana one, +and so the checksum/config annotation hashes the config instead of its template. +Retention is enforced here: the compactor deletes chunks older than +observability.logs.retentionPeriod, which Loki otherwise ignores. */}} +{{- define "texera.observability.lokiConfig" -}} +# No multi-tenancy: one implicit tenant for the in-cluster stack. +auth_enabled: false +server: + http_listen_port: {{ .Values.observability.logs.port }} + grpc_listen_port: 9096 +common: + instance_addr: 127.0.0.1 + # Every component's data path defaults under this prefix, the PVC at /loki. + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + # Required for OTLP ingest (OTLP attributes -> structured metadata). + allow_structured_metadata: true + reject_old_samples: false + # Age after which log chunks are eligible for deletion by the compactor. + retention_period: {{ .Values.observability.logs.retentionPeriod }} +compactor: + working_directory: /loki/compactor + # Without this the retention_period above is parsed but never enforced, + # so logs accumulate until the volume fills. + retention_enabled: true + delete_request_store: filesystem +{{- end -}} diff --git a/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml index 99419d1f549..be518a09717 100644 --- a/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml +++ b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml @@ -89,6 +89,7 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/base/config-service/config-service-deployment.yaml b/bin/k8s/templates/base/config-service/config-service-deployment.yaml index 7b86369f256..54fd1e99594 100644 --- a/bin/k8s/templates/base/config-service/config-service-deployment.yaml +++ b/bin/k8s/templates/base/config-service/config-service-deployment.yaml @@ -59,6 +59,7 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/base/file-service/file-service-deployment.yaml b/bin/k8s/templates/base/file-service/file-service-deployment.yaml index b60eddf11b5..e70f2d32135 100644 --- a/bin/k8s/templates/base/file-service/file-service-deployment.yaml +++ b/bin/k8s/templates/base/file-service/file-service-deployment.yaml @@ -82,4 +82,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/notebook-migration-service/notebook-migration-service-deployment.yaml b/bin/k8s/templates/base/notebook-migration-service/notebook-migration-service-deployment.yaml index 2d18f7dc6f4..6abaa3f28fd 100644 --- a/bin/k8s/templates/base/notebook-migration-service/notebook-migration-service-deployment.yaml +++ b/bin/k8s/templates/base/notebook-migration-service/notebook-migration-service-deployment.yaml @@ -90,6 +90,7 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/base/observability/grafana.yaml b/bin/k8s/templates/base/observability/grafana.yaml new file mode 100644 index 00000000000..36b1e05fe3e --- /dev/null +++ b/bin/k8s/templates/base/observability/grafana.yaml @@ -0,0 +1,125 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.grafana.enabled }} +# Grafana: dashboard/UI, k8s counterpart of the docker-compose grafana +# service. Reads the Apache-2.0 backends (Loki, VictoriaMetrics, Jaeger) +# as pre-provisioned datasources via the ConfigMap below. Grafana is +# AGPLv3, so it is an optional operator-run component, not part of any +# Texera release artifact (see bin/observability/README.md). Only this +# ASF-licensed template lives in the tree. The Service is ClusterIP (no +# Ingress), matching the rest of the stack; expose it via your own +# Ingress / port-forward to reach the UI. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-grafana-datasources + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-grafana +data: + datasources.yaml: | + {{- include "texera.observability.grafanaDatasources" . | nindent 4 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-grafana + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-grafana +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-grafana + template: + metadata: + labels: + app: {{ .Release.Name }}-grafana + annotations: + # Roll the pod when the provisioned datasources change. + checksum/config: {{ include "texera.observability.grafanaDatasources" . | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 472 + fsGroup: 472 + containers: + - name: grafana + image: {{ .Values.observability.grafana.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + env: + # Admin login. Override for any environment reachable beyond + # an internal port-forward. + - name: GF_SECURITY_ADMIN_USER + value: {{ .Values.observability.grafana.adminUser | quote }} + - name: GF_SECURITY_ADMIN_PASSWORD + value: {{ .Values.observability.grafana.adminPassword | quote }} + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + - name: GF_USERS_ALLOW_SIGN_UP + value: "false" + ports: + - name: http + containerPort: {{ .Values.observability.grafana.port }} + volumeMounts: + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + readOnly: true + - name: data + mountPath: /var/lib/grafana + livenessProbe: + httpGet: + path: /api/health + port: {{ .Values.observability.grafana.port }} + initialDelaySeconds: 15 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.grafana.resources | nindent 12 }} + volumes: + - name: datasources + configMap: + name: {{ .Release.Name }}-grafana-datasources + # Grafana's SQLite state. emptyDir: datasources are re-provisioned + # on every start, so no PVC is required for the reference stack. + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-grafana + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-grafana +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-grafana + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.grafana.port }} + targetPort: {{ .Values.observability.grafana.port }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/jaeger.yaml b/bin/k8s/templates/base/observability/jaeger.yaml new file mode 100644 index 00000000000..2821a977736 --- /dev/null +++ b/bin/k8s/templates/base/observability/jaeger.yaml @@ -0,0 +1,81 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.traces.enabled }} +# Jaeger v2: traces backend + UI, k8s counterpart of the docker-compose +# jaeger service. v2 accepts OTLP natively (the collector exports here on +# otlpPort). In-memory storage: a restart wipes traces, same as the +# single-node compose. Swap in a persistent storage backend for +# multi-node / production use. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-jaeger + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-jaeger +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-jaeger + template: + metadata: + labels: + app: {{ .Release.Name }}-jaeger + spec: + securityContext: + runAsNonRoot: true + containers: + - name: jaeger + image: {{ .Values.observability.traces.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: query + containerPort: {{ .Values.observability.traces.queryPort }} + - name: otlp-grpc + containerPort: {{ .Values.observability.traces.otlpPort }} + resources: + {{- toYaml .Values.observability.traces.resources | nindent 12 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-jaeger + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-jaeger +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-jaeger + ports: + - name: query + protocol: TCP + port: {{ .Values.observability.traces.queryPort }} + targetPort: {{ .Values.observability.traces.queryPort }} + - name: otlp-grpc + protocol: TCP + port: {{ .Values.observability.traces.otlpPort }} + targetPort: {{ .Values.observability.traces.otlpPort }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/loki.yaml b/bin/k8s/templates/base/observability/loki.yaml new file mode 100644 index 00000000000..d26ca944e28 --- /dev/null +++ b/bin/k8s/templates/base/observability/loki.yaml @@ -0,0 +1,136 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.logs.enabled }} +# Loki: log store, k8s counterpart of the docker-compose loki service. +# OTLP ingest (/otlp) + LogQL query API on one port. Loki is AGPLv3, so +# it is an optional operator-run component, not part of any Texera release +# artifact (see bin/observability/README.md). Only this ASF-licensed +# template lives in the tree. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-loki-config + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +data: + config.yaml: | + {{- include "texera.observability.lokiConfig" . | nindent 4 }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-loki-data + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +spec: + accessModes: ["ReadWriteOnce"] + {{- if .Values.observability.logs.storageClass }} + storageClassName: {{ .Values.observability.logs.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.observability.logs.storage }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-loki + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +spec: + replicas: 1 + # Single writer bound to one RWO volume; never run two at once. + strategy: + type: Recreate + selector: + matchLabels: + app: {{ .Release.Name }}-loki + template: + metadata: + labels: + app: {{ .Release.Name }}-loki + annotations: + # Roll the pod when the rendered config changes. + checksum/config: {{ include "texera.observability.lokiConfig" . | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + containers: + - name: loki + image: {{ .Values.observability.logs.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: ["-config.file=/etc/loki/config.yaml"] + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: http + containerPort: {{ .Values.observability.logs.port }} + volumeMounts: + - name: config + mountPath: /etc/loki + readOnly: true + - name: data + mountPath: /loki + # Read-only root filesystem, so scratch space is an emptyDir. + - name: tmp + mountPath: /tmp + livenessProbe: + httpGet: + path: /ready + port: {{ .Values.observability.logs.port }} + initialDelaySeconds: 15 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.logs.resources | nindent 12 }} + volumes: + - name: config + configMap: + name: {{ .Release.Name }}-loki-config + - name: data + persistentVolumeClaim: + claimName: {{ .Release.Name }}-loki-data + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-loki + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-loki + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.logs.port }} + targetPort: {{ .Values.observability.logs.port }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/otel-collector-configmap.yaml b/bin/k8s/templates/base/observability/otel-collector-configmap.yaml new file mode 100644 index 00000000000..4a2cd2deee1 --- /dev/null +++ b/bin/k8s/templates/base/observability/otel-collector-configmap.yaml @@ -0,0 +1,109 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +# OpenTelemetry Collector config, k8s counterpart of +# bin/observability/otel-collector/config.yaml. Same pipeline topology; +# exporters point at the in-cluster backend Services instead of the +# docker-compose service names. Only the pipelines for enabled signals +# are rendered, so disabling a signal drops its exporter cleanly. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-otel-collector-config + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:{{ .Values.observability.collector.grpcPort }} + max_recv_msg_size_mib: 4 + http: + endpoint: 0.0.0.0:{{ .Values.observability.collector.httpPort }} + + processors: + batch: + send_batch_size: 1024 + timeout: 5s + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + + exporters: + {{- if .Values.observability.logs.enabled }} + # Loki accepts OTLP at /otlp (Loki 3.x maps it to /otlp/v1/logs). + otlphttp/loki: + endpoint: http://{{ .Release.Name }}-loki:{{ .Values.observability.logs.port }}/otlp + compression: gzip + timeout: 10s + tls: + insecure: true + {{- end }} + {{- if .Values.observability.metrics.enabled }} + prometheusremotewrite: + endpoint: http://{{ .Release.Name }}-victoriametrics:{{ .Values.observability.metrics.port }}/api/v1/write + tls: + insecure: true + remote_write_queue: + queue_size: 1000 + num_consumers: 4 + {{- end }} + {{- if .Values.observability.traces.enabled }} + otlp/jaeger: + endpoint: {{ .Release.Name }}-jaeger:{{ .Values.observability.traces.otlpPort }} + tls: + insecure: true + {{- end }} + + service: + pipelines: + {{- if .Values.observability.logs.enabled }} + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + {{- end }} + {{- if .Values.observability.metrics.enabled }} + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + {{- end }} + {{- if .Values.observability.traces.enabled }} + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/jaeger] + {{- end }} + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + logs: + level: info +{{- end }} diff --git a/bin/k8s/templates/base/observability/otel-collector.yaml b/bin/k8s/templates/base/observability/otel-collector.yaml new file mode 100644 index 00000000000..89c537317d8 --- /dev/null +++ b/bin/k8s/templates/base/observability/otel-collector.yaml @@ -0,0 +1,95 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +# OpenTelemetry Collector: the single OTLP ingress for all signals, +# k8s counterpart of the docker-compose otel-collector service. The +# receiver Ports are exposed only on the ClusterIP Service (no Ingress), +# mirroring the loopback-only posture of the compose deployment. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-otel-collector + template: + metadata: + labels: + app: {{ .Release.Name }}-otel-collector + annotations: + # Roll the pod when the rendered config changes. + checksum/config: {{ include (print $.Template.BasePath "/base/observability/otel-collector-configmap.yaml") . | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + containers: + - name: otel-collector + image: {{ .Values.observability.collector.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: ["--config=/etc/otelcol-contrib/config.yaml"] + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: otlp-grpc + containerPort: {{ .Values.observability.collector.grpcPort }} + - name: otlp-http + containerPort: {{ .Values.observability.collector.httpPort }} + volumeMounts: + - name: config + mountPath: /etc/otelcol-contrib + readOnly: true + resources: + {{- toYaml .Values.observability.collector.resources | nindent 12 }} + volumes: + - name: config + configMap: + name: {{ .Release.Name }}-otel-collector-config +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-otel-collector + ports: + - name: otlp-grpc + protocol: TCP + port: {{ .Values.observability.collector.grpcPort }} + targetPort: {{ .Values.observability.collector.grpcPort }} + - name: otlp-http + protocol: TCP + port: {{ .Values.observability.collector.httpPort }} + targetPort: {{ .Values.observability.collector.httpPort }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/victoriametrics.yaml b/bin/k8s/templates/base/observability/victoriametrics.yaml new file mode 100644 index 00000000000..5a6a1ea97b8 --- /dev/null +++ b/bin/k8s/templates/base/observability/victoriametrics.yaml @@ -0,0 +1,112 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.metrics.enabled }} +# VictoriaMetrics: metrics store, k8s counterpart of the docker-compose +# victoriametrics service. Accepts Prometheus remote-write from the +# collector; MetricsQL query API on the same port. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-victoriametrics-data + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + accessModes: ["ReadWriteOnce"] + {{- if .Values.observability.metrics.storageClass }} + storageClassName: {{ .Values.observability.metrics.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.observability.metrics.storage }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-victoriametrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + replicas: 1 + # Single writer bound to one RWO volume; never run two at once. + strategy: + type: Recreate + selector: + matchLabels: + app: {{ .Release.Name }}-victoriametrics + template: + metadata: + labels: + app: {{ .Release.Name }}-victoriametrics + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10003 + fsGroup: 10003 + containers: + - name: victoriametrics + image: {{ .Values.observability.metrics.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: + - "-storageDataPath=/data" + - "-retentionPeriod={{ .Values.observability.metrics.retentionPeriod }}" + - "-httpListenAddr=:{{ .Values.observability.metrics.port }}" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: http + containerPort: {{ .Values.observability.metrics.port }} + volumeMounts: + - name: data + mountPath: /data + livenessProbe: + httpGet: + path: /health + port: {{ .Values.observability.metrics.port }} + initialDelaySeconds: 10 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.metrics.resources | nindent 12 }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ .Release.Name }}-victoriametrics-data +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-victoriametrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-victoriametrics + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.metrics.port }} + targetPort: {{ .Values.observability.metrics.port }} +{{- end }} diff --git a/bin/k8s/templates/base/webserver/webserver-deployment.yaml b/bin/k8s/templates/base/webserver/webserver-deployment.yaml index 99a58870957..f71c2211903 100644 --- a/bin/k8s/templates/base/webserver/webserver-deployment.yaml +++ b/bin/k8s/templates/base/webserver/webserver-deployment.yaml @@ -77,4 +77,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml index a538ce4a86f..0bc09731390 100644 --- a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml +++ b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml @@ -69,4 +69,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml index 1f6ef3f99b3..163a0e8e840 100644 --- a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml +++ b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml @@ -151,4 +151,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index 4be79adf73a..4c6f43a9755 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -634,3 +634,84 @@ envoy-gateway: extensionApis: enableBackend: true enableEnvoyPatchPolicy: true + +# Observability stack (hand-rolled, mirrors bin/observability + the +# single-node docker-compose profiles). Everything is gated behind +# `enabled` (default off) so the standard install is unchanged. When +# enabled, the Scala services emit OTLP to the in-cluster collector, +# which fans logs/metrics/traces to the backends below. All Services are +# ClusterIP only (no Ingress), matching the loopback-only posture of the +# compose deployment. Profiling (Parca) is intentionally not included: it +# needs a privileged eBPF DaemonSet and is a separate change. +observability: + # Master switch for the whole stack. + enabled: false + + # OpenTelemetry Collector: the single OTLP ingress. Required whenever + # any signal below is enabled (it is what the services emit to). + collector: + enabled: true + image: otel/opentelemetry-collector-contrib:0.153.0 + grpcPort: 4317 + httpPort: 4318 + resources: + limits: + memory: 768Mi + cpu: "1" + + # Loki: log store (OTLP ingest at /otlp + LogQL). AGPLv3, so it is an + # optional operator-run component, not part of any Texera release + # artifact (see bin/observability/README.md). + logs: + enabled: true + image: grafana/loki:3.2.0 + port: 3100 + retentionPeriod: 30d + storage: 10Gi + storageClass: local-path + resources: + limits: + memory: 1Gi + cpu: "1" + + # VictoriaMetrics: metrics store (Prometheus remote-write + MetricsQL). + metrics: + enabled: true + image: victoriametrics/victoria-metrics:v1.144.0 + port: 8428 + retentionPeriod: 90d + storage: 10Gi + storageClass: local-path + resources: + limits: + memory: 1Gi + cpu: "1" + + # Jaeger v2: traces backend + UI. In-memory storage (restart wipes + # traces); swap in persistent storage for production. + traces: + enabled: true + image: jaegertracing/jaeger:2.18.0 + queryPort: 16686 + otlpPort: 4317 + resources: + limits: + memory: 1Gi + cpu: "1" + + # Grafana: dashboard/UI reading Loki + VictoriaMetrics + Jaeger as + # pre-provisioned datasources. AGPLv3, so it is an optional operator-run + # component, not part of any Texera release artifact (see + # bin/observability/README.md). ClusterIP only; expose via your own + # Ingress or `kubectl port-forward`. Override the admin credentials for + # any environment reachable beyond an internal port-forward. + grafana: + enabled: true + image: grafana/grafana:11.2.0 + port: 3000 + adminUser: admin + adminPassword: texera + resources: + limits: + memory: 512Mi + cpu: "0.5" diff --git a/bin/local-dev/docker-compose.override.yml b/bin/local-dev/docker-compose.override.yml index 3b053a85700..bde7cbd37e5 100644 --- a/bin/local-dev/docker-compose.override.yml +++ b/bin/local-dev/docker-compose.override.yml @@ -22,6 +22,14 @@ # Node services *natively* on the host, so we expose each backend port here. services: + # A host-native backend (sbt / IntelliJ) emits OTLP to the collector that + # runs inside compose, so publish the collector's receiver ports on the host + # loopback. Only takes effect when the observability-collector profile is on. + otel-collector: + ports: + - "127.0.0.1:4317:4317" + - "127.0.0.1:4318:4318" + postgres: ports: - "5432:5432" diff --git a/bin/observability/README.md b/bin/observability/README.md new file mode 100644 index 00000000000..04ff0a832ea --- /dev/null +++ b/bin/observability/README.md @@ -0,0 +1,131 @@ + + +# Texera Observability Stack + +Texera's backend services are instrumented with [OpenTelemetry](https://opentelemetry.io/) +and export OTLP (logs, metrics, and traces). This directory is an **optional, +bundled reference stack** that an operator can start alongside a single-node +deployment to collect and visualize that telemetry. It is opt-in: nothing here +runs unless you enable the observability compose profiles (see below), and +Texera emits no telemetry unless `OTEL_SDK_DISABLED=false` is set. + +## Topology + +A single OpenTelemetry Collector is the only OTLP ingress. It fans each signal +out to a dedicated backend, and Grafana sits on top as the query UI: + +``` + Texera services + │ OTLP/gRPC (4317) or OTLP/HTTP (4318) + ▼ + otel-collector + ┌──────────────┬──────┴────────┬───────────────┐ + OTLP │ remote-write OTLP │ │ (eBPF agent push) + ▼ ▼ ▼ ▼ + Loki VictoriaMetrics Jaeger Parca + (logs) (metrics) (traces) (profiles) + │ │ │ + └──────────────┼────────────────┘ + ▼ + Grafana + (reads Loki + VictoriaMetrics + Jaeger) +``` + +- **Loki** stores logs; the collector exports OTLP to `http://loki:3100/otlp`. +- **VictoriaMetrics** stores metrics; the collector exports Prometheus + remote-write to `http://victoriametrics:8428/api/v1/write`. +- **Jaeger** stores traces; the collector exports OTLP to `jaeger:4317`. +- **Parca** stores continuous profiles, pushed by its eBPF agent (not read by + Grafana; use the Parca UI on `:7070`). +- **Grafana** reads Loki, VictoriaMetrics (via the Prometheus query API), and + Jaeger as pre-provisioned datasources. + +## Starting it + +The stack is driven by docker-compose profiles. Enable the signals you want in +`bin/single-node/.env` via `COMPOSE_PROFILES`, then bring the deployment up: + +``` +# bin/single-node/.env +COMPOSE_PROFILES=observability-collector,observability-logs,observability-metrics,observability-traces,observability-grafana +``` + +| Profile | Service(s) | +|---------------------------|-----------------------------------------| +| `observability-collector` | otel-collector (OTLP ingress; required) | +| `observability-logs` | loki | +| `observability-metrics` | victoriametrics | +| `observability-traces` | jaeger | +| `observability-profiles` | parca + privileged eBPF parca-agent | +| `observability-grafana` | grafana | + +Also set `OTEL_SDK_DISABLED=false` so the services actually emit telemetry, then: + +``` +cd bin/single-node && docker compose up -d +# or the dedicated entry point: +cd bin/observability && docker compose up -d +``` + +Reach **Grafana at http://localhost:3000**. The default login is +`admin` / `texera` (set via `GRAFANA_ADMIN_USER` / `GRAFANA_ADMIN_PASSWORD` in +`.env`); change it for anything reachable beyond localhost. + +### Pre-provisioned datasources + +Grafana comes with three datasources already wired up +(`grafana/provisioning/datasources/datasources.yaml`), so you can start querying +immediately: + +- **Metrics** (default) - VictoriaMetrics at `http://victoriametrics:8428`, + queried through the Prometheus API. +- **Loki** - logs at `http://loki:3100`. +- **Jaeger** - traces at `http://jaeger:16686`. + +The backend query ports are also published on `127.0.0.1` for ad-hoc `curl` +debugging (Loki `3100`, VictoriaMetrics `8428`, Jaeger `16686`, Parca `7070`). + +## Using your own stack + +Nothing here is mandatory. The Collector speaks standard OTLP and +Prometheus remote-write, so an operator can point their own tooling at the same +endpoints instead of running the bundled backends: + +- Point an **external Grafana** at the bundled Loki / VictoriaMetrics / Jaeger, + or swap the backends entirely. +- Send logs to **OpenSearch** (or any OTLP-capable log store) by changing the + collector's logs exporter in `otel-collector/config.yaml`. +- Replace the metrics or traces exporters with any Prometheus-remote-write or + OTLP-compatible destination. +- Or drop the collector and have an external collector scrape/receive directly; + the services only need an OTLP endpoint via `OTEL_EXPORTER_OTLP_ENDPOINT`. + +## Licensing + +> **NOTE:** **Grafana and Loki are licensed under the AGPLv3.** They are +> included here only as an **optional, operator-run** convenience and are **NOT +> bundled into, or distributed as part of, any Apache Texera release artifact**. +> An operator who runs them does so on their own; their images are pulled at +> deploy time, not shipped by the project. +> +> The rest of the stack - the **OpenTelemetry Collector, VictoriaMetrics, +> Jaeger, and Parca** - is **Apache-2.0** licensed, consistent with Texera +> itself. Only Apache-2.0 configuration files (collector, Loki, Grafana +> datasources, Parca) live in this repository. diff --git a/bin/observability/docker-compose.yml b/bin/observability/docker-compose.yml new file mode 100644 index 00000000000..adce75a4a40 --- /dev/null +++ b/bin/observability/docker-compose.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Dedicated entry point for the observability stack, so it can be started like +# the other dev targets (e.g. file-service's own compose) instead of reaching +# into the single-node deployment file. +# +# It does NOT redefine the services: it `include`s the single-node compose, the +# single source of truth for every container definition. The observability +# services there are gated behind compose profiles, so they are selected via +# COMPOSE_PROFILES + the service list (see start_app.sh `up_obs`). The other +# single-node services are not profile-gated, so they are simply not named and +# stay down. +# +# Relative volume paths inside the included file (e.g. the collector/parca +# configs under ../observability/) are resolved relative to the included file's +# own directory, so they keep working unchanged. +# +# The project name matches single-node so this manages the SAME containers +# whether obs is brought up here or as part of the full single-node deployment +# (the containers use fixed container_names, so a different project would clash). +name: texera-single-node + +include: + - ../single-node/docker-compose.yml diff --git a/bin/observability/grafana/provisioning/datasources/datasources.yaml b/bin/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 00000000000..b4cd9c09461 --- /dev/null +++ b/bin/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Grafana datasource provisioning for the Texera bundled observability +# stack. Grafana (AGPLv3) reads the three Apache-2.0 query backends by +# their docker-compose bridge-network service names. This file is only +# ASF-licensed configuration; Grafana itself is an optional operator-run +# component and is NOT shipped in any Texera release artifact (see +# bin/observability/README.md). +apiVersion: 1 + +datasources: + # Metrics: VictoriaMetrics speaks the Prometheus query API, so it is + # added as a prometheus-type datasource. Marked default. + - name: Metrics + type: prometheus + uid: metrics + access: proxy + url: http://victoriametrics:8428 + isDefault: true + editable: false + + # Logs: Loki, LogQL over OTLP-ingested logs. + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + editable: false + + # Traces: Jaeger query API / UI backend. + - name: Jaeger + type: jaeger + uid: jaeger + access: proxy + url: http://jaeger:16686 + editable: false diff --git a/bin/observability/loki/config.yaml b/bin/observability/loki/config.yaml new file mode 100644 index 00000000000..5d695805e77 --- /dev/null +++ b/bin/observability/loki/config.yaml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Loki configuration for the Texera bundled observability stack. +# +# Bundled image: grafana/loki:3.2.0 (AGPLv3). Loki is an optional, +# operator-run component; it is NOT shipped in any Texera release +# artifact (see bin/observability/README.md). Only this ASF-licensed +# config file lives in the tree. +# +# This is a minimal single-binary, filesystem-backed deployment for +# single-node development. Replace the filesystem storage with an +# object-storage backend (S3/GCS/Azure) before any multi-node or +# production use. + +# No multi-tenancy: a single-node dev stack has one implicit tenant, +# so requests need no X-Scope-OrgID header. +auth_enabled: false + +server: + # LogQL query API + OTLP ingest (/otlp/v1/logs) are served here. + http_listen_port: 3100 + # Internal gRPC; not published to the host. + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + # Every component's working/data path defaults under this prefix, + # which is the loki_data volume mounted at /loki in docker-compose. + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + # Single binary, single replica: no distributed ring. + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + # tsdb index + filesystem object store is the current (v13) layout. + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + # Required for OTLP ingest: OTLP log attributes are written as + # structured metadata. Enabled by default in Loki 3.x; kept explicit. + allow_structured_metadata: true + # Drop out-of-order rejections for a low-volume dev stack. + reject_old_samples: false + # Age after which log chunks are eligible for deletion by the compactor. + retention_period: 30d + +compactor: + working_directory: /loki/compactor + # Without this the retention_period above is parsed but never enforced, + # so logs accumulate until the loki_data volume fills. + retention_enabled: true + delete_request_store: filesystem + +# OTLP ingest needs no extra config: Loki 3.x serves it at +# http://loki:3100/otlp out of the box. The collector exports there +# (see bin/observability/otel-collector/config.yaml). diff --git a/bin/observability/otel-collector/config.yaml b/bin/observability/otel-collector/config.yaml new file mode 100644 index 00000000000..8cbee85742d --- /dev/null +++ b/bin/observability/otel-collector/config.yaml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OpenTelemetry Collector configuration for the Texera bundled +# observability stack. +# +# Bundled image: otel/opentelemetry-collector-contrib:0.153.0 +# (Apache-2.0). The contrib distribution is needed for the +# prometheusremotewrite exporter that fans metrics to VictoriaMetrics. +# +# Pipeline topology: +# +# Texera services +# │ OTLP/gRPC (4317) or OTLP/HTTP (4318) +# ▼ +# otel-collector +# ├── logs ── OTLP ──────────────────► loki:3100/otlp +# ├── metrics ── prometheusremotewrite ─► victoriametrics:8428 +# └── traces ── OTLP ──────────────────► jaeger:4317 +# +# Network: every receiver and exporter is reachable only on the +# compose bridge network. No host-port mapping for the receivers in +# docker-compose.yml, that keeps OTLP ingest cluster-local, which +# is the security posture the PR plan calls for. +# +# Hostnames below are docker-compose service names, they resolve via +# docker's embedded DNS inside the network. + +receivers: + otlp: + protocols: + grpc: + # 0.0.0.0 binds inside the container only; the compose service + # does NOT publish this port to the host. Reachable only via + # the texera-single-node bridge network. + endpoint: 0.0.0.0:4317 + # 4 MiB cap on a single OTLP message, anyone trying to spam + # the collector with a huge payload gets a clean reject. + max_recv_msg_size_mib: 4 + http: + endpoint: 0.0.0.0:4318 + +processors: + # batch is the upstream-recommended first processor. Caps memory + # per export by batching by either size or timeout, whichever hits + # first. Defaults are sensible for a single-node deployment. + batch: + send_batch_size: 1024 + timeout: 5s + + # memory_limiter prevents the collector OOMing under a sudden burst + # of telemetry. We set a soft limit, not a hard percentage, so the + # numbers are reviewable. + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + +exporters: + # Loki accepts OTLP for logs natively at /otlp (Loki 3.x maps it to + # /otlp/v1/logs). OTLP resource attributes become stream labels / + # structured metadata. Verify the endpoint on every version bump. + otlphttp/loki: + endpoint: http://loki:3100/otlp + compression: gzip + timeout: 10s + # tls is disabled for the compose bridge network. Any deploy + # that opens these ports beyond the bridge must enable TLS here. + tls: + insecure: true + + # VictoriaMetrics consumes Prometheus remote-write natively. The + # prometheusremotewrite exporter is the upstream-recommended way + # to bridge OTel metrics into Prom-ecosystem stores. + prometheusremotewrite: + endpoint: http://victoriametrics:8428/api/v1/write + tls: + insecure: true + # Keep the remote-write batch bounded so a metric burst can't + # become an oversized HTTP request. + remote_write_queue: + queue_size: 1000 + num_consumers: 4 + + # Jaeger v2 accepts OTLP directly, no jaeger exporter type needed. + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + +service: + pipelines: + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/jaeger] + + # Collector telemetry binds to loopback inside the container only + # so we can scrape /metrics from a sidecar if needed without + # exposing the collector's own metrics to the host. + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + logs: + level: info diff --git a/bin/observability/parca/README.md b/bin/observability/parca/README.md new file mode 100644 index 00000000000..d799ba5e7aa --- /dev/null +++ b/bin/observability/parca/README.md @@ -0,0 +1,72 @@ + + +# Parca profiles + +Configuration for the **profiles** signal in the Texera observability +stack: the Parca server and its eBPF agent. The compose services that +run them are in `bin/single-node/docker-compose.yml`; this directory +holds only their configuration. + +Both components are Apache-2.0 (see +[`docs/observability/LICENSING.md`](../../../docs/observability/LICENSING.md)): + +| File | Component | Image | +|---|---|---| +| `parca.yaml` | Parca server v0.28.0 | `ghcr.io/parca-dev/parca:v0.28.0` | +| `parca-agent.env` | Parca eBPF agent v0.47.1 | `ghcr.io/parca-dev/parca-agent:v0.47.1` | + +## Host requirements + +The agent uses eBPF to sample stack traces, which constrains the host: + +- **Linux only.** eBPF is a Linux kernel feature; the agent cannot run + on macOS or Windows. The rest of the stack (logs, metrics, traces) + is cross-platform. +- **Privileged container.** The agent needs `CAP_SYS_ADMIN`-class + permissions to load eBPF programs and open the perf-event facility, + so the compose service runs `privileged: true` and bind-mounts + `/sys/kernel/debug`, `/proc`, and `/sys` read-only. +- **Outbound only.** The agent writes to the bundled Parca server on + `parca:7070` over plain gRPC on the docker bridge network and opens + no other ports. Any deploy that exposes those ports to a wider + network must add TLS at the compose/k8s layer. + +## Opt-in + +Profiling is off by default. It is enabled per host by selecting the +`profiles` profile before `docker compose up`; see `bin/single-node/.env` +for the flags. Enable it only after reviewing the privilege and scope +notes below. + +## What gets profiled + +The agent profiles **every process on the host**, not only Texera's. +It is not scoped to Texera's containers; doing that would require +configuring the agent against specific cgroups, which is not done +here. The static labels in `parca-agent.env` (`deployment=texera`, +`cluster=local`) only let a query filter which profiles are *shown*, +not what is *collected*. Treat enabling the agent as profiling the +whole machine. + +Profiles are not labeled with `workflow.id` / `execution.id` (unbounded +identifiers that would blow up Parca's cardinality). The agent samples +stack traces and does not read application-level OpenTelemetry trace +ids, so profiles cannot be joined to individual traces by `trace_id`; +they correlate only by the coarse labels above and by time. diff --git a/bin/observability/parca/parca-agent.env b/bin/observability/parca/parca-agent.env new file mode 100644 index 00000000000..bfd61dde79f --- /dev/null +++ b/bin/observability/parca/parca-agent.env @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Parca eBPF agent environment. +# +# Bundled image: ghcr.io/parca-dev/parca-agent:v0.47.1 (Apache-2.0). +# +# Consumed by the docker-compose parca-agent service via env_file:. +# The agent reads its config from CLI args; staging them here keeps +# them version-controlled and reusable across compose and Helm. +# +# Deploy requirements (Linux-only, privileged): +# * The agent uses eBPF and must run on a Linux host kernel. +# * Requires CAP_SYS_ADMIN-class permissions to load eBPF programs. +# The compose service runs the agent container with +# `privileged: true` and mounts /sys/kernel/debug, /proc, /sys +# read-only. macOS / Windows developers must leave the +# observability-profiles profile out of COMPOSE_PROFILES (see README.md). +# +# Labels: +# * node: host identifier — set by the compose layer to the docker +# host's hostname. Overridable. +# * metadata-external-labels: static labels attached to every +# profile (deployment=texera;cluster=local). Coarse fields only; +# no workflow/execution id (cardinality DoS in Parca storage). + +PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070 +PARCA_AGENT_REMOTE_STORE_INSECURE=true + +# Per-host identification. Compose can override at deploy time: +# `--node=${HOSTNAME:-texera-dev}` +PARCA_AGENT_NODE=texera-dev + +# Static labels attached to every profile this agent emits. +# Must NOT include workflow.id / execution.id — cardinality DoS in +# Parca's storage. Keep to coarse fields only. +PARCA_AGENT_METADATA_EXTERNAL_LABELS=deployment=texera;cluster=local + +# CPU sample rate. 19 Hz is upstream default and a sensible balance +# between overhead and resolution; we keep it explicit so an +# accidental upstream change doesn't silently shift it. +PARCA_AGENT_PROFILING_CPU_SAMPLING_FREQUENCY=19 + +# Log level — info by default; switch to warn in production to keep +# the agent quiet. +PARCA_AGENT_LOG_LEVEL=info diff --git a/bin/observability/parca/parca.yaml b/bin/observability/parca/parca.yaml new file mode 100644 index 00000000000..0af9e88a393 --- /dev/null +++ b/bin/observability/parca/parca.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Parca server configuration. +# +# This is the minimal config the Texera bundled observability stack +# uses. We deliberately keep scrape_configs empty — profiles arrive +# from the Parca eBPF agent (push model) rather than pull-mode scraping. +# Bundled image: ghcr.io/parca-dev/parca:v0.28.0 (Apache-2.0). +# +# Storage is intentionally filesystem-backed for the single-node +# docker-compose deployment. Replace with an object-storage backend +# (S3/GCS/Azure) before any multi-node or production use. + +object_storage: + bucket: + # FILESYSTEM is the simplest backend for single-node development. + # Data lives inside the container at this path; mount a volume + # in docker-compose to persist across container restarts. + type: "FILESYSTEM" + config: + directory: "/var/lib/parca" + +# scrape_configs is intentionally empty: the Texera deployment pushes +# profiles from the parca-agent via OTLP/gRPC. Adding scrape targets +# here would generate significantly more data than agent-based +# profiling per the upstream recommendation. +scrape_configs: [] diff --git a/bin/observability/smoke_test.py b/bin/observability/smoke_test.py new file mode 100644 index 00000000000..b109496bd7f --- /dev/null +++ b/bin/observability/smoke_test.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# End-to-end routing check for the bundled observability stack. +# +# Emits one log, one metric, and one trace span (each tagged with a test +# service name and a unique run id) at the OpenTelemetry Collector over +# OTLP/HTTP, exactly the way OtelInit's exporters do, then asserts that: +# +# logs land in Loki (not elsewhere) +# metrics land in VictoriaMetrics (not elsewhere) +# traces land in Jaeger (not elsewhere) +# +# and that each signal is attributed to the emitting service. This proves +# the Collector routes each signal type to the right backend per service. +# +# Run against a live stack (OTLP ports published on loopback by the dev +# override). Exit code 0 = all routes correct, 1 = a routing failure. +# +# Usage: python3 bin/observability/smoke_test.py [--service NAME] + +import argparse +import json +import sys +import time +import urllib.parse +import urllib.request +import uuid + +COLLECTOR = "http://127.0.0.1:4318" # OTLP/HTTP receiver +LOKI = "http://127.0.0.1:3100" # logs backend +VICTORIAMETRICS = "http://127.0.0.1:8428" # metrics backend +JAEGER = "http://127.0.0.1:16686" # traces backend + +TIMEOUT_S = 60 # how long to wait for a signal to become queryable +POLL_S = 3 + + +def _post(url, payload): + req = urllib.request.Request( + url, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=10) as r: + return r.status + + +def _get(url): + with urllib.request.urlopen(url, timeout=10) as r: + return json.load(r) + + +def _attr(key, value): + return {"key": key, "value": {"stringValue": value}} + + +def emit(service, run_id): + now_ns = str(time.time_ns()) + resource = {"attributes": [_attr("service.name", service)]} + + # LOG -> logs pipeline -> Loki + _post(f"{COLLECTOR}/v1/logs", {"resourceLogs": [{ + "resource": resource, + "scopeLogs": [{"logRecords": [{ + "timeUnixNano": now_ns, + "severityText": "INFO", + "severityNumber": 9, + "body": {"stringValue": f"smoke-test log {run_id}"}, + }]}], + }]}) + + # METRIC -> metrics pipeline -> VictoriaMetrics + metric_name = f"obs_smoke_test_{run_id}" + _post(f"{COLLECTOR}/v1/metrics", {"resourceMetrics": [{ + "resource": resource, + "scopeMetrics": [{"metrics": [{ + "name": metric_name, + "gauge": {"dataPoints": [{ + "asDouble": 1, + "timeUnixNano": now_ns, + "attributes": [_attr("run", run_id)], + }]}, + }]}], + }]}) + + # TRACE -> traces pipeline -> Jaeger + trace_id = (uuid.uuid4().hex + uuid.uuid4().hex)[:32] + span_id = uuid.uuid4().hex[:16] + _post(f"{COLLECTOR}/v1/traces", {"resourceSpans": [{ + "resource": resource, + "scopeSpans": [{"spans": [{ + "traceId": trace_id, + "spanId": span_id, + "name": f"smoke.test.{run_id}", + "kind": 1, + "startTimeUnixNano": now_ns, + "endTimeUnixNano": str(int(now_ns) + 1_000_000), + }]}], + }]}) + return metric_name + + +def _poll(check): + deadline = time.time() + TIMEOUT_S + while time.time() < deadline: + try: + if check(): + return True + except Exception: + pass + time.sleep(POLL_S) + return False + + +def loki_has(service, run_id): + q = urllib.parse.quote(f'{{service_name="{service}"}}') + end = time.time_ns() + start = end - 10 * 60 * 1_000_000_000 + url = f"{LOKI}/loki/api/v1/query_range?query={q}&start={start}&end={end}&limit=500" + res = _get(url)["data"]["result"] + return any(run_id in v[1] for s in res for v in s["values"]) + + +def vm_has(metric_name): + url = f"{VICTORIAMETRICS}/api/v1/query?query={urllib.parse.quote(metric_name)}" + return len(_get(url)["data"]["result"]) > 0 + + +def jaeger_has(service, run_id): + svcs = _get(f"{JAEGER}/api/services").get("data") or [] + if service not in svcs: + return False + url = f"{JAEGER}/api/traces?service={urllib.parse.quote(service)}&limit=50&lookback=1h" + data = _get(url).get("data") or [] + return any(run_id in s.get("operationName", "") for t in data for s in t["spans"]) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "--services", + default="obs-smoke-a,obs-smoke-b", + help="comma-separated service names to emit as (default tests two)", + ) + args = ap.parse_args() + services = [s.strip() for s in args.services.split(",") if s.strip()] + + # Emit a log + metric + trace for every service, each with its own run id. + emitted = {} + for svc in services: + run_id = uuid.uuid4().hex[:12] + metric_name = emit(svc, run_id) + emitted[svc] = (run_id, metric_name) + print(f"emitted service={svc} run_id={run_id}") + + results = [] + for svc, (run_id, metric_name) in emitted.items(): + checks = [ + (f"{svc}: logs -> Loki", lambda s=svc, r=run_id: loki_has(s, r)), + (f"{svc}: metrics -> VictoriaMetrics", lambda m=metric_name: vm_has(m)), + (f"{svc}: traces -> Jaeger", lambda s=svc, r=run_id: jaeger_has(s, r)), + ] + for label, check in checks: + ok = _poll(check) + results.append(ok) + print(f" {'PASS' if ok else 'FAIL'} {label}") + + # Isolation: one service's log marker must NOT appear under another service. + if len(services) >= 2: + a, b = services[0], services[1] + a_run = emitted[a][0] + leaked = False + try: + leaked = loki_has(b, a_run) # a's marker should not be under b + except Exception: + leaked = False + ok = not leaked + results.append(ok) + print(f" {'PASS' if ok else 'FAIL'} isolation: {a} logs not attributed to {b}") + + if all(results): + print("RESULT: PASS (each signal routed to the correct backend, per service)") + return 0 + print("RESULT: FAIL (a signal was mis-routed or mis-attributed)") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/single-node/.env b/bin/single-node/.env index 40098c54c01..4fcfe17d32e 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -101,17 +101,60 @@ TEXERA_DASHBOARD_SERVICE_ENDPOINT=http://dashboard-service:8080 WORKFLOW_COMPILING_SERVICE_ENDPOINT=http://workflow-compiling-service:9090 WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://workflow-runtime-coordinator-service:8085 -# OpenTelemetry (observability). Disabled by default; the SDK stays inert until -# OTEL_SDK_DISABLED=false. Point the endpoint at a reachable OTLP collector -# (http/https only) before enabling. -OTEL_SDK_DISABLED=true -OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4317 -# Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service. -OTEL_RESOURCE_ATTRIBUTES= -# Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default). -TEXERA_OTEL_ALLOWED_HOSTS= -# Metric export interval in ms. -OTEL_METRIC_EXPORT_INTERVAL=30000 +# ============================================================================ +# Observability +# ============================================================================ +# Each signal is a separate docker-compose profile and is OFF by default: +# COMPOSE_PROFILES is empty, so `docker compose up` runs only core Texera. +# Observability is opt-in, for operators who want it; enable it by listing +# the profiles you want (docker compose reads COMPOSE_PROFILES natively): +# COMPOSE_PROFILES=observability-collector,observability-logs,observability-metrics,observability-traces,observability-grafana +# +# Profiles: +# observability-collector otel-collector (OTLP ingress; the others need it) +# observability-logs loki (AGPLv3; see bin/observability/README.md) +# observability-metrics victoriametrics +# observability-traces jaeger +# observability-profiles parca + privileged eBPF parca-agent +# observability-grafana grafana UI (AGPLv3; reads loki/victoriametrics/jaeger) +# +# Grafana and Loki are AGPLv3 optional operator-run components; they are not +# bundled into any Texera release artifact. The collector, VictoriaMetrics, +# Jaeger, and Parca are Apache-2.0. +# +# observability-profiles is Linux-only, privileged, and profiles every +# process on the host (see bin/observability/parca/README.md). Add it only +# deliberately, after reading that file's privilege and scope notes. +COMPOSE_PROFILES= + +# Query endpoints the dashboard gateway calls to read each signal. The +# defaults in observability-gateway.conf are host-local (127.0.0.1), which +# is what a natively-run backend (sbt / IntelliJ) needs. Inside this compose +# the gateway runs in a container, so it must reach the backends by their +# bridge-network service names, these overrides do that. +TEXERA_OBS_LOGS_URL=http://loki:3100 +TEXERA_OBS_METRICS_URL=http://victoriametrics:8428 +TEXERA_OBS_TRACES_URL=http://jaeger:16686 +TEXERA_OBS_PROFILES_URL=http://parca:7070 + +# Grafana UI (observability-grafana profile). The port is bound to +# loopback (127.0.0.1) like the backends, so the UI is reached at +# http://localhost:3000. The admin login defaults below are weak dev +# defaults; override them for any deployment reachable beyond localhost. +GRAFANA_PORT=3000 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=texera + +# Emission side: the Scala services (via OtelInit) push OTLP to the bundled +# collector over the bridge network. Emission is controlled explicitly by +# OTEL_SDK_DISABLED, which defaults to disabled, so no service emits unless +# you opt in by setting OTEL_SDK_DISABLED=false (do that together with the +# observability-collector profile). The endpoint and allowed-hosts below only +# take effect once emission is enabled: OtelInit validates the endpoint host +# against an allowlist (localhost only by default), so the collector's compose +# service name must be allowlisted explicitly. +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +TEXERA_OTEL_ALLOWED_HOSTS=otel-collector # Notebook migration tool # Toggles the tool in the GUI; the migration itself needs an LLM key above. diff --git a/bin/single-node/docker-compose.observability.yml b/bin/single-node/docker-compose.observability.yml new file mode 100644 index 00000000000..a3adf913119 --- /dev/null +++ b/bin/single-node/docker-compose.observability.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Observability emission overlay. Loaded only by `bin/single-node.sh up +# --with-observability`, it sets OTEL_SDK_DISABLED=false on the instrumented +# services so they emit to the bundled collector. The base .env keeps emission +# off by default; this overlay flips it on without editing .env. The collector +# and signal backends themselves come up via the observability-* compose +# profiles the same command enables. +name: texera-single-node + +services: + access-control-service: + environment: + OTEL_SDK_DISABLED: "false" + config-service: + environment: + OTEL_SDK_DISABLED: "false" + dashboard-service: + environment: + OTEL_SDK_DISABLED: "false" + file-service: + environment: + OTEL_SDK_DISABLED: "false" + notebook-migration-service: + environment: + OTEL_SDK_DISABLED: "false" + workflow-compiling-service: + environment: + OTEL_SDK_DISABLED: "false" + workflow-computing-unit-managing-service: + environment: + OTEL_SDK_DISABLED: "false" + workflow-runtime-coordinator-service: + environment: + OTEL_SDK_DISABLED: "false" diff --git a/bin/single-node/docker-compose.override.yml b/bin/single-node/docker-compose.override.yml new file mode 100644 index 00000000000..4ed7f5708fc --- /dev/null +++ b/bin/single-node/docker-compose.override.yml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Local-dev override, auto-loaded by `docker compose` when run from this +# directory. It publishes the OTel collector's OTLP receiver ports on the +# host loopback so a backend running outside docker (sbt / IntelliJ) can +# emit telemetry to the collector. The released bundle does not ship this +# file, so the base compose stays network-clean in a deploy. +name: texera-single-node + +services: + otel-collector: + ports: + - "127.0.0.1:4317:4317" + - "127.0.0.1:4318:4318" diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index d09cbc09a3d..d0752c2ca40 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -590,6 +590,247 @@ services: command: > sh -c 'apk add --no-cache curl jq bash > /dev/null 2>&1 && bash /examples/load-examples.sh' + # ======================================================================== + # Observability stack: collector + logs, metrics, traces, profiles. + # + # Each signal is a separate compose profile; enable them via + # COMPOSE_PROFILES in bin/single-node/.env. Every port binds to + # loopback (127.0.0.1) or stays on the bridge network; no 0.0.0.0 + # host bindings. + # ======================================================================== + + # OpenTelemetry Collector, the single OTLP ingress for all three + # signals. Reads bin/observability/otel-collector/config.yaml. + # Apache-2.0; pinned to the contrib distribution. + otel-collector: + image: otel/opentelemetry-collector-contrib:0.153.0 + container_name: texera-otel-collector + profiles: [observability-collector] + restart: always + user: "10001:10001" + read_only: true + security_opt: + - no-new-privileges:true + volumes: + - ../observability/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml:ro,z + command: ["--config=/etc/otelcol-contrib/config.yaml"] + # In-compose services reach the OTLP receivers over the bridge + # network as otel-collector:4317/:4318, so the base file publishes + # no host ports. For a backend running outside docker (sbt / + # IntelliJ), bin/local-dev/docker-compose.override.yml, which the + # local-dev launcher loads, publishes them on the host loopback; the + # released bundle omits that override. + deploy: + resources: + limits: + memory: 768M + cpus: "1.0" + + # Loki (AGPLv3) log store. OTLP ingest at /otlp, LogQL query API on + # 3100. AGPLv3, so it is an optional operator-run component and is NOT + # shipped in any Texera release artifact (see bin/observability/README.md). + # Reads bin/observability/loki/config.yaml. + loki: + image: grafana/loki:3.2.0 + container_name: texera-loki + profiles: [observability-logs] + restart: always + user: "10001:10001" + read_only: true + security_opt: + - no-new-privileges:true + volumes: + - ../observability/loki/config.yaml:/etc/loki/config.yaml:ro,z + - loki_data:/loki + # Read-only root filesystem, so Loki's scratch space is an in-memory + # tmpfs; all durable state lives on the loki_data volume under /loki. + tmpfs: + - /tmp + command: ["-config.file=/etc/loki/config.yaml"] + # Loopback-only host binding so an operator can curl the query + # API from the host for ad-hoc debugging without exposing it to + # the network. + ports: + - "127.0.0.1:3100:3100" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Grafana (AGPLv3) dashboard/UI. Reads the Apache-2.0 backends below + # (Loki, VictoriaMetrics, Jaeger) as pre-provisioned datasources. The UI + # is meant to be opened by a human, but like the backends its port is + # bound to loopback only (http://localhost:3000). AGPLv3, so it is an + # optional operator-run component and is NOT shipped in any Texera + # release artifact (see bin/observability/README.md). + grafana: + image: grafana/grafana:11.2.0 + container_name: texera-grafana + profiles: [observability-grafana] + restart: always + security_opt: + - no-new-privileges:true + depends_on: + - loki + - victoriametrics + - jaeger + environment: + # Admin login. Defaults are weak dev defaults; override in .env for + # any deployment reachable beyond localhost. + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-texera} + # No anonymous access: the UI requires the admin login above. + - GF_AUTH_ANONYMOUS_ENABLED=false + # Users cannot self-register. + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + # Datasource provisioning (Loki + VictoriaMetrics + Jaeger), read-only. + - ../observability/grafana/provisioning:/etc/grafana/provisioning:ro,z + - grafana_data:/var/lib/grafana + # Loopback-only host binding, like the backends: a human opens + # http://localhost:3000, and the UI is never reachable from the + # network by default. + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + + # VictoriaMetrics, Apache-2.0 metrics store. Accepts Prometheus + # remote-write from the collector at /api/v1/write, MetricsQL on + # /api/v1/query. + # Fix /data ownership before VictoriaMetrics starts. VM runs non-root + # (uid 10003) with a read-only rootfs, but a freshly created named volume + # is owned by root, so without this VM cannot create its lock file and + # crash-loops. Idempotent on an existing volume. + victoriametrics-init: + image: alpine:3.19 + container_name: texera-victoriametrics-init + profiles: [observability-metrics] + restart: "no" + user: "0:0" + entrypoint: ["/bin/sh", "-c"] + command: + - chown -R 10003:10003 /data + volumes: + - victoriametrics_data:/data + + victoriametrics: + image: victoriametrics/victoria-metrics:v1.144.0 + container_name: texera-victoriametrics + profiles: [observability-metrics] + restart: always + user: "10003:10003" + depends_on: + victoriametrics-init: + condition: service_completed_successfully + read_only: true + security_opt: + - no-new-privileges:true + command: + - "-storageDataPath=/data" + - "-retentionPeriod=90d" + - "-httpListenAddr=:8428" + volumes: + - victoriametrics_data:/data + ports: + - "127.0.0.1:8428:8428" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Jaeger v2, Apache-2.0 traces backend + UI. v2 accepts OTLP + # natively. The query API lives at :16686, OTLP ingest at :4317 + # (kept inside the bridge network only, only the collector talks + # to it). + jaeger: + image: jaegertracing/jaeger:2.18.0 + container_name: texera-jaeger + profiles: [observability-traces] + restart: always + security_opt: + - no-new-privileges:true + # In-memory storage for the single-node deployment, restarts + # wipe traces. Any deploy needing trace persistence must swap + # in Cassandra or OpenSearch-backed storage here. + ports: + - "127.0.0.1:16686:16686" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Parca server (Apache-2.0) profiles backend. Reads + # bin/observability/parca/parca.yaml. FILESYSTEM storage. + parca: + image: ghcr.io/parca-dev/parca:v0.28.0 + container_name: texera-parca + profiles: [observability-profiles] + restart: always + security_opt: + - no-new-privileges:true + volumes: + - ../observability/parca/parca.yaml:/parca.yaml:ro,z + - parca_data:/var/lib/parca + command: + - "/parca" + - "--config-path=/parca.yaml" + ports: + - "127.0.0.1:7070:7070" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Parca eBPF agent (Apache-2.0). Linux-only, privileged: it is the + # only observability service needing elevated permissions, and it + # profiles every process on the host, not just Texera's. Keep it + # opt-in and read bin/observability/parca/README.md for the privilege + # and scope implications before enabling it. + parca-agent: + image: ghcr.io/parca-dev/parca-agent:v0.47.1 + container_name: texera-parca-agent + profiles: [observability-profiles] + restart: always + depends_on: + parca: + condition: service_started + env_file: + - ../observability/parca/parca-agent.env + # eBPF requires CAP_SYS_ADMIN-class privileges. macOS / Windows + # developers cannot run this, they should drop the + # observability-profiles profile from COMPOSE_PROFILES. + privileged: true + pid: "host" + # Bind-mount kernel state read-only, the agent reads /proc and + # /sys for stack-trace symbolization but cannot write to either. + volumes: + - /sys/kernel/debug:/sys/kernel/debug:ro + - /proc:/host/proc:ro + - /sys:/host/sys:ro + # parca-agent ships as a distroless image (no /bin/sh), so flags + # are passed directly. env_file values above are kept as the + # source of truth, edit both if you change a value. + command: + - "--remote-store-address=parca:7070" + - "--remote-store-insecure=true" + - "--node=texera-dev" + - "--metadata-external-labels=deployment=texera;cluster=local" + - "--profiling-cpu-sampling-frequency=19" + - "--log-level=info" + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + networks: default: name: texera-single-node @@ -598,4 +839,11 @@ networks: volumes: rustfs_data: postgres_data: - workflow_result_data: \ No newline at end of file + workflow_result_data: + # Observability storage. Each backend gets its own named volume so + # an operator can drop one signal's history (e.g. logs) without + # touching the others. + loki_data: + grafana_data: + victoriametrics_data: + parca_data: \ No newline at end of file diff --git a/bin/single-node/main.sh b/bin/single-node/main.sh index 770ddc11163..e9734cea0f6 100755 --- a/bin/single-node/main.sh +++ b/bin/single-node/main.sh @@ -20,12 +20,16 @@ # # Subcommands: # bin/single-node.sh same as `status` (no-arg default). -# bin/single-node.sh up [--with-examples] pre-flight docker check, then +# bin/single-node.sh up [--with-examples] [--with-observability] +# pre-flight docker check, then # `docker compose up -d`. Detached # so you can keep using the shell. -# Add `--with-examples` to also -# pre-create two demo workflows + -# datasets (the `examples` profile). +# --with-examples pre-creates two +# demo workflows + datasets; and +# --with-observability starts the +# collector + Grafana/Loki/Jaeger/ +# VictoriaMetrics and turns telemetry +# emission on (off by default). # bin/single-node.sh down [--volumes] stop every container. `--volumes` # also drops data volumes # (full reset — destroys workflows, @@ -43,6 +47,10 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" COMPOSE_FILE="$REPO_ROOT/bin/single-node/docker-compose.yml" +OBS_OVERRIDE="$REPO_ROOT/bin/single-node/docker-compose.observability.yml" +# Signal profiles enabled by --with-observability. Parca/eBPF profiling is a +# separate privileged opt-in and is deliberately not included here. +OBS_PROFILES=(--profile observability-collector --profile observability-logs --profile observability-metrics --profile observability-traces --profile observability-grafana) # --------- output helpers --------- if [[ -t 1 ]]; then @@ -93,29 +101,44 @@ print_url_tip() { printf " ${DIM}Compose:${RESET} docker compose -f bin/single-node/docker-compose.yml ps\n\n" } +print_observability_urls() { + printf " ${BOLD}Observability${RESET} (enabled, default-off otherwise):\n" + printf " ${DIM}Grafana:${RESET} ${BOLD}http://localhost:3000${RESET} admin / texera, dashboards over all signals\n" + printf " ${DIM}Jaeger:${RESET} ${BOLD}http://localhost:16686${RESET} traces\n" + printf " ${DIM}Metrics:${RESET} ${BOLD}http://localhost:8428/vmui/${RESET} VictoriaMetrics\n\n" +} + # --------- subcommands --------- cmd_up() { - local with_examples=false - case "${1:-}" in - "") ;; - --with-examples) with_examples=true ;; - *) - tui_err "unknown flag for up: $1 (only --with-examples is accepted)" - exit 1 - ;; - esac + local with_examples=false with_observability=false + while [[ $# -gt 0 ]]; do + case "$1" in + --with-examples) with_examples=true ;; + --with-observability) with_observability=true ;; + *) + tui_err "unknown flag for up: $1 (accepted: --with-examples, --with-observability)" + exit 1 + ;; + esac + shift + done need_docker tui_header "Texera single-node — up" - if $with_examples; then - tui_step "docker compose --profile examples up -d (with demo workflows + datasets)" - compose --profile examples up -d - else - tui_step "docker compose up -d" - compose up -d + local file_args=() profile_args=() + $with_examples && profile_args+=(--profile examples) + if $with_observability; then + profile_args+=("${OBS_PROFILES[@]}") + file_args+=(-f "$OBS_OVERRIDE") fi + local mode="docker compose up -d" + $with_observability && mode="$mode (observability on)" + $with_examples && mode="$mode (with examples)" + tui_step "$mode" + compose "${file_args[@]}" "${profile_args[@]}" up -d tui_ok "stack started (first boot pulls ~5 min of images)" printf "\n" print_url_tip + $with_observability && print_observability_urls } cmd_down() { @@ -130,17 +153,16 @@ cmd_down() { esac need_docker tui_header "Texera single-node — down" - # Always include `--profile examples` so we also stop the demo - # containers if the user started with --with-examples. compose - # silently ignores profile services that aren't running, so this is - # safe in both modes. + # Include every optional profile (examples + observability) so down stops + # them too, however the stack was started. compose silently ignores profile + # services that aren't running, so this is safe in all modes. if $drop_volumes; then - tui_step "docker compose --profile examples down --volumes" - compose --profile examples down --volumes + tui_step "docker compose down --volumes" + compose --profile examples "${OBS_PROFILES[@]}" down --volumes tui_ok "stack stopped & data volumes removed" else - tui_step "docker compose --profile examples down" - compose --profile examples down + tui_step "docker compose down" + compose --profile examples "${OBS_PROFILES[@]}" down tui_ok "stack stopped (data volumes preserved)" fi } diff --git a/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala new file mode 100644 index 00000000000..4f263eb08d5 --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} + +/** + * Smoke tests for the single-node docker-compose + collector config. + * Same design as [[ParcaConfigSpec]]: string-level assertions, no YAML + * parser, because the goal is to catch typos and licence-pin drift, + * not to validate the upstream schemas. + */ +class ObservabilityComposeSpec extends AnyFlatSpec with Matchers { + + private def resolveBundled(relative: String): Path = { + var dir = Paths.get("").toAbsolutePath + var hops = 0 + while (hops < 10) { + val candidate = dir.resolve(relative) + if (Files.exists(candidate)) return candidate + val parent = dir.getParent + if (parent == null) return candidate + dir = parent + hops += 1 + } + Paths.get(relative) + } + + private def read(p: Path): String = + new String(Files.readAllBytes(p), StandardCharsets.UTF_8) + + // ----- bundled paths -------------------------------------------------- + + private val compose = resolveBundled("bin/single-node/docker-compose.yml") + private val envFile = resolveBundled("bin/single-node/.env") + private val overrideFile = resolveBundled("bin/single-node/docker-compose.override.yml") + private val collector = resolveBundled("bin/observability/otel-collector/config.yaml") + + // ----- docker-compose: pinned images --------------------------------- + + "docker-compose.yml" should "exist" in { + Files.exists(compose) shouldBe true + } + + it should "pin the OSS observability image tags from LICENSING.md" in { + val text = read(compose) + text should include("otel/opentelemetry-collector-contrib:0.153.0") + text should include("grafana/loki:3.2.0") + text should include("grafana/grafana:11.2.0") + text should include("victoriametrics/victoria-metrics:v1.144.0") + text should include("jaegertracing/jaeger:2.18.0") + text should include("ghcr.io/parca-dev/parca:v0.28.0") + text should include("ghcr.io/parca-dev/parca-agent:v0.47.1") + } + + it should "NOT reference the VictoriaMetrics enterprise images" in { + // Tripwire: '-enterprise' images are not Apache-2.0. Texera ships + // OSS only, so anyone editing the tags must be reminded of that. + val text = read(compose) + text should not include "-enterprise" + } + + it should "bind every observability host port to loopback (127.0.0.1)" in { + val text = read(compose) + // The backend query/UI endpoints we expose to the host MUST use + // the 127.0.0.1: prefix. Any "0.0.0.0:3100" or naked "3100:3100" + // would publish to all interfaces, a misconfig that would expose + // the data store to the network. (Grafana's own loopback binding is + // checked separately below, since its port is a variable.) + Seq("3100", "8428", "16686", "7070").foreach { port => + val pattern = s""""127.0.0.1:$port:$port"""" + withClue(s"port $port should be loopback-only: ") { + text should include(pattern) + } + } + } + + it should "bind the Grafana UI to loopback too (not all interfaces)" in { + val text = read(compose) + // Grafana publishes a configurable port but still only on loopback, + // so the UI is reachable at http://localhost:3000 and never from the + // network by default. + text should include("127.0.0.1:${GRAFANA_PORT:-3000}:3000") + text should not include "0.0.0.0:3000" + } + + it should "publish the OTel collector OTLP receivers only in the dev override, on loopback" in { + // The base compose publishes no OTLP host ports (deploy-clean); the + // dev override exposes 127.0.0.1:4317-4318 so a backend running + // outside docker (sbt / IntelliJ) can emit telemetry. The hard + // requirement is that the receiver is NEVER bound on a non-loopback + // address, otherwise OTLP would be reachable from the LAN. + val base = read(compose) + base should not include "4317:4317" + base should not include "4318:4318" + val dev = read(overrideFile) + dev should include("\"127.0.0.1:4317:4317\"") + dev should include("\"127.0.0.1:4318:4318\"") + dev should not include "0.0.0.0:4317" + dev should not include "0.0.0.0:4318" + dev should not include "\"4317:4317\"" + dev should not include "\"4318:4318\"" + } + + it should "only mark the parca-agent privileged (eBPF needs CAP_SYS_ADMIN)" in { + // privileged: true is dangerous; we want it on exactly one + // service. If a future contributor copies the agent block as a + // template for another service, this test trips. + val text = read(compose) + val priv = "privileged: true".r.findAllIn(text).length + priv shouldBe 1 + } + + it should "give every observability backend a profile so it can be disabled" in { + val text = read(compose) + Seq( + "observability-collector", + "observability-logs", + "observability-grafana", + "observability-metrics", + "observability-traces", + "observability-profiles" + ).foreach { profile => + text should include(profile) + } + } + + it should "set a memory limit on every observability service" in { + // The new section lives below the "Observability stack" banner. + val text = read(compose).split("Observability stack").last + // Seven services, each gets a `deploy.resources.limits.memory:` line. + val memoryLimits = "memory:".r.findAllIn(text).length + memoryLimits should be >= 7 + } + + // ----- .env defaults ------------------------------------------------- + + ".env" should "default COMPOSE_PROFILES to empty so observability is opt-in" in { + val text = read(envFile) + // The active (non-comment) COMPOSE_PROFILES line must be empty: no + // observability profile runs unless the operator opts in. + val activeLine = text + .split('\n') + .map(_.trim) + .find(l => !l.startsWith("#") && l.startsWith("COMPOSE_PROFILES=")) + withClue("an active COMPOSE_PROFILES= line should exist: ")(activeLine shouldBe defined) + activeLine.get shouldBe "COMPOSE_PROFILES=" + } + + // ----- otel-collector config ----------------------------------------- + + "otel-collector/config.yaml" should "exist and declare all three signal pipelines" in { + Files.exists(collector) shouldBe true + val text = read(collector) + // The Service block defines exactly three pipelines. + text should include("logs:") + text should include("metrics:") + text should include("traces:") + } + + it should "route metrics via prometheusremotewrite to VictoriaMetrics" in { + val text = read(collector) + text should include("prometheusremotewrite") + text should include("victoriametrics:8428") + } + + it should "route logs to Loki over OTLP HTTP" in { + val text = read(collector) + text should include("otlphttp/loki") + text should include("loki:3100") + } + + it should "route traces to Jaeger over OTLP gRPC" in { + val text = read(collector) + text should include("otlp/jaeger") + text should include("jaeger:4317") + } + + it should "cap incoming OTLP message size (DoS guard)" in { + val text = read(collector) + text should include("max_recv_msg_size_mib") + } + + it should "configure memory_limiter to bound collector memory" in { + val text = read(collector) + text should include("memory_limiter") + text should include("limit_mib") + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala new file mode 100644 index 00000000000..42e29c6d75d --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} + +/** + * Smoke test for the bundled Parca configuration files. + * + * Intentionally lightweight: we are guarding against accidental + * deletion / emptying / tag drift, not validating Parca's schema. + * The real config validation happens when the agent starts up + * inside its container — but a unit-level smoke test catches typos + * before a developer pushes them. + */ +class ParcaConfigSpec extends AnyFlatSpec with Matchers { + + // sbt runs tests with the module dir as CWD, but a developer who + // runs `sbt test` from the project root has a different CWD. Walk + // upwards until we find the file, bounded so a missing file fails + // loudly rather than infinite-looping. + private def resolveBundled(relative: String): Path = { + var dir = Paths.get("").toAbsolutePath + var hops = 0 + while (hops < 10) { + val candidate = dir.resolve(relative) + if (Files.exists(candidate)) return candidate + val parent = dir.getParent + if (parent == null) return candidate + dir = parent + hops += 1 + } + Paths.get(relative) // will fail the existence assert below + } + + private val parcaYaml = resolveBundled("bin/observability/parca/parca.yaml") + private val agentEnv = resolveBundled("bin/observability/parca/parca-agent.env") + private val readme = resolveBundled("bin/observability/parca/README.md") + + // ----- parca.yaml ----------------------------------------------------- + + "parca.yaml" should "exist in bin/observability/parca/" in { + Files.exists(parcaYaml) shouldBe true + } + + it should "declare object_storage with the FILESYSTEM bucket type" in { + val text = new String(Files.readAllBytes(parcaYaml), StandardCharsets.UTF_8) + text should include("object_storage") + text should include("FILESYSTEM") + text should include("/var/lib/parca") + } + + it should "include the ASF license header" in { + val text = new String(Files.readAllBytes(parcaYaml), StandardCharsets.UTF_8) + text should include("Apache License, Version 2.0") + } + + // ----- parca-agent.env ------------------------------------------------ + + "parca-agent.env" should "exist and pin the bundled image version reference" in { + Files.exists(agentEnv) shouldBe true + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("v0.47.1") + } + + it should "point the agent at the bundled Parca server hostname" in { + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070") + } + + it should "carry the deployment label so the gateway can filter Texera processes" in { + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("deployment=texera") + } + + it should "NOT include high-cardinality labels (workflow.id / execution.id)" in { + // Tripwire: a future contributor might be tempted to add + // workflow.id as a static label. That blows up Parca storage. + // This assertion makes the design intent enforceable. Comments + // are skipped — they're allowed (and required) to explain the + // rule. + val configLines = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8).linesIterator + .map(_.trim) + .filter(line => line.nonEmpty && !line.startsWith("#")) + .toSeq + configLines.foreach { line => + line should not include "workflow.id" + line should not include "workflow_id" + line should not include "execution.id" + line should not include "execution_id" + } + } + + // ----- README --------------------------------------------------------- + + "README.md" should "exist and document that profiling is opt-in / off by default" in { + Files.exists(readme) shouldBe true + val text = new String(Files.readAllBytes(readme), StandardCharsets.UTF_8).toLowerCase + text should include("opt-in") + text should include("off by default") + } + + it should "document the Linux-only / privileged-container requirement" in { + val text = new String(Files.readAllBytes(readme), StandardCharsets.UTF_8) + text.toLowerCase should include("linux") + text.toLowerCase should include("privileged") + } +} diff --git a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala index 1d39f1d9993..afd05e1aeee 100644 --- a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala +++ b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala @@ -387,6 +387,46 @@ object ComputingUnitManagingResource { lookup(name).map(_.trim).filter(_.nonEmpty).map(name -> _) }.toMap + // Forwarded to a created unit so it emits to the same collector as the rest of + // the deployment. service.instance.id is NOT here: it is built per unit below. + private val observabilityComputingUnitEnvNames: Seq[String] = Seq( + EnvironmentalVariable.ENV_OTEL_SDK_DISABLED, + EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT, + EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS, + EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL + ) + + /** + * Observability environment for a created unit. Without this a unit runs with + * the SDK-disabled default even when the manager has telemetry enabled, so a + * worker failure leaves no unit logs, metrics, or traces to investigate. The + * unit inherits the manager's collector endpoint and related settings, but is + * given its own service.instance.id (service.name stays shared) so units with + * the same name do not collapse into one metric series; an instance id the + * manager itself carries is dropped rather than copied. Returns nothing when no + * collector endpoint is configured, leaving the unit at its default-off state. + */ + private[resource] def observabilityComputingUnitEnv( + cuid: Int, + lookup: String => Option[String] + ): Map[String, String] = { + val get = (name: String) => lookup(name).map(_.trim).filter(_.nonEmpty) + if (get(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT).isEmpty) { + Map.empty + } else { + val forwarded = + observabilityComputingUnitEnvNames.flatMap(name => get(name).map(name -> _)).toMap + val kept = get(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES) + .getOrElse("") + .split(",") + .map(_.trim) + .filter(_.nonEmpty) + .filterNot(_.toLowerCase.startsWith("service.instance.id=")) + val resourceAttrs = (kept :+ s"service.instance.id=computing-unit-$cuid").mkString(",") + forwarded + (EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES -> resourceAttrs) + } + } + // Environment variables passed to the created computing unit(pod) private lazy val computingUnitEnvironmentVariables: Map[String, Any] = icebergEnvironmentVariables ++ Map( @@ -738,7 +778,7 @@ class ComputingUnitManagingResource( computingUnitEnvironmentVariables ++ Map( EnvironmentalVariable.ENV_USER_JWT_TOKEN -> userToken, EnvironmentalVariable.ENV_JAVA_OPTS -> s"-Xmx${param.jvmMemorySize}" - ), + ) ++ observabilityComputingUnitEnv(cuid, EnvironmentalVariable.get), Some(param.shmSize), curatedImage ) diff --git a/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala b/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala index 4dde85509c3..8e65dc6c08d 100644 --- a/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala +++ b/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala @@ -651,4 +651,41 @@ class ComputingUnitManagingResourceSpec ComputingUnitManagingResource.optionalComputingUnitEnv(_ => Some(" ")) shouldBe empty } + "observabilityComputingUnitEnv" should "forward collector settings and give the unit its own instance id when enabled" in { + val enabled = Map( + EnvironmentalVariable.ENV_OTEL_SDK_DISABLED -> "false", + EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT -> "http://texera-otel-collector:4317", + EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS -> "texera-otel-collector", + EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL -> "30000" + ) + val env = ComputingUnitManagingResource.observabilityComputingUnitEnv(7, enabled.get) + env(EnvironmentalVariable.ENV_OTEL_SDK_DISABLED) shouldBe "false" + env(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT) shouldBe + "http://texera-otel-collector:4317" + env(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES) shouldBe + "service.instance.id=computing-unit-7" + } + + // Each unit must be distinguishable, so a unit never inherits the manager's instance id. + it should "replace an inherited service.instance.id with the unit's own, keeping other attributes" in { + val lookup: String => Option[String] = Map( + EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT -> "http://texera-otel-collector:4317", + EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES -> + "deployment.environment=staging,service.instance.id=manager,service.namespace=texera" + ).get + val attrs = ComputingUnitManagingResource + .observabilityComputingUnitEnv(8, lookup)(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES) + attrs should include("deployment.environment=staging") + attrs should include("service.namespace=texera") + attrs should include("service.instance.id=computing-unit-8") + attrs should not include "service.instance.id=manager" + } + + // Telemetry stays opt-in: with no collector endpoint the unit is left default-off. + it should "forward nothing when no collector endpoint is configured" in { + ComputingUnitManagingResource.observabilityComputingUnitEnv(1, _ => None) shouldBe empty + val blankEndpoint = Map(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT -> " ") + ComputingUnitManagingResource.observabilityComputingUnitEnv(1, blankEndpoint.get) shouldBe empty + } + }