diff --git a/.dockerignore b/.dockerignore index 0a608d39bbd..0455ee63f4a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -48,13 +48,17 @@ query-results/ # Ignoring scala related files hs_err_pid* -# Ignoring Python related files -venv/ -__pycache__/ -*.py[cod] -*$py.class -.ipynb_checkpoints -.pytype/ +# Ignoring Python related files. computing-unit-master copies +# amber/src/main/python into its image, so these must match at any depth. +**/venv/ +**/__pycache__/ +**/*.py[cod] +**/*$py.class +**/.ipynb_checkpoints +**/.pytype/ +**/.pytest_cache/ +# Regenerated by bin/python-proto-gen.sh during the image build. +amber/src/main/python/proto/ # Ignoring Python-generated files *.model diff --git a/.github/release/vote-email-template.md b/.github/release/vote-email-template.md index 5b8c9082f91..c7d0f940a4e 100644 --- a/.github/release/vote-email-template.md +++ b/.github/release/vote-email-template.md @@ -50,6 +50,16 @@ The Dockerfiles are included in the source for audit and verification. https://github.com/apache/texera/releases/tag/${TAG_NAME} Commit: ${COMMIT_HASH} +== Reproducing the Artifacts == + +Both tarballs are reproducible. To rebuild them from the tag (needs GNU tar) +and compare against the staged checksums: + + git clone https://github.com/apache/texera.git && cd texera + git checkout ${TAG_NAME} + ./.github/scripts/create-release-tarballs.sh ${TAG_NAME} ${VERSION} ${IMAGE_REGISTRY} ${IMAGE_TAG} /tmp/texera-rc + sha512sum /tmp/texera-rc/*.tar.gz + == Keys == The release was signed with GPG key [${GPG_KEY_ID}] (${GPG_EMAIL}) @@ -70,6 +80,7 @@ The vote will be open for at least 72 hours. [ ] All files have ASF license headers where appropriate [ ] No unexpected binary files [ ] Source tarball matches the Git tag +[ ] Rebuilt tarballs match the staged SHA512 checksums [ ] Can compile from source successfully [ ] Docker Compose bundle deploys successfully with the published images diff --git a/.github/renovate.json5 b/.github/renovate.json5 index c6fe7f387a8..d18b4159bfb 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -27,9 +27,11 @@ $schema: 'https://docs.renovatebot.com/renovate-schema.json', extends: ['config:recommended'], - // Only the ecosystems dependabot.yml covered; enable more (docker, - // gradle, etc.) deliberately in follow-ups rather than by default. - enabledManagers: ['npm', 'bun', 'sbt', 'pip_requirements', 'github-actions'], + // The ecosystems dependabot.yml covered, plus the Dockerfiles: their base + // images are pinned by digest for reproducible builds, and a digest nobody + // bumps would freeze OS and JRE security fixes. Enable more (gradle, etc.) + // deliberately in follow-ups rather than by default. + enabledManagers: ['npm', 'bun', 'sbt', 'pip_requirements', 'github-actions', 'dockerfile'], timezone: 'Etc/UTC', schedule: ['before 8am on monday'], @@ -88,6 +90,18 @@ semanticCommitScope: 'deps, amber', }, + { + matchManagers: ['dockerfile'], + semanticCommitScope: 'deps, docker', + }, + // Base-image digest refreshes (a tag moving to a rebuilt image) and + // patch/minor tag bumps land together in one weekly PR. + { + matchManagers: ['dockerfile'], + matchUpdateTypes: ['digest', 'patch', 'minor'], + groupName: 'docker base images', + }, + // Patch bumps grouped into one weekly PR per area. Minor bumps are // also grouped per area — except sbt, where minors ship behavior // changes often enough that per-package PRs keep CI failures diff --git a/.github/scripts/create-release-tarballs.sh b/.github/scripts/create-release-tarballs.sh new file mode 100755 index 00000000000..9d1cfa1ef11 --- /dev/null +++ b/.github/scripts/create-release-tarballs.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds the release source tarball and Docker Compose bundle from a git tag. +# Both archives are reproducible: the same tag and arguments give the same +# bytes on any machine, so a voter can rebuild them and compare the checksums +# against the staged artifacts. Everything the archives would otherwise pick up +# from the build machine (clock, umask, user, file order, locale) is pinned to +# the tagged commit or to a constant. +# +# Usage: create-release-tarballs.sh [output-dir] +# Needs GNU tar (gtar on macOS) and gzip. Writes +# apache-texera--src.tar.gz +# apache-texera--docker-compose.tar.gz +# to output-dir (default: the current directory). + +set -euo pipefail + +if [[ $# -lt 4 || $# -gt 5 ]]; then + echo "Usage: $0 [output-dir]" >&2 + exit 2 +fi + +tag="$1" +version="$2" +image_registry="$3" +image_tag="$4" +out_dir="${5:-.}" + +export LC_ALL=C TZ=UTC + +# These land in file names and in a sed replacement, so anything outside the +# characters a version, image tag or registry path can contain is rejected. +check_arg() { + if [[ ! "$2" =~ $3 ]]; then + echo "Error: invalid $1 '$2'" >&2 + exit 2 + fi +} +check_arg version "$version" '^[A-Za-z0-9][A-Za-z0-9._-]*$' +check_arg image-tag "$image_tag" '^[A-Za-z0-9_][A-Za-z0-9._-]*$' +check_arg image-registry "$image_registry" '^[A-Za-z0-9][A-Za-z0-9._:/-]*$' + +if tar --version 2>/dev/null | grep -q 'GNU tar'; then + gnu_tar=tar +elif command -v gtar >/dev/null && gtar --version | grep -q 'GNU tar'; then + gnu_tar=gtar +else + echo "Error: GNU tar is required (on macOS: brew install gnu-tar)" >&2 + exit 1 +fi + +if ! commit=$(git rev-parse --verify --quiet "${tag}^{commit}"); then + echo "Error: '${tag}' does not name a commit" >&2 + exit 1 +fi +mkdir -p "$out_dir" +out_dir=$(cd "$out_dir" && pwd) +# git archive resolves pathspecs against the working directory. +cd "$(git rev-parse --show-toplevel)" + +# Derived from the commit rather than read from the environment, so a stray +# SOURCE_DATE_EPOCH in a voter's shell cannot change the result. +SOURCE_DATE_EPOCH=$(git log -1 --format=%ct "$commit") +export SOURCE_DATE_EPOCH + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +# Recipe from https://reproducible-builds.org/docs/archives/. --mode strips the +# umask's influence on permission bits; gzip -n keeps the input name and time +# out of the gzip header. +reproducible_tar_gz() { + local parent="$1" dir="$2" output="$3" + "$gnu_tar" --sort=name --format=pax \ + --mtime="@${SOURCE_DATE_EPOCH}" \ + --owner=0 --group=0 --numeric-owner \ + --mode='u+rwX,go+rX,go-w' \ + --pax-option='exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime' \ + -C "$parent" -cf - "$dir" | gzip -9 -n > "$output" +} + +# Replaces KEY's line in an env file, or appends it when absent. Writes through +# a temp file instead of `sed -i`, whose flags differ between GNU and BSD sed. +set_env_var() { + local file="$1" key="$2" value="$3" + if grep -q "^${key}=" "$file"; then + sed "s|^${key}=.*|${key}=${value}|" "$file" > "$file.tmp" + mv "$file.tmp" "$file" + else + echo "${key}=${value}" >> "$file" + fi +} + +src_name="apache-texera-${version}-src" +mkdir -p "$work/src" +git archive --format=tar --prefix="${src_name}/" "$commit" | "$gnu_tar" -x -C "$work/src" +reproducible_tar_gz "$work/src" "$src_name" "$out_dir/${src_name}.tar.gz" + +compose_name="apache-texera-${version}-docker-compose" +raw="$work/raw" +bundle="$work/compose/$compose_name" +mkdir -p "$raw" "$bundle" +git archive --format=tar "$commit" -- bin/single-node/ bin/observability/ sql/ NOTICE licenses/LICENSE-BSD-3-Clause.txt | "$gnu_tar" -x -C "$raw" + +for f in docker-compose.yml nginx.conf litellm-config.yaml LICENSE DISCLAIMER README.md .env; do + cp "$raw/bin/single-node/$f" "$bundle/" +done +cp "$raw/NOTICE" "$bundle/" +# bin/single-node/LICENSE points at licenses/LICENSE-BSD-3-Clause.txt. +mkdir -p "$bundle/licenses" +cp "$raw/licenses/LICENSE-BSD-3-Clause.txt" "$bundle/licenses/" +cp -R "$raw/sql" "$bundle/" +# Read by the opt-in observability compose profiles. +cp -R "$raw/bin/observability" "$bundle/observability" +if [[ -d "$raw/bin/single-node/examples" ]]; then + cp -R "$raw/bin/single-node/examples" "$bundle/" +fi + +# The repo mounts ../../sql and ../observability relative to bin/single-node/; +# the bundle keeps both next to docker-compose.yml. +sed -e 's|\.\./\.\./sql|./sql|g' -e 's|\.\./observability|./observability|g' \ + "$bundle/docker-compose.yml" > "$bundle/docker-compose.yml.tmp" +mv "$bundle/docker-compose.yml.tmp" "$bundle/docker-compose.yml" + +set_env_var "$bundle/.env" IMAGE_REGISTRY "$image_registry" +set_env_var "$bundle/.env" IMAGE_TAG "$image_tag" +set_env_var "$bundle/.env" TEXERA_SERVICE_LOG_LEVEL ERROR + +reproducible_tar_gz "$work/compose" "$compose_name" "$out_dir/${compose_name}.tar.gz" + +echo "Created ${out_dir}/${src_name}.tar.gz" +echo "Created ${out_dir}/${compose_name}.tar.gz" +echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} (commit ${commit})" diff --git a/.github/scripts/test_create_release_tarballs.sh b/.github/scripts/test_create_release_tarballs.sh new file mode 100755 index 00000000000..aa89c8e8d6a --- /dev/null +++ b/.github/scripts/test_create_release_tarballs.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Regression tests for create-release-tarballs.sh. Builds the release archives +# for a throwaway repository twice, under a different umask, clock, time zone, +# working directory and SOURCE_DATE_EPOCH, and requires byte-identical output. +# Also pins the bundle's contents and the script's rejection of bad input. + +set -uo pipefail + +command -v python3 >/dev/null || { echo "python3 is required to run these tests" >&2; exit 1; } + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +create="$script_dir/create-release-tarballs.sh" +work="$(mktemp -d 2>/dev/null || mktemp -d -t release-tarballs)" +trap 'rm -rf "$work"' EXIT +rc=0 + +pass() { echo "ok: $1"; } +failed() { echo "FAIL: $1"; rc=1; } + +VERSION="9.9.9-incubating" +SRC="apache-texera-${VERSION}-src.tar.gz" +COMPOSE="apache-texera-${VERSION}-docker-compose.tar.gz" + +repo="$work/repo" +mkdir -p "$repo/bin/single-node/examples" "$repo/bin/observability/parca" "$repo/sql/updates" "$repo/licenses" +( + cd "$repo" + git init -q + git config user.email "test@example.com" + git config user.name "test" + printf 'services:\n postgres:\n volumes:\n - ../../sql:/docker-entrypoint-initdb.d\n' \ + > bin/single-node/docker-compose.yml + printf ' parca-agent:\n env_file:\n - ../observability/parca/parca-agent.env\n' \ + >> bin/single-node/docker-compose.yml + echo "PARCA=1" > bin/observability/parca/parca-agent.env + printf 'TEXERA_SERVICE_LOG_LEVEL=INFO\nIMAGE_TAG=latest\n' > bin/single-node/.env + for f in nginx.conf litellm-config.yaml LICENSE DISCLAIMER README.md; do + echo "$f" > "bin/single-node/$f" + done + printf '#!/usr/bin/env bash\necho load\n' > bin/single-node/examples/load-examples.sh + chmod +x bin/single-node/examples/load-examples.sh + echo "CREATE TABLE t();" > sql/texera_ddl.sql + echo "ALTER TABLE t;" > "sql/updates/01-ünïcode.sql" + echo "NOTICE" > NOTICE + echo "BSD" > licenses/LICENSE-BSD-3-Clause.txt + echo "not in the bundle" > README.md + git add -A + GIT_COMMITTER_DATE="2024-02-03T04:05:06Z" git commit -qm "release" --date="2024-02-03T04:05:06Z" + git tag v9.9.9-incubating-rc1 + git tag -a -m "annotated" v9.9.9-incubating-rc1-annotated +) +commit_time=$(git -C "$repo" log -1 --format=%ct) + +run() { (cd "$repo" && "$create" "$@") >"$work/run.log" 2>&1; } + +# --- the same tag twice, with every host input that could leak changed --- +(umask 022; cd "$repo" && TZ=UTC "$create" v9.9.9-incubating-rc1 "$VERSION" ghcr.io/apache 1.0.0 "$work/out1") \ + >/dev/null 2>&1 || failed "first build should succeed" +sleep 1 +(umask 077; cd "$repo/sql" && TZ=Asia/Tokyo SOURCE_DATE_EPOCH=1 \ + "$create" v9.9.9-incubating-rc1-annotated "$VERSION" ghcr.io/apache 1.0.0 "$work/out2") \ + >/dev/null 2>&1 || failed "second build should succeed" + +for a in "$SRC" "$COMPOSE"; do + if [[ -f "$work/out1/$a" ]] && cmp -s "$work/out1/$a" "$work/out2/$a"; then + pass "$a is byte-identical across umask, clock, TZ, cwd and tag kind" + else + failed "$a differs between two builds of the same commit" + fi +done + +# --- archive metadata comes from the commit, not the machine --- +if python3 - "$work/out2" "$commit_time" "$SRC" "$COMPOSE" <<'EOF' +import gzip, sys, tarfile +out, commit_time, *names = sys.argv[1], int(sys.argv[2]), *sys.argv[3:] +for name in names: + raw = open(f"{out}/{name}", "rb").read() + assert raw[4:8] == b"\0\0\0\0", f"{name}: gzip header carries a timestamp" + with tarfile.open(f"{out}/{name}") as tar: + members = tar.getmembers() + paths = [m.name for m in members] + assert paths == sorted(paths), f"{name}: entries not sorted: {paths}" + for m in members: + assert m.mtime == commit_time, f"{m.name}: mtime {m.mtime} != {commit_time}" + assert (m.uid, m.gid, m.uname, m.gname) == (0, 0, "", ""), f"{m.name}: owner leaked" + expected = 0o755 if m.isdir() or m.name.endswith(".sh") else 0o644 + assert m.mode == expected, f"{m.name}: mode {oct(m.mode)} != {oct(expected)}" +EOF +then + pass "entries are sorted, owned by 0:0, stamped with the commit time, umask-free" +else + failed "archive metadata leaked from the build machine" +fi + +# --- the compose bundle is assembled and patched as before --- +mkdir -p "$work/x" && tar -xzf "$work/out1/$COMPOSE" -C "$work/x" +b="$work/x/apache-texera-${VERSION}-docker-compose" +env_file="$(cat "$b/.env" 2>/dev/null)" +expected_env=$'TEXERA_SERVICE_LOG_LEVEL=ERROR\nIMAGE_TAG=1.0.0\nIMAGE_REGISTRY=ghcr.io/apache' +if [[ "$env_file" == "$expected_env" ]]; then + pass ".env: IMAGE_TAG replaced, IMAGE_REGISTRY appended, log level forced to ERROR" +else + failed ".env not patched as expected: $env_file" +fi +if grep -q -- '- ./sql:/docker-entrypoint-initdb.d' "$b/docker-compose.yml" \ + && grep -q -- '- ./observability/parca/parca-agent.env' "$b/docker-compose.yml" \ + && ! grep -q '\.\./' "$b/docker-compose.yml"; then + pass "docker-compose.yml mounts ./sql and ./observability" +else + failed "docker-compose.yml sql or observability paths not rewritten" +fi +if [[ -f "$b/sql/updates/01-ünïcode.sql" && -x "$b/examples/load-examples.sh" && -f "$b/NOTICE" && -f "$b/licenses/LICENSE-BSD-3-Clause.txt" \ + && -f "$b/observability/parca/parca-agent.env" && ! -e "$b/README.md.tmp" ]] \ + && [[ "$(cat "$b/README.md")" == "README.md" ]]; then + pass "bundle carries sql/, examples/ (still executable), observability/, NOTICE, licenses/ and the single-node README" +else + failed "bundle contents incomplete" +fi + +# --- a different image tag changes only the compose bundle --- +run v9.9.9-incubating-rc1 "$VERSION" ghcr.io/apache 2.0.0 "$work/out3" || failed "third build should succeed" +if cmp -s "$work/out1/$SRC" "$work/out3/$SRC" && ! cmp -s "$work/out1/$COMPOSE" "$work/out3/$COMPOSE"; then + pass "image tag affects the compose bundle only" +else + failed "image tag should change the compose bundle and nothing else" +fi + +# --- bad input is rejected before anything is written --- +expect_rejected() { + local label="$1" want="$2"; shift 2 + rm -rf "$work/bad" + run "$@" + local got=$? + if [[ "$got" -eq "$want" && ! -e "$work/bad/$SRC" && ! -e "$work/bad/$COMPOSE" ]]; then + pass "$label -> exit $want, nothing written" + else + failed "$label should exit $want without output (got $got): $(cat "$work/run.log")" + fi +} +expect_rejected "unknown tag" 1 v0.0.0-missing "$VERSION" ghcr.io/apache 1.0.0 "$work/bad" +expect_rejected "too few arguments" 2 v9.9.9-incubating-rc1 "$VERSION" ghcr.io/apache +expect_rejected "image tag with a sed delimiter" 2 v9.9.9-incubating-rc1 "$VERSION" ghcr.io/apache '1.0|x' "$work/bad" +expect_rejected "registry with a sed metacharacter" 2 v9.9.9-incubating-rc1 "$VERSION" 'ghcr.io/a&b' 1.0.0 "$work/bad" +expect_rejected "empty version" 2 v9.9.9-incubating-rc1 "" ghcr.io/apache 1.0.0 "$work/bad" +expect_rejected "version with a path separator" 2 v9.9.9-incubating-rc1 "9/../x" ghcr.io/apache 1.0.0 "$work/bad" +expect_rejected "image tag with a colon" 2 v9.9.9-incubating-rc1 "$VERSION" ghcr.io/apache 'a:b' "$work/bad" + +if [[ "$rc" -ne 0 ]]; then + echo "create-release-tarballs regression tests FAILED" + exit 1 +fi +echo "create-release-tarballs regression tests passed" diff --git a/.github/workflows/build-and-push-images.yml b/.github/workflows/build-and-push-images.yml index 1f69cb36070..3f96a584ad4 100644 --- a/.github/workflows/build-and-push-images.yml +++ b/.github/workflows/build-and-push-images.yml @@ -357,8 +357,21 @@ jobs: sudo rm -rf /usr/local/share/boost df -h + # Image digests can change between BuildKit versions, so the builder is + # pinned for voters to reproduce the published images (bin/README.md). - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + with: + driver-opts: image=moby/buildkit:v0.33.1 + + # Reproducible images: SOURCE_DATE_EPOCH (the commit time) fixes every + # timestamp in the image, and PACKAGE_SNAPSHOT (that commit's UTC day) + # pins apt and pip to the archives as they stood that morning. + - name: Derive SOURCE_DATE_EPOCH and PACKAGE_SNAPSHOT + run: | + SOURCE_DATE_EPOCH="$(git log -1 --format=%ct)" + echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" >> "$GITHUB_ENV" + echo "PACKAGE_SNAPSHOT=$(date -u -d "@$SOURCE_DATE_EPOCH" +%F)" >> "$GITHUB_ENV" - name: Log in to GitHub Container Registry if: startsWith(needs.set-parameters.outputs.docker_registry, 'ghcr.io/') @@ -381,7 +394,11 @@ jobs: context: ${{ matrix.context }} file: ${{ matrix.dockerfile }} platforms: linux/amd64 - push: true + # Pushes, with every layer mtime clamped to SOURCE_DATE_EPOCH. + outputs: type=registry,rewrite-timestamp=true + build-args: | + SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} + PACKAGE_SNAPSHOT=${{ env.PACKAGE_SNAPSHOT }} tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-amd64 cache-from: type=gha,scope=${{ matrix.image_name }}-amd64 cache-to: type=gha,mode=max,scope=${{ matrix.image_name }}-amd64 @@ -445,6 +462,17 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + with: + driver-opts: image=moby/buildkit:v0.33.1 + + # Reproducible images: SOURCE_DATE_EPOCH (the commit time) fixes every + # timestamp in the image, and PACKAGE_SNAPSHOT (that commit's UTC day) + # pins apt and pip to the archives as they stood that morning. + - name: Derive SOURCE_DATE_EPOCH and PACKAGE_SNAPSHOT + run: | + SOURCE_DATE_EPOCH="$(git log -1 --format=%ct)" + echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" >> "$GITHUB_ENV" + echo "PACKAGE_SNAPSHOT=$(date -u -d "@$SOURCE_DATE_EPOCH" +%F)" >> "$GITHUB_ENV" - name: Log in to GitHub Container Registry if: startsWith(needs.set-parameters.outputs.docker_registry, 'ghcr.io/') @@ -467,7 +495,11 @@ jobs: context: ${{ matrix.context }} file: ${{ matrix.dockerfile }} platforms: linux/arm64 - push: true + # Pushes, with every layer mtime clamped to SOURCE_DATE_EPOCH. + outputs: type=registry,rewrite-timestamp=true + build-args: | + SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} + PACKAGE_SNAPSHOT=${{ env.PACKAGE_SNAPSHOT }} tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-arm64 cache-from: type=gha,scope=${{ matrix.image_name }}-arm64 cache-to: type=gha,mode=max,scope=${{ matrix.image_name }}-arm64 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3d44cc34d7e..9f590eb4965 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -519,10 +519,10 @@ jobs: -e RUSTFS_ACCESS_KEY=texera_rustfs \ -e RUSTFS_SECRET_KEY=password \ -e RUSTFS_REGION=us-west-2 \ - rustfs/rustfs:1.0.0-rc.6 + rustfs/rustfs:1.0.0 else curl -sSfL -o /tmp/rustfs.zip \ - https://dl.rustfs.com/artifacts/rustfs/release/rustfs-macos-aarch64-v1.0.0-rc.6.zip + https://dl.rustfs.com/artifacts/rustfs/release/rustfs-macos-aarch64-v1.0.0.zip unzip -q -o /tmp/rustfs.zip -d /tmp/rustfs-bin chmod +x /tmp/rustfs-bin/rustfs mkdir -p /tmp/rustfs-data @@ -1059,7 +1059,7 @@ jobs: -e RUSTFS_ACCESS_KEY=texera_rustfs \ -e RUSTFS_SECRET_KEY=password \ -e RUSTFS_REGION=us-west-2 \ - rustfs/rustfs:1.0.0-rc.6 + rustfs/rustfs:1.0.0 for i in $(seq 1 15); do curl -sf http://localhost:9000/health && break echo "Waiting for RustFS... (attempt $i)"; sleep 1 diff --git a/.github/workflows/create-release-candidate.yml b/.github/workflows/create-release-candidate.yml index d73b1df6dda..017d8f6e919 100644 --- a/.github/workflows/create-release-candidate.yml +++ b/.github/workflows/create-release-candidate.yml @@ -67,6 +67,9 @@ jobs: uses: actions/checkout@v7 with: fetch-depth: 0 # Full history for proper tagging + # Build with the tag's own create-release-tarballs.sh: the copy a + # voter runs to reproduce the artifacts. + ref: ${{ github.event.inputs.tag }} - name: Validate tag exists run: | @@ -133,109 +136,17 @@ jobs: echo "Bundled IMAGE_TAG: $IMAGE_TAG" echo "Staging directory: dist/dev/incubator/texera/$RC_DIR" - - name: Create source tarball + - name: Create source tarball and Docker Compose bundle run: | - TAG_NAME="${{ steps.vars.outputs.tag_name }}" - SRC_TARBALL="${{ steps.vars.outputs.src_tarball }}" - VERSION="${{ steps.vars.outputs.version }}" - - TEMP_DIR=$(mktemp -d) - - # Export the git repository at the tag - git archive --format=tar --prefix="apache-texera-${VERSION}-src/" "$TAG_NAME" | tar -x -C "$TEMP_DIR" - - # Create tarball - cd "$TEMP_DIR" - tar -czf "$GITHUB_WORKSPACE/$SRC_TARBALL" "apache-texera-${VERSION}-src" - - cd "$GITHUB_WORKSPACE" - - # Verify tarball was created - if [[ ! -f "$SRC_TARBALL" ]]; then - echo "Error: Source tarball was not created" - exit 1 - fi - - # Show tarball info - ls -lh "$SRC_TARBALL" - echo "✓ Created source tarball: $SRC_TARBALL" - - - name: Create Docker Compose deployment bundle - run: | - VERSION="${{ steps.vars.outputs.version }}" - TAG_NAME="${{ steps.vars.outputs.tag_name }}" - IMAGE_REGISTRY="${{ steps.vars.outputs.image_registry }}" - IMAGE_TAG="${{ steps.vars.outputs.image_tag }}" - COMPOSE_TARBALL="${{ steps.vars.outputs.compose_tarball }}" - - TEMP_DIR=$(mktemp -d) - BUNDLE_DIR="$TEMP_DIR/apache-texera-${VERSION}-docker-compose" - mkdir -p "$BUNDLE_DIR" - - # Export the single-node directory from the tagged source - mkdir -p "$TEMP_DIR/_raw" - git archive --format=tar "$TAG_NAME" -- bin/single-node/ bin/observability/ sql/ NOTICE licenses/LICENSE-BSD-3-Clause.txt | tar -x -C "$TEMP_DIR/_raw" - - # Copy deployment files - cp "$TEMP_DIR/_raw/bin/single-node/docker-compose.yml" "$BUNDLE_DIR/" - cp "$TEMP_DIR/_raw/bin/single-node/nginx.conf" "$BUNDLE_DIR/" - cp "$TEMP_DIR/_raw/bin/single-node/litellm-config.yaml" "$BUNDLE_DIR/" - cp "$TEMP_DIR/_raw/bin/single-node/LICENSE" "$BUNDLE_DIR/" - cp "$TEMP_DIR/_raw/NOTICE" "$BUNDLE_DIR/" - cp "$TEMP_DIR/_raw/bin/single-node/DISCLAIMER" "$BUNDLE_DIR/" - mkdir -p "$BUNDLE_DIR/licenses" - cp "$TEMP_DIR/_raw/licenses/LICENSE-BSD-3-Clause.txt" "$BUNDLE_DIR/licenses/" - cp -r "$TEMP_DIR/_raw/sql" "$BUNDLE_DIR/" - - # Observability configs (OTel collector + Parca) referenced by the - # opt-in observability compose profiles. Shipped next to the compose - # file so the ../observability mounts resolve in the flattened bundle. - cp -r "$TEMP_DIR/_raw/bin/observability" "$BUNDLE_DIR/observability" - - # Patch the SQL mount path for the self-contained bundle layout - # In the repo it's ../../sql (relative to bin/single-node/), in the bundle it's ./sql - sed -i 's|../../sql|./sql|g' "$BUNDLE_DIR/docker-compose.yml" - - # Patch the observability mount paths: ../observability (relative to - # bin/single-node/ in the repo) becomes ./observability in the bundle. - sed -i 's|../observability|./observability|g' "$BUNDLE_DIR/docker-compose.yml" - - # Generate a release-pinned .env file with the version tag - # Start from the source .env and ensure IMAGE_REGISTRY and IMAGE_TAG are set - cp "$TEMP_DIR/_raw/bin/single-node/.env" "$BUNDLE_DIR/.env" - # Replace if line exists, otherwise append - if grep -q '^IMAGE_REGISTRY=' "$BUNDLE_DIR/.env"; then - sed -i "s|^IMAGE_REGISTRY=.*|IMAGE_REGISTRY=${IMAGE_REGISTRY}|" "$BUNDLE_DIR/.env" - else - echo "IMAGE_REGISTRY=${IMAGE_REGISTRY}" >> "$BUNDLE_DIR/.env" - fi - if grep -q '^IMAGE_TAG=' "$BUNDLE_DIR/.env"; then - sed -i "s|^IMAGE_TAG=.*|IMAGE_TAG=${IMAGE_TAG}|" "$BUNDLE_DIR/.env" - else - echo "IMAGE_TAG=${IMAGE_TAG}" >> "$BUNDLE_DIR/.env" - fi - if grep -q '^TEXERA_SERVICE_LOG_LEVEL=' "$BUNDLE_DIR/.env"; then - sed -i "s|^TEXERA_SERVICE_LOG_LEVEL=.*|TEXERA_SERVICE_LOG_LEVEL=ERROR|" "$BUNDLE_DIR/.env" - else - echo "TEXERA_SERVICE_LOG_LEVEL=ERROR" >> "$BUNDLE_DIR/.env" - fi - - # Include the README from the repo - cp "$TEMP_DIR/_raw/bin/single-node/README.md" "$BUNDLE_DIR/" - - # Include example datasets, workflows, and the loader script - if [ -d "$TEMP_DIR/_raw/bin/single-node/examples" ]; then - cp -r "$TEMP_DIR/_raw/bin/single-node/examples" "$BUNDLE_DIR/" - echo "✓ Included examples directory (datasets, workflows, load-examples.sh)" - fi - - # Create tarball - cd "$TEMP_DIR" - tar -czf "$GITHUB_WORKSPACE/$COMPOSE_TARBALL" "apache-texera-${VERSION}-docker-compose" - - cd "$GITHUB_WORKSPACE" - ls -lh "$COMPOSE_TARBALL" - echo "✓ Created Docker Compose bundle: $COMPOSE_TARBALL" + # Reproducible: voters can rerun this script on the tag with the + # IMAGE_REGISTRY/IMAGE_TAG from the bundle's .env and compare checksums. + ./.github/scripts/create-release-tarballs.sh \ + "${{ steps.vars.outputs.tag_name }}" \ + "${{ steps.vars.outputs.version }}" \ + "${{ steps.vars.outputs.image_registry }}" \ + "${{ steps.vars.outputs.image_tag }}" \ + "$GITHUB_WORKSPACE" + ls -lh "${{ steps.vars.outputs.src_tarball }}" "${{ steps.vars.outputs.compose_tarball }}" - name: Import GPG key run: | @@ -272,6 +183,7 @@ jobs: RC_DIR="${{ steps.vars.outputs.rc_dir }}" COMMIT_HASH="${{ steps.vars.outputs.commit_hash }}" IMAGE_REGISTRY="${{ steps.vars.outputs.image_registry }}" + IMAGE_TAG="${{ steps.vars.outputs.image_tag }}" # Get GPG key ID from the imported key GPG_KEY_ID=$(gpg --list-secret-keys --keyid-format LONG | grep 'sec' | head -n1 | awk '{print $2}' | cut -d'/' -f2) @@ -289,6 +201,7 @@ jobs: sed -i "s|\${GPG_KEY_ID}|${GPG_KEY_ID}|g" vote-email.txt sed -i "s|\${GPG_EMAIL}|${GPG_EMAIL}|g" vote-email.txt sed -i "s|\${IMAGE_REGISTRY}|${IMAGE_REGISTRY}|g" vote-email.txt + sed -i "s|\${IMAGE_TAG}|${IMAGE_TAG}|g" vote-email.txt echo "✓ Vote email template generated!" diff --git a/.gitignore b/.gitignore index adb3ef7844d..88a384f755e 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,8 @@ values-*.yaml # Any environment-specific value overrides # Ignore nested node modules **/node_modules **/package-lock.json +# The one npm-installed image; its lockfile makes `npm ci` reproducible. +!bin/y-websocket-server/package-lock.json .env # agent-service is Bun-based; yarn/npm artifacts aren't needed diff --git a/bin/README.md b/bin/README.md index 4bbe7c5cc49..c74655786e4 100644 --- a/bin/README.md +++ b/bin/README.md @@ -47,6 +47,34 @@ must be built with the project root as the Docker build context: docker build -f bin/dockerfiles/texera-web-application.dockerfile -t your-repo/texera-web-application:test . ``` +### Reproducible builds + +Built from the same commit with the same builder, an image comes out byte for +byte the same. Everything that would otherwise drift is pinned: + +| Input | Pinned by | +| --- | --- | +| Base images | the digest in each `FROM` (Renovate refreshes them weekly) | +| apt and pip packages | `PACKAGE_SNAPSHOT`, a UTC date: [`dockerfiles/snapshot/install.sh`](dockerfiles/snapshot/install.sh) installs from snapshot.ubuntu.com / snapshot.debian.org and from PyPI as they stood at 00:00 that day | +| Timestamps (jars, dist zips, the frontend build number, `/etc/shadow`, layer mtimes) | `SOURCE_DATE_EPOCH`, the commit time | +| BuildKit | `moby/buildkit:v0.33.1`, set in `.github/workflows/build-and-push-images.yml`; other versions can produce different digests | + +`build-images.sh` and the image workflow pass both. To rebuild one image by +hand (after generating the jOOQ sources with `sbt DAO/jooqGenerate`): + +```bash +docker buildx create --name texera-repro --driver docker-container \ + --driver-opt image=moby/buildkit:v0.33.1 --use +export SOURCE_DATE_EPOCH=$(git log -1 --format=%ct) +docker buildx build --platform linux/amd64 \ + --build-arg SOURCE_DATE_EPOCH --build-arg PACKAGE_SNAPSHOT=$(date -u -d @$SOURCE_DATE_EPOCH +%F) \ + --output type=oci,dest=config-service.tar,rewrite-timestamp=true \ + -f bin/dockerfiles/config-service.dockerfile . +``` + +`y-websocket-server/Dockerfile` rebuilds the same way, with +`bin/y-websocket-server` as the build context instead of `.`. + | Script | Purpose | | --- | --- | | `build-images.sh` | Convenience wrapper to build (and push) platform-dependent images. Run `bin/build-images.sh --help`. | diff --git a/bin/build-images.sh b/bin/build-images.sh index 1f5960784da..5616fedb903 100755 --- a/bin/build-images.sh +++ b/bin/build-images.sh @@ -107,6 +107,15 @@ docker buildx create --name texera-builder --use --bootstrap > /dev/null 2>&1 || cd "$(dirname "$0")" +# Reproducible images: the commit time fixes every timestamp in the image, and +# its UTC day pins apt and pip (bin/dockerfiles/snapshot/install.sh). Passed only +# when known: an empty SOURCE_DATE_EPOCH would fail the sbt dist. +epoch_args=() +if SOURCE_DATE_EPOCH="$(git log -1 --format=%ct 2>/dev/null)" && [[ -n "$SOURCE_DATE_EPOCH" ]]; then + epoch_args=(--build-arg "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" + --build-arg "PACKAGE_SNAPSHOT=$(date -u -d "@$SOURCE_DATE_EPOCH" +%F 2>/dev/null || date -u -r "$SOURCE_DATE_EPOCH" +%F)") +fi + # Auto-detect Dockerfiles in bin/dockerfiles/. We stay cd'd in bin/ so the # pylsp + y-websocket-server build stages below keep working (they use # relative paths like ./pylsp and ./y-websocket-server from here), and so @@ -135,9 +144,10 @@ for dockerfile in "${dockerfiles[@]}"; do docker buildx build \ --platform "$PLATFORM" \ + "${epoch_args[@]}" \ -f "$dockerfile" \ -t "$image" \ - --push \ + --output type=registry,rewrite-timestamp=true \ .. done @@ -159,9 +169,10 @@ if should_build "y-websocket-server"; then echo "👉 Building $image from y-websocket-server/Dockerfile" docker buildx build \ --platform "$PLATFORM" \ + "${epoch_args[@]}" \ -f "y-websocket-server/Dockerfile" \ -t "$image" \ - --push \ + --output type=registry,rewrite-timestamp=true \ ./y-websocket-server fi diff --git a/bin/dockerfiles/access-control-service.dockerfile b/bin/dockerfiles/access-control-service.dockerfile index 43d4c4e5a47..e86bd4ab5c5 100644 --- a/bin/dockerfiles/access-control-service.dockerfile +++ b/bin/dockerfiles/access-control-service.dockerfile @@ -24,7 +24,7 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11 AS build +FROM docker.io/sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11@sha256:c20fad6183112843d6c87fd54bb9b507a424ab7893c6a570b1624acf2e0324c4 AS build # Set working directory WORKDIR /texera @@ -37,27 +37,25 @@ COPY build.sbt build.sbt COPY .jvmopts .jvmopts # Update system and install dependencies -RUN apt-get update && apt-get install -y \ - netcat \ - unzip \ - libpq-dev \ - && apt-get clean +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt netcat unzip libpq-dev -# Add .git for runtime calls to jgit from OPversion -COPY .git .git COPY LICENSE NOTICE DISCLAIMER ./ COPY licenses/ licenses/ +# Pins jar and dist-zip entry timestamps; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN sbt clean AccessControlService/dist # Unzip the texera binary RUN unzip access-control-service/target/universal/access-control-service-*.zip -d target/ -FROM eclipse-temurin:17-jre-jammy AS runtime +FROM docker.io/library/eclipse-temurin:17-jre-jammy@sha256:97137382c6f0c30427d9b7c44ad8b2d55ac823b0768c171d81a643ed219023c5 AS runtime WORKDIR /texera -COPY --from=build /texera/.git /texera/.git # Copy the built texera binary from the build phase COPY --from=build /texera/target/access-control-service* /texera/ # Copy resources directories from build phase @@ -71,6 +69,9 @@ COPY --from=build /texera/access-control-service/NOTICE-binary /texera/NOTICE COPY --from=build /texera/licenses /texera/licenses COPY --from=build /texera/DISCLAIMER /texera/ +# Dates the texera account in /etc/shadow; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN groupadd --system --gid 1001 texera \ && useradd --system --uid 1001 --gid texera --home-dir /texera --no-create-home texera \ && chown -R texera:texera /texera diff --git a/bin/dockerfiles/agent-service.dockerfile b/bin/dockerfiles/agent-service.dockerfile index 7edf38287aa..6deaa5005d9 100644 --- a/bin/dockerfiles/agent-service.dockerfile +++ b/bin/dockerfiles/agent-service.dockerfile @@ -24,13 +24,16 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -FROM oven/bun:1-alpine +FROM docker.io/oven/bun:1.3.3-alpine@sha256:d2bc1fbc3afcd3d70afc2bb2544235bf559caae2a3084e9abed126e233797511 WORKDIR /app COPY agent-service/package.json agent-service/bun.lock ./ -RUN bun install --frozen-lockfile --production +# The download cache holds registry manifests fetched at build time; node_modules +# keeps its own copy of every file. +RUN bun install --frozen-lockfile --production \ + && rm -rf /root/.bun/install/cache COPY agent-service/src ./src COPY agent-service/tsconfig.json ./ @@ -40,8 +43,11 @@ COPY NOTICE ./NOTICE COPY DISCLAIMER ./DISCLAIMER COPY licenses ./licenses +# busybox adduser stamps the build day into /etc/shadow and ignores +# SOURCE_DATE_EPOCH; blank that field so the layer does not depend on the date. RUN addgroup -S -g 1001 texera \ && adduser -S -u 1001 -G texera -h /app texera \ + && sed -i 's/^texera:!:[0-9]*:/texera:!::/' /etc/shadow \ && chown -R texera:texera /app USER texera diff --git a/bin/dockerfiles/computing-unit-master.dockerfile b/bin/dockerfiles/computing-unit-master.dockerfile index df9928323ce..c523e367e63 100644 --- a/bin/dockerfiles/computing-unit-master.dockerfile +++ b/bin/dockerfiles/computing-unit-master.dockerfile @@ -24,7 +24,7 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11 AS build +FROM docker.io/sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11@sha256:c20fad6183112843d6c87fd54bb9b507a424ab7893c6a570b1624acf2e0324c4 AS build # Set working directory WORKDIR /texera @@ -38,14 +38,9 @@ COPY .jvmopts .jvmopts # python3-minimal is needed by bin/licensing/concat_license_binary.py; # python3-pip installs the betterproto plugin; unzip + curl fetch protoc. -RUN apt-get update && apt-get install -y \ - netcat \ - unzip \ - curl \ - libpq-dev \ - python3-minimal \ - python3-pip \ - && apt-get clean +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt netcat unzip curl libpq-dev python3-minimal python3-pip # Install protoc (version pinned in bin/protoc-version.txt) and the # betterproto plugin (version pinned via amber/requirements.txt as a @@ -54,7 +49,8 @@ RUN apt-get update && apt-get install -y \ # amber/src/main/python/proto/ before `sbt dist`. COPY bin/protoc-version.txt bin/protoc-version.txt COPY bin/python-proto-gen.sh bin/python-proto-gen.sh -RUN PROTOC_VERSION=$(cat bin/protoc-version.txt) \ +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + PROTOC_VERSION=$(cat bin/protoc-version.txt) \ && case "$(uname -m)" in \ x86_64 | amd64) PROTOC_ARCH=x86_64 ;; \ aarch64 | arm64) PROTOC_ARCH=aarch_64 ;; \ @@ -64,15 +60,16 @@ RUN PROTOC_VERSION=$(cat bin/protoc-version.txt) \ && unzip -o /tmp/protoc.zip -d /usr/local \ && chmod +x /usr/local/bin/protoc \ && rm /tmp/protoc.zip \ - && pip3 install --no-cache-dir -c amber/requirements.txt 'betterproto[compiler]' \ + && bash /snapshot/install.sh pip -c amber/requirements.txt 'betterproto[compiler]' \ && bash bin/python-proto-gen.sh -# Add .git for runtime calls to jgit from OPversion -COPY .git .git COPY LICENSE NOTICE DISCLAIMER ./ COPY licenses/ licenses/ COPY bin/licensing/ bin/licensing/ +# Pins jar and dist-zip entry timestamps; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN sbt clean WorkflowExecutionService/dist # Unzip the texera binary @@ -87,7 +84,7 @@ RUN python3 bin/licensing/concat_license_binary.py amber/LICENSE-binary-combined amber/LICENSE-binary-java \ amber/LICENSE-binary-python -FROM eclipse-temurin:17-jdk-jammy AS runtime +FROM docker.io/library/eclipse-temurin:17-jdk-jammy@sha256:60fcdd4a85c94a23ef86d0d55867c6a8dbc4f7c3efc8286e259956c3ea4ef08c AS runtime WORKDIR /texera/amber @@ -95,20 +92,17 @@ COPY --from=build /texera/amber/requirements.txt /tmp/requirements.txt COPY --from=build /texera/amber/operator-requirements.txt /tmp/operator-requirements.txt # Install Python runtime dependencies -RUN apt-get update && apt-get install -y \ - python3-pip \ - python3-dev \ - python3-venv \ - libpq-dev \ - && apt-get clean +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt python3-pip python3-dev python3-venv libpq-dev # Install Python packages -RUN pip3 install --upgrade pip setuptools wheel && \ - pip3 install -r /tmp/requirements.txt && \ - pip3 install -r /tmp/operator-requirements.txt +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh pip --upgrade setuptools wheel \ + && bash /snapshot/install.sh pip-requirements /tmp/requirements.txt \ + && bash /snapshot/install.sh pip-requirements /tmp/operator-requirements.txt # Copy the built texera binary from the build phase -COPY --from=build /texera/.git /texera/amber/.git COPY --from=build /texera/amber/target/amber-* /texera/amber/ # Copy resources directories from build phase COPY --from=build /texera/common/config/src/main/resources /texera/amber/common/config/src/main/resources @@ -124,6 +118,9 @@ COPY --from=build /texera/amber/NOTICE-binary /texera/NOTICE COPY --from=build /texera/licenses /texera/licenses COPY --from=build /texera/DISCLAIMER /texera/ +# Dates the texera account in /etc/shadow; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN groupadd --system --gid 1001 texera \ && useradd --system --uid 1001 --gid texera --home-dir /texera --no-create-home texera \ && chown -R texera:texera /texera diff --git a/bin/dockerfiles/computing-unit-worker.dockerfile b/bin/dockerfiles/computing-unit-worker.dockerfile index fc89c0f352d..e3078b1cf34 100644 --- a/bin/dockerfiles/computing-unit-worker.dockerfile +++ b/bin/dockerfiles/computing-unit-worker.dockerfile @@ -24,7 +24,7 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11 AS build +FROM docker.io/sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11@sha256:c20fad6183112843d6c87fd54bb9b507a424ab7893c6a570b1624acf2e0324c4 AS build # Set working directory WORKDIR /texera @@ -38,14 +38,9 @@ COPY .jvmopts .jvmopts # python3-minimal is needed by bin/licensing/concat_license_binary.py; # python3-pip installs the betterproto plugin; unzip + curl fetch protoc. -RUN apt-get update && apt-get install -y \ - netcat \ - unzip \ - curl \ - libpq-dev \ - python3-minimal \ - python3-pip \ - && apt-get clean +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt netcat unzip curl libpq-dev python3-minimal python3-pip # Install protoc (version pinned in bin/protoc-version.txt) and the # betterproto plugin (version pinned via amber/requirements.txt as a @@ -54,7 +49,8 @@ RUN apt-get update && apt-get install -y \ # amber/src/main/python/proto/ before `sbt dist`. COPY bin/protoc-version.txt bin/protoc-version.txt COPY bin/python-proto-gen.sh bin/python-proto-gen.sh -RUN PROTOC_VERSION=$(cat bin/protoc-version.txt) \ +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + PROTOC_VERSION=$(cat bin/protoc-version.txt) \ && case "$(uname -m)" in \ x86_64 | amd64) PROTOC_ARCH=x86_64 ;; \ aarch64 | arm64) PROTOC_ARCH=aarch_64 ;; \ @@ -64,15 +60,16 @@ RUN PROTOC_VERSION=$(cat bin/protoc-version.txt) \ && unzip -o /tmp/protoc.zip -d /usr/local \ && chmod +x /usr/local/bin/protoc \ && rm /tmp/protoc.zip \ - && pip3 install --no-cache-dir -c amber/requirements.txt 'betterproto[compiler]' \ + && bash /snapshot/install.sh pip -c amber/requirements.txt 'betterproto[compiler]' \ && bash bin/python-proto-gen.sh -# Add .git for runtime calls to jgit from OPversion -COPY .git .git COPY LICENSE NOTICE DISCLAIMER ./ COPY licenses/ licenses/ COPY bin/licensing/ bin/licensing/ +# Pins jar and dist-zip entry timestamps; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN sbt clean WorkflowExecutionService/dist # Unzip the texera binary @@ -87,7 +84,7 @@ RUN python3 bin/licensing/concat_license_binary.py amber/LICENSE-binary-combined amber/LICENSE-binary-java \ amber/LICENSE-binary-python -FROM eclipse-temurin:17-jre-jammy AS runtime +FROM docker.io/library/eclipse-temurin:17-jre-jammy@sha256:97137382c6f0c30427d9b7c44ad8b2d55ac823b0768c171d81a643ed219023c5 AS runtime WORKDIR /texera/amber @@ -95,17 +92,15 @@ COPY --from=build /texera/amber/requirements.txt /tmp/requirements.txt COPY --from=build /texera/amber/operator-requirements.txt /tmp/operator-requirements.txt # Install Python runtime dependencies -RUN apt-get update && apt-get install -y \ - python3-pip \ - python3-dev \ - libpq-dev \ - && apt-get clean +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt python3-pip python3-dev libpq-dev # Install Python packages -RUN pip3 install --upgrade pip setuptools wheel && \ - pip3 install -r /tmp/requirements.txt && \ - (pip3 install --no-cache-dir --find-links https://pypi.org/simple/ -r /tmp/operator-requirements.txt || \ - pip3 install --no-cache-dir wordcloud==1.9.2) +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh pip --upgrade setuptools wheel \ + && bash /snapshot/install.sh pip-requirements /tmp/requirements.txt \ + && bash /snapshot/install.sh pip-requirements /tmp/operator-requirements.txt # Copy the built texera binary from the build phase COPY --from=build /texera/amber/target/amber-* /texera/amber/ @@ -121,6 +116,9 @@ COPY --from=build /texera/amber/NOTICE-binary /texera/NOTICE COPY --from=build /texera/licenses /texera/licenses COPY --from=build /texera/DISCLAIMER /texera/ +# Dates the texera account in /etc/shadow; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN groupadd --system --gid 1001 texera \ && useradd --system --uid 1001 --gid texera --home-dir /texera --no-create-home texera \ && chown -R texera:texera /texera diff --git a/bin/dockerfiles/config-service.dockerfile b/bin/dockerfiles/config-service.dockerfile index df986ee2ff6..0e52ef30578 100644 --- a/bin/dockerfiles/config-service.dockerfile +++ b/bin/dockerfiles/config-service.dockerfile @@ -24,7 +24,7 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11 AS build +FROM docker.io/sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11@sha256:c20fad6183112843d6c87fd54bb9b507a424ab7893c6a570b1624acf2e0324c4 AS build # Set working directory WORKDIR /texera @@ -37,27 +37,25 @@ COPY build.sbt build.sbt COPY .jvmopts .jvmopts # Update system and install dependencies -RUN apt-get update && apt-get install -y \ - netcat \ - unzip \ - libpq-dev \ - && apt-get clean +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt netcat unzip libpq-dev -# Add .git for runtime calls to jgit from OPversion -COPY .git .git COPY LICENSE NOTICE DISCLAIMER ./ COPY licenses/ licenses/ +# Pins jar and dist-zip entry timestamps; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN sbt clean ConfigService/dist # Unzip the texera binary RUN unzip config-service/target/universal/config-service-*.zip -d target/ -FROM eclipse-temurin:17-jre-jammy AS runtime +FROM docker.io/library/eclipse-temurin:17-jre-jammy@sha256:97137382c6f0c30427d9b7c44ad8b2d55ac823b0768c171d81a643ed219023c5 AS runtime WORKDIR /texera -COPY --from=build /texera/.git /texera/.git # Copy the built texera binary from the build phase COPY --from=build /texera/target/config-service-* /texera/ # Copy resources directories from build phase @@ -72,6 +70,9 @@ COPY --from=build /texera/config-service/NOTICE-binary /texera/NOTICE COPY --from=build /texera/licenses /texera/licenses COPY --from=build /texera/DISCLAIMER /texera/ +# Dates the texera account in /etc/shadow; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN groupadd --system --gid 1001 texera \ && useradd --system --uid 1001 --gid texera --home-dir /texera --no-create-home texera \ && chown -R texera:texera /texera diff --git a/bin/dockerfiles/file-service.dockerfile b/bin/dockerfiles/file-service.dockerfile index 1bf7032a747..b56253edfee 100644 --- a/bin/dockerfiles/file-service.dockerfile +++ b/bin/dockerfiles/file-service.dockerfile @@ -24,7 +24,7 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11 AS build +FROM docker.io/sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11@sha256:c20fad6183112843d6c87fd54bb9b507a424ab7893c6a570b1624acf2e0324c4 AS build # Set working directory WORKDIR /texera @@ -37,23 +37,22 @@ COPY build.sbt build.sbt COPY .jvmopts .jvmopts # Update system and install dependencies -RUN apt-get update && apt-get install -y \ - netcat \ - unzip \ - libpq-dev \ - && apt-get clean +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt netcat unzip libpq-dev -# Add .git for runtime calls to jgit from OPversion -COPY .git .git COPY LICENSE NOTICE DISCLAIMER ./ COPY licenses/ licenses/ +# Pins jar and dist-zip entry timestamps; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN sbt clean FileService/dist # Unzip the texera binary RUN unzip file-service/target/universal/file-service-*.zip -d target/ -FROM eclipse-temurin:17-jre-jammy AS runtime +FROM docker.io/library/eclipse-temurin:17-jre-jammy@sha256:97137382c6f0c30427d9b7c44ad8b2d55ac823b0768c171d81a643ed219023c5 AS runtime WORKDIR /texera @@ -71,6 +70,9 @@ COPY --from=build /texera/file-service/NOTICE-binary /texera/NOTICE COPY --from=build /texera/licenses /texera/licenses COPY --from=build /texera/DISCLAIMER /texera/ +# Dates the texera account in /etc/shadow; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN groupadd --system --gid 1001 texera \ && useradd --system --uid 1001 --gid texera --home-dir /texera --no-create-home texera \ && chown -R texera:texera /texera diff --git a/bin/dockerfiles/jupyter.dockerfile b/bin/dockerfiles/jupyter.dockerfile index 2d234c2a3e3..8f22c1c8c62 100644 --- a/bin/dockerfiles/jupyter.dockerfile +++ b/bin/dockerfiles/jupyter.dockerfile @@ -24,7 +24,7 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -FROM jupyter/base-notebook:notebook-6.5.4 +FROM docker.io/jupyter/base-notebook:notebook-6.5.4@sha256:c6afcd9f9bfca7441b60f4ffb21cebac73bf5f3d722735eff5ddc30aeb2aa34e # The customizations live with notebook-migration-service, which owns the Jupyter # integration. Paths are repo-root relative: the build context is the repo root. diff --git a/bin/dockerfiles/mounter.dockerfile b/bin/dockerfiles/mounter.dockerfile index d385e2aa57a..34f7c81ee31 100644 --- a/bin/dockerfiles/mounter.dockerfile +++ b/bin/dockerfiles/mounter.dockerfile @@ -19,16 +19,15 @@ # FUSE mount that computing-unit pods used to do themselves, so that CU pods (which run # untrusted user code) can be unprivileged. It needs python3 (the mounter), fuse3 + geesefs # (to mount), and util-linux (umount) — all part of a minimal Debian base. -FROM debian:bookworm-slim +FROM docker.io/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 ARG GEESEFS_VERSION=v0.43.8 -RUN apt-get update \ - && apt-get install -y --no-install-recommends python3 fuse3 mount ca-certificates curl \ +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt --no-install-recommends python3 fuse3 mount ca-certificates curl \ && curl -fsSL -o /usr/local/bin/geesefs \ "https://github.com/yandex-cloud/geesefs/releases/download/${GEESEFS_VERSION}/geesefs-linux-$(dpkg --print-architecture)" \ && chmod 755 /usr/local/bin/geesefs \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ # allow FUSE mounts to be accessible by other users (the unprivileged CU pod's UID) && echo "user_allow_other" >> /etc/fuse.conf diff --git a/bin/dockerfiles/notebook-migration-service.dockerfile b/bin/dockerfiles/notebook-migration-service.dockerfile index 1f45d24ff29..bc775a9c1cc 100644 --- a/bin/dockerfiles/notebook-migration-service.dockerfile +++ b/bin/dockerfiles/notebook-migration-service.dockerfile @@ -24,7 +24,7 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11 AS build +FROM docker.io/sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11@sha256:c20fad6183112843d6c87fd54bb9b507a424ab7893c6a570b1624acf2e0324c4 AS build # Set working directory WORKDIR /texera @@ -37,21 +37,22 @@ COPY build.sbt build.sbt COPY .jvmopts .jvmopts # Update system and install dependencies -RUN apt-get update && apt-get install -y \ - netcat \ - unzip \ - libpq-dev \ - && apt-get clean +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt netcat unzip libpq-dev COPY LICENSE NOTICE DISCLAIMER ./ COPY licenses/ licenses/ +# Pins jar and dist-zip entry timestamps; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN sbt clean NotebookMigrationService/dist # Unzip the texera binary RUN unzip notebook-migration-service/target/universal/notebook-migration-service-*.zip -d target/ -FROM eclipse-temurin:17-jre-jammy AS runtime +FROM docker.io/library/eclipse-temurin:17-jre-jammy@sha256:97137382c6f0c30427d9b7c44ad8b2d55ac823b0768c171d81a643ed219023c5 AS runtime WORKDIR /texera @@ -69,6 +70,9 @@ COPY --from=build /texera/notebook-migration-service/NOTICE-binary /texera/NOTIC COPY --from=build /texera/licenses /texera/licenses COPY --from=build /texera/DISCLAIMER /texera/ +# Dates the texera account in /etc/shadow; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN groupadd --system --gid 1001 texera \ && useradd --system --uid 1001 --gid texera --home-dir /texera --no-create-home texera \ && chown -R texera:texera /texera diff --git a/bin/dockerfiles/snapshot/install.sh b/bin/dockerfiles/snapshot/install.sh new file mode 100755 index 00000000000..b96e927e3c7 --- /dev/null +++ b/bin/dockerfiles/snapshot/install.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Installs the OS and Python packages of the image builds as they stood at +# 00:00 UTC on PACKAGE_SNAPSHOT (YYYY-MM-DD), so rebuilding a commit installs +# the same versions no matter how much later it runs. apt reads the +# snapshot.ubuntu.com / snapshot.debian.org copy of the archive for that +# moment; pip only considers files uploaded to PyPI before it. Without +# PACKAGE_SNAPSHOT it installs from the live archives and PyPI. +# +# Bind-mount the directory rather than COPY it, so it never lands in a layer: +# ARG PACKAGE_SNAPSHOT +# RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ +# bash /snapshot/install.sh apt [apt-get install option]... ... +# ... bash /snapshot/install.sh pip ... +# ... bash /snapshot/install.sh pip-requirements ... + +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Prints the snapshot as " ", or +# nothing when PACKAGE_SNAPSHOT is unset. +snapshot_times() { + local snapshot="${PACKAGE_SNAPSHOT:-}" + [[ -z "$snapshot" ]] && return 0 + local y m d leap era yoe doy epoch + # Plain arithmetic rather than date(1): GNU and BSD date disagree on parsing. + if [[ "$snapshot" =~ ^([0-9]{4})-([0-9]{2})-([0-9]{2})$ ]]; then + y=$((10#${BASH_REMATCH[1]})) m=$((10#${BASH_REMATCH[2]})) d=$((10#${BASH_REMATCH[3]})) + leap=$(( (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 )) + fi + if [[ -z "${y:-}" ]] || (( m < 1 || m > 12 || d < 1 \ + || d > (m == 2 ? 28 + leap : 30 + ((m + m / 8) % 2)) )); then + echo "PACKAGE_SNAPSHOT must be a UTC date (YYYY-MM-DD), got '$snapshot'" >&2 + return 1 + fi + # Days since 1970-01-01 (Hinnant's days_from_civil). + (( y -= m <= 2, era = y / 400, yoe = y - era * 400, + doy = (153 * (m > 2 ? m - 3 : m + 9) + 2) / 5 + d - 1, + epoch = (era * 146097 + yoe * 365 + yoe / 4 - yoe / 100 + doy - 719468) * 86400 )) + echo "${snapshot//-/}T000000Z ${snapshot}T00:00:00Z ${epoch}" +} + +# Prints a one-line-style sources.list pointing the distribution in +# /etc/os-release (or $1) at the archive snapshot taken at $2. +snapshot_sources() { + local os_release="$1" stamp="$2" id codename + id=$(. "$os_release" && echo "$ID") + codename=$(. "$os_release" && echo "$VERSION_CODENAME") + case "$id" in + ubuntu) + # One snapshot serves every architecture, ports (arm64) included. + local base="https://snapshot.ubuntu.com/ubuntu/${stamp}" + for suite in "$codename" "${codename}-updates" "${codename}-security"; do + echo "deb ${base} ${suite} main restricted universe multiverse" + done + ;; + debian) + # Plain http: slim images have no CA bundle before this install, and apt + # checks the signed InRelease either way. Snapshots outlive Valid-Until. + local opts="[check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg]" + echo "deb ${opts} http://snapshot.debian.org/archive/debian/${stamp} ${codename} main" + echo "deb ${opts} http://snapshot.debian.org/archive/debian/${stamp} ${codename}-updates main" + echo "deb ${opts} http://snapshot.debian.org/archive/debian-security/${stamp} ${codename}-security main" + ;; + *) + echo "No archive snapshot known for distribution '$id'" >&2 + return 1 + ;; + esac +} + +# apt-get writes logs, solver dumps and caches that record when it ran. +apt_cleanup() { + apt-get clean + rm -rf /var/lib/apt/lists/* /var/log/apt/* /var/log/dpkg.log /var/log/alternatives.log \ + /var/cache/ldconfig/aux-cache /var/cache/debconf/*-old /var/lib/dpkg/*-old +} + +apt_install() { + local times opts=() + times=$(snapshot_times) + if [[ -n "$times" ]]; then + local work + work=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf '$work'" RETURN + snapshot_sources /etc/os-release "${times%% *}" > "$work/sources.list" + mkdir "$work/sources.list.d" + # An alternate source list leaves the image's own apt sources untouched, + # so later `apt-get install`s in derived images still use the live archive. + opts=(-o "Dir::Etc::SourceList=$work/sources.list" -o "Dir::Etc::SourceParts=$work/sources.list.d") + fi + apt-get "${opts[@]}" update + DEBIAN_FRONTEND=noninteractive apt-get "${opts[@]}" install -y "$@" + apt_cleanup +} + +# Splits requirements files into the pins that can only come from an extra +# index (a local version such as torch==2.13.0+cpu, which PyPI rejects) and +# everything else. Writes /local.txt, /pypi.txt and /indexes, +# the extra index URLs. The local pins stay in pypi.txt too, where the +# already-installed wheel satisfies them and pip resolves their dependencies. +split_requirements() { + local dir="$1" + shift + : > "$dir/local.txt" + : > "$dir/pypi.txt" + : > "$dir/indexes" + local file line spec + for file in "$@"; do + while IFS= read -r line || [[ -n "$line" ]]; do + spec="${line%%#*}" + if [[ "$spec" =~ ^[[:space:]]*--(extra-)?index-url[[:space:]=]+([^[:space:]]+) ]]; then + echo "${BASH_REMATCH[2]}" >> "$dir/indexes" + continue + fi + [[ "$spec" =~ ^[[:space:]]*$ ]] && continue + if [[ "$spec" =~ ^[[:space:]]*- ]]; then + echo "Unsupported option in $file: $spec" >&2 + return 1 + fi + echo "$spec" >> "$dir/pypi.txt" + if [[ "${spec%%;*}" =~ ==[^[:space:]]*\+ ]]; then + echo "$spec" >> "$dir/local.txt" + fi + done < "$file" + done +} + +# Runs `pip install`. With a snapshot, byte-compilation is made deterministic: +# SOURCE_DATE_EPOCH switches .pyc files to hash-based invalidation (no source +# mtime in the header), and a fixed hash seed fixes set ordering in them. +pip_install() { + local times + times=$(snapshot_times) + if [[ -z "$times" ]]; then + python3 -m pip install --no-cache-dir "$@" + else + PYTHONHASHSEED=0 SOURCE_DATE_EPOCH="${times##* }" python3 -m pip install --no-cache-dir "$@" + fi +} + +pip_bootstrap() { + # The distribution's pip predates --uploaded-prior-to. + pip_install --no-deps -r "$here/pip-bootstrap-requirements.txt" +} + +# pip install, considering only files uploaded before the snapshot. +pip_snapshot() { + local times + times=$(snapshot_times) + if [[ -z "$times" ]]; then + pip_install "$@" + else + local rest="${times#* }" + pip_install --uploaded-prior-to "${rest%% *}" "$@" + fi +} + +pip_requirements() { + local times + times=$(snapshot_times) + if [[ -z "$times" ]]; then + local args=() + for f in "$@"; do args+=(-r "$f"); done + pip_install "${args[@]}" + return + fi + local work + work=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf '$work'" RETURN + split_requirements "$work" "$@" + if [[ -s "$work/local.txt" ]]; then + # Extra indexes such as download.pytorch.org publish no upload times, so + # the cutoff cannot apply there; an exact pin of a local version is the + # one file it can resolve to, which is what makes this reproducible. + local index_args=() url + while IFS= read -r url; do index_args+=(--extra-index-url "$url"); done < "$work/indexes" + pip_install --no-deps "${index_args[@]}" -r "$work/local.txt" + fi + pip_snapshot -r "$work/pypi.txt" +} + +main() { + local cmd="${1:-}" + shift || true + case "$cmd" in + apt) apt_install "$@" ;; + pip) pip_bootstrap && pip_snapshot "$@" ;; + pip-requirements) pip_bootstrap && pip_requirements "$@" ;; + *) + echo "Usage: $0 {apt|pip|pip-requirements} ..." >&2 + return 2 + ;; + esac +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/bin/dockerfiles/snapshot/pip-bootstrap-requirements.txt b/bin/dockerfiles/snapshot/pip-bootstrap-requirements.txt new file mode 100644 index 00000000000..e731f8c27ae --- /dev/null +++ b/bin/dockerfiles/snapshot/pip-bootstrap-requirements.txt @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The pip that install.sh installs before anything else: Ubuntu 22.04's own +# pip (22.0) predates --uploaded-prior-to. +pip==26.2.1 diff --git a/bin/dockerfiles/snapshot/test_install.sh b/bin/dockerfiles/snapshot/test_install.sh new file mode 100755 index 00000000000..11e1d7c9094 --- /dev/null +++ b/bin/dockerfiles/snapshot/test_install.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Tests for install.sh's pure logic: snapshot date handling, the archive +# sources it points apt at, and how it splits requirements between PyPI and an +# extra index. The installs themselves are exercised by the image builds. + +set -uo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_dir/../../.." && pwd)" +# shellcheck source=install.sh +source "$script_dir/install.sh" +set +e +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +rc=0 + +pass() { echo "ok: $1"; } +failed() { echo "FAIL: $1"; rc=1; } +expect_eq() { + if [[ "$2" == "$3" ]]; then pass "$1"; else failed "$1: expected [$3], got [$2]"; fi +} + +# --- snapshot_times --- +expect_eq "unset PACKAGE_SNAPSHOT -> live archives" "$(unset PACKAGE_SNAPSHOT; snapshot_times)" "" +expect_eq "empty PACKAGE_SNAPSHOT -> live archives" "$(PACKAGE_SNAPSHOT= snapshot_times)" "" +expect_eq "date -> apt stamp, pip cutoff and epoch" "$(PACKAGE_SNAPSHOT=2026-09-30 snapshot_times)" \ + "20260930T000000Z 2026-09-30T00:00:00Z 1790726400" +expect_eq "read in UTC whatever the build host's zone" \ + "$(TZ=Pacific/Kiritimati PACKAGE_SNAPSHOT=1970-01-01 snapshot_times)" "19700101T000000Z 1970-01-01T00:00:00Z 0" +expect_eq "leap day accepted" "$(PACKAGE_SNAPSHOT=2028-02-29 snapshot_times | cut -d' ' -f1)" "20280229T000000Z" +for bad in 2026-9-30 2026-02-30 2027-02-29 20260930 2026-09-30T00:00 " 2026-09-30" 2026-13-01 x; do + out=$(PACKAGE_SNAPSHOT="$bad" snapshot_times 2>&1) + if [[ $? -ne 0 && "$out" == *"PACKAGE_SNAPSHOT must be a UTC date"* ]]; then + pass "rejects PACKAGE_SNAPSHOT [$bad]" + else + failed "should reject PACKAGE_SNAPSHOT [$bad], got [$out]" + fi +done + +# --- snapshot_sources --- +printf 'ID=ubuntu\nVERSION_CODENAME=jammy\n' > "$work/ubuntu" +expect_eq "ubuntu: all three suites from one snapshot (any architecture)" \ + "$(snapshot_sources "$work/ubuntu" 20260930T000000Z)" \ + "deb https://snapshot.ubuntu.com/ubuntu/20260930T000000Z jammy main restricted universe multiverse +deb https://snapshot.ubuntu.com/ubuntu/20260930T000000Z jammy-updates main restricted universe multiverse +deb https://snapshot.ubuntu.com/ubuntu/20260930T000000Z jammy-security main restricted universe multiverse" +printf 'ID=debian\nVERSION_CODENAME=bookworm\n' > "$work/debian" +debian_sources="$(snapshot_sources "$work/debian" 20260930T000000Z)" +expect_eq "debian: security suite from the debian-security snapshot" \ + "$(grep -c 'archive/debian-security/20260930T000000Z bookworm-security main' <<< "$debian_sources")" "1" +expect_eq "debian: every line skips Valid-Until and names the keyring" \ + "$(grep -c '^deb \[check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg\] http://' <<< "$debian_sources")" "3" +printf 'ID=alpine\nVERSION_CODENAME=\n' > "$work/alpine" +if out=$(snapshot_sources "$work/alpine" 20260930T000000Z 2>&1); then + failed "unknown distribution should fail" +else + expect_eq "unknown distribution -> error" "$out" "No archive snapshot known for distribution 'alpine'" +fi + +# --- split_requirements --- +cat > "$work/req.txt" <<'EOF' +# comment +numpy==2.1.0 # inline comment +--extra-index-url https://download.pytorch.org/whl/cpu + +torch==2.13.0+cpu ; platform_system == "Linux" and platform_machine == "x86_64" +torch==2.13.0 ; platform_system != "Linux" or platform_machine != "x86_64" +odd==1.0 ; extra == "a+b" +EOF +printf -- '--index-url=https://example.org/simple\nwheel==0.45.1' > "$work/req2.txt" +mkdir "$work/split" +if split_requirements "$work/split" "$work/req.txt" "$work/req2.txt"; then + expect_eq "local versions (and only those) go to the extra index" "$(cat "$work/split/local.txt")" \ + 'torch==2.13.0+cpu ; platform_system == "Linux" and platform_machine == "x86_64"' + expect_eq "every requirement stays in the PyPI pass" "$(wc -l < "$work/split/pypi.txt" | tr -d ' ')" "5" + expect_eq "a '+' in a marker is not a local version" "$(grep -c '^odd' "$work/split/local.txt")" "0" + expect_eq "index options are collected, not passed to PyPI" "$(cat "$work/split/indexes")" \ + "https://download.pytorch.org/whl/cpu +https://example.org/simple" + expect_eq "a last line without a newline is kept" "$(tail -n1 "$work/split/pypi.txt")" "wheel==0.45.1" +else + failed "split_requirements failed on valid input" +fi +printf 'numpy==2.1.0\n-r other.txt\n' > "$work/nested.txt" +if out=$(split_requirements "$work/split" "$work/nested.txt" 2>&1); then + failed "nested -r should be rejected" +else + pass "rejects options it cannot carry over (nested -r): $out" +fi +mkdir "$work/real" +split_requirements "$work/real" "$repo_root/amber/requirements.txt" "$repo_root/amber/operator-requirements.txt" +expect_eq "the repo's requirements: only the +cpu torch needs the PyTorch index" \ + "$(sed 's/ *;.*//' "$work/real/local.txt")" "torch==2.13.0+cpu" + +# --- pip invocations (python3 stubbed to record what it was asked to run) --- +python3() { echo "hashseed=${PYTHONHASHSEED-unset} sde=${SOURCE_DATE_EPOCH-unset} $*"; } +expect_eq "without a snapshot pip runs as before" "$(unset PACKAGE_SNAPSHOT; pip_snapshot wheel)" \ + "hashseed=unset sde=unset -m pip install --no-cache-dir wheel" +expect_eq "with a snapshot pip gets the cutoff and deterministic byte-compilation" \ + "$(PACKAGE_SNAPSHOT=2026-09-30 pip_snapshot wheel)" \ + "hashseed=0 sde=1790726400 -m pip install --no-cache-dir --uploaded-prior-to 2026-09-30T00:00:00Z wheel" +expect_eq "the pip bootstrap is byte-compiled deterministically too" \ + "$(PACKAGE_SNAPSHOT=2026-09-30 pip_bootstrap)" \ + "hashseed=0 sde=1790726400 -m pip install --no-cache-dir --no-deps -r $script_dir/pip-bootstrap-requirements.txt" +pip_calls="$(PACKAGE_SNAPSHOT=2026-09-30 pip_requirements "$work/req.txt")" +expect_eq "local pins: from the extra index, exact, no dependencies, no cutoff" "$(sed -n 1p <<< "$pip_calls")" \ + "hashseed=0 sde=1790726400 -m pip install --no-cache-dir --no-deps --extra-index-url https://download.pytorch.org/whl/cpu -r $(sed -n 1p <<< "$pip_calls" | sed 's/.* -r //')" +expect_eq "then everything else from PyPI under the cutoff" "$(sed -n 2p <<< "$pip_calls" | sed 's/ -r .*//')" \ + "hashseed=0 sde=1790726400 -m pip install --no-cache-dir --uploaded-prior-to 2026-09-30T00:00:00Z" +expect_eq "without local pins only the PyPI pass runs" \ + "$(printf 'numpy==2.1.0\n' > "$work/plain.txt"; PACKAGE_SNAPSHOT=2026-09-30 pip_requirements "$work/plain.txt" | wc -l | tr -d ' ')" "1" +expect_eq "without a snapshot the files go to pip untouched" \ + "$(unset PACKAGE_SNAPSHOT; pip_requirements "$work/req.txt" "$work/plain.txt")" \ + "hashseed=unset sde=unset -m pip install --no-cache-dir -r $work/req.txt -r $work/plain.txt" +unset -f python3 + +# --- main --- +out=$(main bogus 2>&1) +expect_eq "unknown command -> usage, exit 2" "$? ${out%% *}" "2 Usage:" + +if [[ "$rc" -ne 0 ]]; then + echo "snapshot install.sh tests FAILED" + exit 1 +fi +echo "snapshot install.sh tests passed" diff --git a/bin/dockerfiles/texera-web-application.dockerfile b/bin/dockerfiles/texera-web-application.dockerfile index 276253ec4fa..4b654ee46a3 100644 --- a/bin/dockerfiles/texera-web-application.dockerfile +++ b/bin/dockerfiles/texera-web-application.dockerfile @@ -24,21 +24,24 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -FROM node:24-bookworm AS build-frontend +FROM docker.io/library/node:24.19.0-bookworm@sha256:4196d66a565c6f195728d9952f161f4adfe2ad753052a08b7ec7f1c5a6bda42b AS build-frontend -RUN apt-get update && apt-get install -y --no-install-recommends \ - python3 build-essential git ca-certificates +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt --no-install-recommends python3 build-essential git ca-certificates WORKDIR /frontend COPY frontend /frontend -RUN rm -f /frontend/.yarnrc.yml -RUN corepack enable && corepack prepare yarn@4.5.1 --activate && yarn set version --yarn-path 4.5.1 -RUN echo "nodeLinker: node-modules" >> /frontend/.yarnrc.yml +# The repo's .yarnrc.yml pins Yarn (yarnPath) and the node-modules linker; +# --immutable fails rather than re-resolving anything yarn.lock does not pin. +RUN corepack enable && corepack prepare yarn@4.14.1 --activate -WORKDIR /frontend -RUN yarn install && yarn run build +# Fixes the build number build-version.js bakes into the bundle; pass the +# commit time (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH +RUN yarn install --immutable && yarn run build -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11 AS build +FROM docker.io/sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11@sha256:c20fad6183112843d6c87fd54bb9b507a424ab7893c6a570b1624acf2e0324c4 AS build # Set working directory WORKDIR /texera @@ -52,14 +55,9 @@ COPY .jvmopts .jvmopts # python3-minimal is needed by bin/licensing/concat_license_binary.py; # python3-pip installs the betterproto plugin; unzip + curl fetch protoc. -RUN apt-get update && apt-get install -y \ - netcat \ - unzip \ - curl \ - libpq-dev \ - python3-minimal \ - python3-pip \ - && apt-get clean +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt netcat unzip curl libpq-dev python3-minimal python3-pip # Install protoc (version pinned in bin/protoc-version.txt) and the # betterproto plugin (version pinned via amber/requirements.txt as a @@ -69,7 +67,8 @@ RUN apt-get update && apt-get install -y \ # is packaged. COPY bin/protoc-version.txt bin/protoc-version.txt COPY bin/python-proto-gen.sh bin/python-proto-gen.sh -RUN PROTOC_VERSION=$(cat bin/protoc-version.txt) \ +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + PROTOC_VERSION=$(cat bin/protoc-version.txt) \ && case "$(uname -m)" in \ x86_64 | amd64) PROTOC_ARCH=x86_64 ;; \ aarch64 | arm64) PROTOC_ARCH=aarch_64 ;; \ @@ -79,11 +78,9 @@ RUN PROTOC_VERSION=$(cat bin/protoc-version.txt) \ && unzip -o /tmp/protoc.zip -d /usr/local \ && chmod +x /usr/local/bin/protoc \ && rm /tmp/protoc.zip \ - && pip3 install --no-cache-dir -c amber/requirements.txt 'betterproto[compiler]' \ + && bash /snapshot/install.sh pip -c amber/requirements.txt 'betterproto[compiler]' \ && bash bin/python-proto-gen.sh -# Add .git for runtime calls to jgit from OPversion -COPY .git .git COPY LICENSE NOTICE DISCLAIMER ./ COPY licenses/ licenses/ COPY bin/licensing/ bin/licensing/ @@ -92,6 +89,9 @@ COPY bin/licensing/ bin/licensing/ # LICENSE merge below can union it with amber/LICENSE-binary-java. COPY --from=build-frontend /frontend/LICENSE-binary amber/LICENSE-binary-frontend +# Pins jar and dist-zip entry timestamps; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN sbt clean WorkflowExecutionService/dist # Unzip the texera binary @@ -106,13 +106,12 @@ RUN python3 bin/licensing/concat_license_binary.py amber/LICENSE-binary-combined amber/LICENSE-binary-java \ amber/LICENSE-binary-frontend -FROM eclipse-temurin:17-jre-jammy AS runtime +FROM docker.io/library/eclipse-temurin:17-jre-jammy@sha256:97137382c6f0c30427d9b7c44ad8b2d55ac823b0768c171d81a643ed219023c5 AS runtime WORKDIR /texera/amber # Copy built frontend files from the build-frontend stage to match FileAssetsBundle path (../../frontend/dist from /texera/amber) COPY --from=build-frontend /frontend/dist /frontend/dist # Copy the built texera binary from the build phase -COPY --from=build /texera/.git /texera/amber/.git COPY --from=build /texera/amber/target/amber-* /texera/amber/ # Copy resources directories from build phase COPY --from=build /texera/amber/src/main/resources /texera/amber/src/main/resources @@ -126,6 +125,9 @@ COPY --from=build /texera/amber/NOTICE-binary /texera/NOTICE COPY --from=build /texera/licenses /texera/licenses COPY --from=build /texera/DISCLAIMER /texera/ +# Dates the texera account in /etc/shadow; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN groupadd --system --gid 1001 texera \ && useradd --system --uid 1001 --gid texera --home-dir /texera --no-create-home texera \ && chown -R texera:texera /texera /frontend diff --git a/bin/dockerfiles/workflow-compiling-service.dockerfile b/bin/dockerfiles/workflow-compiling-service.dockerfile index a94940ed0d7..113d69a3c69 100644 --- a/bin/dockerfiles/workflow-compiling-service.dockerfile +++ b/bin/dockerfiles/workflow-compiling-service.dockerfile @@ -24,7 +24,7 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11 AS build +FROM docker.io/sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11@sha256:c20fad6183112843d6c87fd54bb9b507a424ab7893c6a570b1624acf2e0324c4 AS build # Set working directory WORKDIR /texera @@ -37,28 +37,26 @@ COPY build.sbt build.sbt COPY .jvmopts .jvmopts # Update system and install dependencies -RUN apt-get update && apt-get install -y \ - netcat \ - unzip \ - libpq-dev \ - && apt-get clean +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt netcat unzip libpq-dev -# Add .git for runtime calls to jgit from OPversion -COPY .git .git COPY LICENSE NOTICE DISCLAIMER ./ COPY licenses/ licenses/ +# Pins jar and dist-zip entry timestamps; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN sbt clean WorkflowCompilingService/dist # Unzip the texera binary RUN unzip workflow-compiling-service/target/universal/workflow-compiling-service-*.zip -d target/ -FROM eclipse-temurin:17-jre-jammy AS runtime +FROM docker.io/library/eclipse-temurin:17-jre-jammy@sha256:97137382c6f0c30427d9b7c44ad8b2d55ac823b0768c171d81a643ed219023c5 AS runtime WORKDIR /texera # Copy the built texera binary from the build phase -COPY --from=build /texera/.git /texera/.git COPY --from=build /texera/target/workflow-compiling-service-* /texera/ # Copy resources directories from build phase COPY --from=build /texera/common/config/src/main/resources /texera/common/config/src/main/resources @@ -72,6 +70,9 @@ COPY --from=build /texera/workflow-compiling-service/NOTICE-binary /texera/NOTIC COPY --from=build /texera/licenses /texera/licenses COPY --from=build /texera/DISCLAIMER /texera/ +# Dates the texera account in /etc/shadow; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN groupadd --system --gid 1001 texera \ && useradd --system --uid 1001 --gid texera --home-dir /texera --no-create-home texera \ && chown -R texera:texera /texera diff --git a/bin/dockerfiles/workflow-computing-unit-managing-service.dockerfile b/bin/dockerfiles/workflow-computing-unit-managing-service.dockerfile index a47e6ce06e5..d7430a9246f 100644 --- a/bin/dockerfiles/workflow-computing-unit-managing-service.dockerfile +++ b/bin/dockerfiles/workflow-computing-unit-managing-service.dockerfile @@ -24,7 +24,7 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11 AS build +FROM docker.io/sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.3_2.13.11@sha256:c20fad6183112843d6c87fd54bb9b507a424ab7893c6a570b1624acf2e0324c4 AS build # Set working directory WORKDIR /texera @@ -37,27 +37,25 @@ COPY build.sbt build.sbt COPY .jvmopts .jvmopts # Update system and install dependencies -RUN apt-get update && apt-get install -y \ - netcat \ - unzip \ - libpq-dev \ - && apt-get clean +ARG PACKAGE_SNAPSHOT +RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ + bash /snapshot/install.sh apt netcat unzip libpq-dev -# Add .git for runtime calls to jgit from OPversion -COPY .git .git COPY LICENSE NOTICE DISCLAIMER ./ COPY licenses/ licenses/ +# Pins jar and dist-zip entry timestamps; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN sbt clean ComputingUnitManagingService/dist # Unzip the texera binary RUN unzip computing-unit-managing-service/target/universal/computing-unit-managing-service-*.zip -d target/ -FROM eclipse-temurin:17-jre-jammy AS runtime +FROM docker.io/library/eclipse-temurin:17-jre-jammy@sha256:97137382c6f0c30427d9b7c44ad8b2d55ac823b0768c171d81a643ed219023c5 AS runtime WORKDIR /texera -COPY --from=build /texera/.git /texera/.git # Copy the built texera binary from the build phase COPY --from=build /texera/target/computing-unit-managing-service-* /texera/ # Copy resources directories from build phase @@ -72,6 +70,9 @@ COPY --from=build /texera/computing-unit-managing-service/NOTICE-binary /texera/ COPY --from=build /texera/licenses /texera/licenses COPY --from=build /texera/DISCLAIMER /texera/ +# Dates the texera account in /etc/shadow; pass the commit time +# (git log -1 --format=%ct) for a reproducible build. +ARG SOURCE_DATE_EPOCH RUN groupadd --system --gid 1001 texera \ && useradd --system --uid 1001 --gid texera --home-dir /texera --no-create-home texera \ && chown -R texera:texera /texera diff --git a/bin/k8s/Chart.yaml b/bin/k8s/Chart.yaml index aab1fa538d8..2cd520af748 100644 --- a/bin/k8s/Chart.yaml +++ b/bin/k8s/Chart.yaml @@ -47,7 +47,7 @@ dependencies: repository: https://charts.bitnami.com/bitnami - name: rustfs - version: 1.0.0-rc.6 + version: 1.0.0 repository: https://charts.rustfs.com condition: rustfs.enabled diff --git a/bin/k8s/templates/base/lakefs/lakefs-setup-job.yaml b/bin/k8s/templates/base/lakefs/lakefs-setup-job.yaml index b7fdd51364b..85e9033cb4c 100644 --- a/bin/k8s/templates/base/lakefs/lakefs-setup-job.yaml +++ b/bin/k8s/templates/base/lakefs/lakefs-setup-job.yaml @@ -32,7 +32,7 @@ spec: restartPolicy: Never containers: - name: setup-lakefs - image: curlimages/curl:latest + image: curlimages/curl:8.22.0 env: - name: STORAGE_LAKEFS_ENDPOINT value: http://{{ .Release.Name }}-lakefs:8000/api/v1 diff --git a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml index 163a0e8e840..18d6242c6a7 100644 --- a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml +++ b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml @@ -38,7 +38,7 @@ spec: serviceAccountName: {{ .Values.workflowComputingUnitManager.serviceAccountName }} initContainers: - name: wait-lakekeeper - image: curlimages/curl:latest + image: curlimages/curl:8.22.0 command: - /bin/sh - -c diff --git a/bin/k8s/values-development.yaml b/bin/k8s/values-development.yaml index accf9f39c99..5e88dad4737 100644 --- a/bin/k8s/values-development.yaml +++ b/bin/k8s/values-development.yaml @@ -81,7 +81,7 @@ rustfs: image: rustfs: repository: rustfs/rustfs - tag: 1.0.0-rc.6 + tag: 1.0.0 resources: requests: memory: "256Mi" diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index 4c6f43a9755..f8c3db2f4e1 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -102,7 +102,7 @@ rustfs: image: rustfs: repository: rustfs/rustfs - tag: 1.0.0-rc.6 + tag: 1.0.0 # Texera fronts the object store with Envoy Gateway (the `gateway` block # below), not with an Ingress -- the upstream chart turns one on by default. ingress: diff --git a/bin/python-proto-gen.sh b/bin/python-proto-gen.sh index db51bb76265..2a2c9803787 100755 --- a/bin/python-proto-gen.sh +++ b/bin/python-proto-gen.sh @@ -31,10 +31,13 @@ PROTOBUF_CORE_DIR="$CORE_DIR/src/main/protobuf" PROTOC_INCLUDE_DIR="$(dirname "$(dirname "$(command -v protoc)")")/include" # proto-gen +# betterproto merges every file of a proto package into one module, in the +# order protoc receives them, and find's order depends on the filesystem; sort +# so the generated modules are identical on every machine. mkdir -p "$PYAMBER_DIR/proto" protoc --python_betterproto_out="$PYAMBER_DIR/proto" \ -I="$PROTOC_INCLUDE_DIR" \ -I="$PROTOBUF_AMBER_DIR" \ -I="$PROTOBUF_CORE_DIR" \ - $(find "$PROTOBUF_AMBER_DIR" -iname "*.proto") \ - $(find "$PROTOBUF_CORE_DIR" -iname "*.proto") + $(find "$PROTOBUF_AMBER_DIR" -iname "*.proto" | LC_ALL=C sort) \ + $(find "$PROTOBUF_CORE_DIR" -iname "*.proto" | LC_ALL=C sort) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index d0752c2ca40..fc9d26cd48e 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -20,7 +20,7 @@ services: # Part1: Specification of the storage services used by Texera # RustFS is an S3-compatible object storage used to store datasets and files. rustfs: - image: rustfs/rustfs:1.0.0-rc.6 + image: rustfs/rustfs:1.0.0 container_name: texera-rustfs ports: - "${RUSTFS_PORT:-9000}:9000" @@ -531,7 +531,7 @@ services: # Part 3: reverse proxy service for Texera's micro services nginx: - image: nginx:alpine + image: nginx:1.30.5-alpine container_name: texera-nginx depends_on: - workflow-compiling-service @@ -549,7 +549,7 @@ services: - "${TEXERA_PORT:-8080}:8080" startup-message: - image: alpine:latest + image: alpine:3.24.2 depends_on: nginx: condition: service_started @@ -571,7 +571,7 @@ services: # Only runs when the "examples" profile is activated: # docker compose --profile examples up example-data-loader: - image: alpine:latest + image: alpine:3.24.2 depends_on: dashboard-service: condition: service_healthy diff --git a/bin/y-websocket-server/Dockerfile b/bin/y-websocket-server/Dockerfile index 70df94c6f72..a4f3c2c9c7a 100644 --- a/bin/y-websocket-server/Dockerfile +++ b/bin/y-websocket-server/Dockerfile @@ -24,17 +24,17 @@ # completeness or stability of the code, it does indicate that the project # has yet to be fully endorsed by the ASF. -# Use an official Node runtime as a parent image -FROM node:24-slim +FROM docker.io/library/node:24.19.0-slim@sha256:a9f5f7c91a432850b2a8a7797adf5eadb6c733ceed61167806cee7ea7fbc29df -# Set the working directory in the container WORKDIR /usr/src/app -# Copy the current directory contents into the container at /usr/src/app -COPY . . +COPY package.json package-lock.json ./ -# Install any needed packages specified in package.json -RUN npm install +# npm's download cache indexes entries by fetch time, its debug log is named +# after the clock, and Node's compile cache in /tmp varies run to run; +# node_modules keeps its own copy of every file. +RUN npm ci \ + && rm -rf /root/.npm /tmp/node-compile-cache # Define environment variable ENV YPERSISTENCE=./leveldb diff --git a/bin/y-websocket-server/package-lock.json b/bin/y-websocket-server/package-lock.json new file mode 100644 index 00000000000..3cdd0ff1d17 --- /dev/null +++ b/bin/y-websocket-server/package-lock.json @@ -0,0 +1,546 @@ +{ + "name": "shared-editing-service", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "shared-editing-service", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "y-websocket": "^2.0.0" + } + }, + "node_modules/abstract-leveldown": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", + "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", + "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", + "license": "MIT", + "optional": true, + "dependencies": { + "buffer": "^5.5.0", + "immediate": "^3.2.3", + "level-concat-iterator": "~2.0.0", + "level-supports": "~1.0.0", + "xtend": "~4.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/async-limiter": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", + "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==", + "license": "MIT", + "optional": true + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "optional": true + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "optional": true, + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/deferred-leveldown": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz", + "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==", + "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", + "license": "MIT", + "optional": true, + "dependencies": { + "abstract-leveldown": "~6.2.1", + "inherits": "^2.0.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/encoding-down": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz", + "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==", + "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", + "license": "MIT", + "optional": true, + "dependencies": { + "abstract-leveldown": "^6.2.1", + "inherits": "^2.0.3", + "level-codec": "^9.0.0", + "level-errors": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/errno": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", + "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "license": "MIT", + "optional": true, + "dependencies": { + "prr": "~1.0.1" + }, + "bin": { + "errno": "cli.js" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/immediate": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.3.0.tgz", + "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==", + "license": "MIT", + "optional": true + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC", + "optional": true + }, + "node_modules/isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==", + "license": "MIT", + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, + "node_modules/level": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/level/-/level-6.0.1.tgz", + "integrity": "sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw==", + "license": "MIT", + "optional": true, + "dependencies": { + "level-js": "^5.0.0", + "level-packager": "^5.1.0", + "leveldown": "^5.4.0" + }, + "engines": { + "node": ">=8.6.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/level" + } + }, + "node_modules/level-codec": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.2.tgz", + "integrity": "sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==", + "deprecated": "Superseded by level-transcoder (https://github.com/Level/community#faq)", + "license": "MIT", + "optional": true, + "dependencies": { + "buffer": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/level-concat-iterator": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", + "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==", + "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/level-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", + "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", + "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", + "license": "MIT", + "optional": true, + "dependencies": { + "errno": "~0.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/level-iterator-stream": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz", + "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "inherits": "^2.0.4", + "readable-stream": "^3.4.0", + "xtend": "^4.0.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/level-js": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/level-js/-/level-js-5.0.2.tgz", + "integrity": "sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg==", + "deprecated": "Superseded by browser-level (https://github.com/Level/community#faq)", + "license": "MIT", + "optional": true, + "dependencies": { + "abstract-leveldown": "~6.2.3", + "buffer": "^5.5.0", + "inherits": "^2.0.3", + "ltgt": "^2.1.2" + } + }, + "node_modules/level-packager": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-5.1.1.tgz", + "integrity": "sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ==", + "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", + "license": "MIT", + "optional": true, + "dependencies": { + "encoding-down": "^6.3.0", + "levelup": "^4.3.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/level-supports": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", + "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", + "license": "MIT", + "optional": true, + "dependencies": { + "xtend": "^4.0.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/leveldown": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.6.0.tgz", + "integrity": "sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==", + "deprecated": "Superseded by classic-level (https://github.com/Level/community#faq)", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "abstract-leveldown": "~6.2.1", + "napi-macros": "~2.0.0", + "node-gyp-build": "~4.1.0" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/levelup": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.4.0.tgz", + "integrity": "sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==", + "deprecated": "Superseded by abstract-level (https://github.com/Level/community#faq)", + "license": "MIT", + "optional": true, + "dependencies": { + "deferred-leveldown": "~5.3.0", + "level-errors": "~2.0.0", + "level-iterator-stream": "~4.0.0", + "level-supports": "~1.0.0", + "xtend": "~4.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lib0": { + "version": "0.2.119", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.119.tgz", + "integrity": "sha512-vh+TiejVy8Xm+hCdnRfn9BthGNvNUZdEXltNjVZL3TzRIKP3s1lnQh8pWd/Q0wCgY0FPAGVVjFF4ZiHUV3LFgA==", + "license": "MIT", + "dependencies": { + "isomorphic.js": "^0.2.4" + }, + "bin": { + "0ecdsa-generate-keypair": "bin/0ecdsa-generate-keypair.js", + "0gentesthtml": "bin/gentesthtml.js", + "0serve": "bin/0serve.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, + "node_modules/lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", + "license": "MIT" + }, + "node_modules/ltgt": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", + "integrity": "sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA==", + "license": "MIT", + "optional": true + }, + "node_modules/napi-macros": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", + "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==", + "license": "MIT", + "optional": true + }, + "node_modules/node-gyp-build": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", + "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==", + "license": "MIT", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==", + "license": "MIT", + "optional": true + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "optional": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "optional": true + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "optional": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT", + "optional": true + }, + "node_modules/ws": { + "version": "6.2.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.6.tgz", + "integrity": "sha512-XTrf1gv7kXoVf1hbC3PAyAiPgR8Wz1blcrYIjEsUmr08BLksT41R8KbjmS9408C2ERx7v1JDLD/BkpLEttjfKA==", + "license": "MIT", + "optional": true, + "dependencies": { + "async-limiter": "~1.0.0" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.4" + } + }, + "node_modules/y-leveldb": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/y-leveldb/-/y-leveldb-0.1.2.tgz", + "integrity": "sha512-6ulEn5AXfXJYi89rXPEg2mMHAyyw8+ZfeMMdOtBbV8FJpQ1NOrcgi6DTAcXof0dap84NjHPT2+9d0rb6cFsjEg==", + "license": "MIT", + "optional": true, + "dependencies": { + "level": "^6.0.1", + "lib0": "^0.2.31" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "yjs": "^13.0.0" + } + }, + "node_modules/y-protocols": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.7.tgz", + "integrity": "sha512-YSVsLoXxO67J6eE/nV4AtFtT3QEotZf5sK5BHxFBXso7VDUT3Tx07IfA6hsu5Q5OmBdMkQVmFZ9QOA7fikWvnw==", + "license": "MIT", + "dependencies": { + "lib0": "^0.2.85" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "yjs": "^13.0.0" + } + }, + "node_modules/y-websocket": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/y-websocket/-/y-websocket-2.1.0.tgz", + "integrity": "sha512-WHYDRqomaGkkaujtowCDwL8KYk+t1zQCGIgKyvxvchhjTQlMgWXRHJK+FDEcWmHA7I7o/4fy0eniOrtmz0e4mA==", + "license": "MIT", + "dependencies": { + "lib0": "^0.2.52", + "lodash.debounce": "^4.0.8", + "y-protocols": "^1.0.5" + }, + "bin": { + "y-websocket": "bin/server.cjs", + "y-websocket-server": "bin/server.cjs" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "optionalDependencies": { + "ws": "^6.2.1", + "y-leveldb": "^0.1.0" + }, + "peerDependencies": { + "yjs": "^13.5.6" + } + }, + "node_modules/yjs": { + "version": "13.6.33", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.33.tgz", + "integrity": "sha512-q/UYvr1gOk4XII+Cu1CW2dINjYqJJFifTzogEaIARznrvX7uOlHsdTEXcnOBMuIsrCGEEKdNxi74kyMLZlKVfw==", + "license": "MIT", + "peer": true, + "dependencies": { + "lib0": "^0.2.99" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + } + } +} diff --git a/common/workflow-core/src/test/scala/org/apache/texera/service/util/RustFSContainer.scala b/common/workflow-core/src/test/scala/org/apache/texera/service/util/RustFSContainer.scala index fd07258f739..4e48fb7c873 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/service/util/RustFSContainer.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/service/util/RustFSContainer.scala @@ -42,7 +42,7 @@ import java.time.Duration object RustFSContainer { /** Image pinned in lockstep with `bin/single-node/docker-compose.yml` and `bin/k8s/values.yaml`. */ - val ImageName: String = "rustfs/rustfs:1.0.0-rc.6" + val ImageName: String = "rustfs/rustfs:1.0.0" /** S3 API port inside the container. */ val Port: Int = 9000 diff --git a/config-service/LICENSE-binary b/config-service/LICENSE-binary index 5f93f2b6780..5482852265e 100644 --- a/config-service/LICENSE-binary +++ b/config-service/LICENSE-binary @@ -315,8 +315,8 @@ Scala/Java jars: - org.jooq.jooq-3.19.36.jar - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar - - org.playframework.play-functional_2.13-3.1.0-M1.jar - - org.playframework.play-json_2.13-3.1.0-M1.jar + - org.playframework.play-functional_2.13-3.0.6.jar + - org.playframework.play-json_2.13-3.0.6.jar - org.scala-lang.modules.scala-collection-compat_2.13-2.13.0.jar - org.scala-lang.scala-library-2.13.18.jar - org.scala-lang.scala-reflect-2.13.18.jar diff --git a/config-service/build.sbt b/config-service/build.sbt index de3e034852f..4876d1f1219 100644 --- a/config-service/build.sbt +++ b/config-service/build.sbt @@ -85,6 +85,6 @@ libraryDependencies ++= Seq( "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.18.8", "jakarta.ws.rs" % "jakarta.ws.rs-api" % "3.1.0", // Ensure Jakarta JAX-RS API is available "org.bitbucket.b_c" % "jose4j" % "0.9.6", - "org.playframework" %% "play-json" % "3.1.0-M1", + "org.playframework" %% "play-json" % "3.0.6", "com.typesafe" % "config" % "1.4.9" // For configuration management ) \ No newline at end of file diff --git a/file-service/LICENSE-binary b/file-service/LICENSE-binary index 681bf5444a9..a8c6a08408f 100644 --- a/file-service/LICENSE-binary +++ b/file-service/LICENSE-binary @@ -443,8 +443,8 @@ Scala/Java jars: - org.json4s.json4s-ast_2.13-4.0.1.jar - org.json4s.json4s-jackson-core_2.13-4.0.1.jar - org.openapitools.jackson-databind-nullable-0.2.6.jar - - org.playframework.play-functional_2.13-3.1.0-M1.jar - - org.playframework.play-json_2.13-3.1.0-M1.jar + - org.playframework.play-functional_2.13-3.0.6.jar + - org.playframework.play-json_2.13-3.0.6.jar - org.roaringbitmap.RoaringBitmap-1.3.0.jar - org.scala-lang.modules.scala-collection-compat_2.13-2.13.0.jar - org.scala-lang.scala-library-2.13.18.jar diff --git a/file-service/build.sbt b/file-service/build.sbt index f060aff17f3..8d8cb22187a 100644 --- a/file-service/build.sbt +++ b/file-service/build.sbt @@ -90,5 +90,5 @@ libraryDependencies ++= Seq( "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.18.8", "jakarta.ws.rs" % "jakarta.ws.rs-api" % "3.1.0", // Ensure Jakarta JAX-RS API is available "org.bitbucket.b_c" % "jose4j" % "0.9.6", - "org.playframework" %% "play-json" % "3.1.0-M1", + "org.playframework" %% "play-json" % "3.0.6", ) diff --git a/file-service/src/main/resources/docker-compose.yml b/file-service/src/main/resources/docker-compose.yml index 98cef980291..ef323ca11de 100644 --- a/file-service/src/main/resources/docker-compose.yml +++ b/file-service/src/main/resources/docker-compose.yml @@ -18,7 +18,7 @@ name: texera-lakefs services: rustfs: - image: rustfs/rustfs:1.0.0-rc.6 + image: rustfs/rustfs:1.0.0 container_name: texera-lakefs-rustfs restart: unless-stopped ports: diff --git a/file-service/src/main/resources/rustfs-config.yml b/file-service/src/main/resources/rustfs-config.yml index f29fed9e27e..6f46ad811f8 100644 --- a/file-service/src/main/resources/rustfs-config.yml +++ b/file-service/src/main/resources/rustfs-config.yml @@ -19,7 +19,7 @@ version: '3.8' services: rustfs: - image: rustfs/rustfs:1.0.0-rc.6 + image: rustfs/rustfs:1.0.0 container_name: rustfs ports: - "9500:9000" # S3 API diff --git a/frontend/build-version.d.ts b/frontend/build-version.d.ts index 379241c3b63..3059795d522 100644 --- a/frontend/build-version.d.ts +++ b/frontend/build-version.d.ts @@ -18,6 +18,7 @@ */ // Types for build-version.js. +export function defaultBuildNumber(version: string, env?: Record): string; export function renderVersionArtifacts( version: string, buildNumber?: string diff --git a/frontend/build-version.js b/frontend/build-version.js index e66586d5c0e..048360c4080 100644 --- a/frontend/build-version.js +++ b/frontend/build-version.js @@ -21,13 +21,35 @@ // version.prod.ts with a timestamped build number; Angular's production // fileReplacements (angular.json) swaps version.ts for version.prod.ts. // Dev builds (`yarn start`) keep the static "dev" string in version.ts. +// When SOURCE_DATE_EPOCH is set the timestamp comes from it rather than the +// clock, so rebuilding the same commit reproduces the same bundle +// (https://reproducible-builds.org/specs/source-date-epoch/). const { generate } = require("build-number-generator"); const { resolve } = require("path"); const { writeFileSync } = require("fs"); +// build-number-generator only reads the local clock, so the epoch path +// reproduces its .YYMMDDCCC format (CCC counts two-minute slots) in UTC. +function buildNumberFromEpoch(version, sourceDateEpoch) { + const date = new Date(Number(sourceDateEpoch) * 1000); + if (!/^\d+$/.test(sourceDateEpoch) || Number.isNaN(date.getTime())) { + throw new Error(`SOURCE_DATE_EPOCH must be a whole number of seconds, got ${JSON.stringify(sourceDateEpoch)}`); + } + const pad = (value, width) => String(value).padStart(width, "0"); + const counter = Math.floor((date.getUTCHours() * 60 + date.getUTCMinutes()) / 2); + const stamp = + pad(date.getUTCFullYear() % 100, 2) + pad(date.getUTCMonth() + 1, 2) + pad(date.getUTCDate(), 2) + pad(counter, 3); + return `${version}.${stamp}`; +} + +function defaultBuildNumber(version, env = process.env) { + const sourceDateEpoch = env.SOURCE_DATE_EPOCH; + return sourceDateEpoch ? buildNumberFromEpoch(version, sourceDateEpoch) : generate(version); +} + // Returns the contents of version.prod.ts and version.json for a version. -function renderVersionArtifacts(version, buildNumber = generate(version)) { +function renderVersionArtifacts(version, buildNumber = defaultBuildNumber(version)) { const prodTs = `// AUTO-GENERATED by build-version.js — do not edit or commit. export const Version = { buildNumber: ${JSON.stringify(buildNumber)}, @@ -46,7 +68,7 @@ function main() { console.log(`build-version: ${buildNumber}`); } -module.exports = { renderVersionArtifacts }; +module.exports = { defaultBuildNumber, renderVersionArtifacts }; // Write files only when run directly, not when required by a test. if (require.main === module) { diff --git a/frontend/src/app/workspace/component/code-editor-dialog/code-editor.component.scss b/frontend/src/app/workspace/component/code-editor-dialog/code-editor.component.scss index 915811e4b4e..e80672ce428 100644 --- a/frontend/src/app/workspace/component/code-editor-dialog/code-editor.component.scss +++ b/frontend/src/app/workspace/component/code-editor-dialog/code-editor.component.scss @@ -20,7 +20,7 @@ #code-editor-container { position: fixed; left: 25vw; - top: random(100) + px; + top: 50px; z-index: 5; resize: both; overflow: hidden; diff --git a/frontend/src/build-version.spec.ts b/frontend/src/build-version.spec.ts index def462cc3f2..d6bf4146eaa 100644 --- a/frontend/src/build-version.spec.ts +++ b/frontend/src/build-version.spec.ts @@ -19,7 +19,8 @@ import { createRequire } from "node:module"; import { dirname, join } from "node:path"; -import { renderVersionArtifacts } from "../build-version"; +import { validate } from "build-number-generator"; +import { defaultBuildNumber, renderVersionArtifacts } from "../build-version"; describe("build-version: renderVersionArtifacts", () => { const VERSION = "1.2.3-incubating"; @@ -68,6 +69,60 @@ describe("build-version: renderVersionArtifacts", () => { }); }); + describe("with a build number derived from SOURCE_DATE_EPOCH (reproducible builds)", () => { + const realTz = process.env["TZ"]; + afterEach(() => { + if (realTz === undefined) { + delete process.env["TZ"]; + } else { + process.env["TZ"] = realTz; + } + }); + + it("formats the epoch as .YYMMDDCCC with a two-minute counter", () => { + // 2023-11-14T22:13:20Z: counter = (22 * 60 + 13) / 2 = 666. + expect(defaultBuildNumber(VERSION, { SOURCE_DATE_EPOCH: "1700000000" })).toBe(`${VERSION}.231114666`); + }); + + it("covers the day's boundaries: the epoch itself and the last minute of a day", () => { + expect(defaultBuildNumber(VERSION, { SOURCE_DATE_EPOCH: "0" })).toBe(`${VERSION}.700101000`); + expect(defaultBuildNumber(VERSION, { SOURCE_DATE_EPOCH: "86399" })).toBe(`${VERSION}.700101719`); + }); + + it("reads the epoch in UTC, so the build host's time zone cannot change the bundle", () => { + process.env["TZ"] = "Pacific/Kiritimati"; + expect(defaultBuildNumber(VERSION, { SOURCE_DATE_EPOCH: "86399" })).toBe(`${VERSION}.700101719`); + process.env["TZ"] = "Pacific/Pago_Pago"; + expect(defaultBuildNumber(VERSION, { SOURCE_DATE_EPOCH: "0" })).toBe(`${VERSION}.700101000`); + }); + + it("stays parseable by build-number-generator", () => { + expect(validate(defaultBuildNumber(VERSION, { SOURCE_DATE_EPOCH: "1700000000" }))).toBe(true); + }); + + it("is the same on every call for the same epoch", () => { + const env = { SOURCE_DATE_EPOCH: "1700000000" }; + expect(renderVersionArtifacts(VERSION, defaultBuildNumber(VERSION, env))).toEqual( + renderVersionArtifacts(VERSION, defaultBuildNumber(VERSION, env)) + ); + }); + + it("falls back to the wall clock when SOURCE_DATE_EPOCH is unset or empty", () => { + for (const env of [{}, { SOURCE_DATE_EPOCH: "" }]) { + const buildNumber = defaultBuildNumber(VERSION, env); + expect(buildNumber.startsWith(`${VERSION}.`)).toBe(true); + expect(validate(buildNumber)).toBe(true); + } + }); + + it.each(["abc", "-1", "1.5", " 12", "1e3", "0x10", "99999999999999999"])( + "rejects a malformed SOURCE_DATE_EPOCH %j instead of silently using the clock", + bad => { + expect(() => defaultBuildNumber(VERSION, { SOURCE_DATE_EPOCH: bad })).toThrow(/SOURCE_DATE_EPOCH/); + } + ); + }); + describe("with a generated build number (default argument)", () => { it("derives a non-empty build number from the version when none is supplied", () => { const { buildNumber, manifestJson } = renderVersionArtifacts(VERSION); @@ -203,6 +258,25 @@ describe("build-version: main()", () => { expect(prodTsWrite!.data).toContain(JSON.stringify(manifest.version)); }); + it("takes the build number from SOURCE_DATE_EPOCH when the build sets it", () => { + const realEpoch = process.env["SOURCE_DATE_EPOCH"]; + process.env["SOURCE_DATE_EPOCH"] = "1700000000"; + try { + const first = loadScript(true).writes; + const second = loadScript(true).writes; + const pkgVersion: string = requireCjs("./package.json").version; + const manifest = JSON.parse(first.find(w => w.path.endsWith("version.json"))!.data); + expect(manifest.buildNumber).toBe(`${pkgVersion}.231114666`); + expect(second).toEqual(first); + } finally { + if (realEpoch === undefined) { + delete process.env["SOURCE_DATE_EPOCH"]; + } else { + process.env["SOURCE_DATE_EPOCH"] = realEpoch; + } + } + }); + it("announces the build number it produced", () => { const { writes, logs } = loadScript(true); const manifest = JSON.parse(writes.find(w => w.path.endsWith("version.json"))!.data);