Skip to content

Publish Current Server #302

Publish Current Server

Publish Current Server #302

name: Publish Current Server
on:
workflow_dispatch:
inputs:
release_tag:
description: Exact semantic release tag declared by the Server image
required: true
type: string
permissions:
contents: write
packages: write
id-token: write
attestations: write
artifact-metadata: write
concurrency:
group: publish-current-server-${{ inputs.release_tag }}
cancel-in-progress: false
jobs:
publish:
if: github.repository == 'PastureStack/server' && github.ref == 'refs/heads/main'
runs-on: ubuntu-24.04
timeout-minutes: 90
env:
RELEASE_TAG: ${{ inputs.release_tag }}
SOURCE_SHA: ${{ github.sha }}
TARGET_IMAGE: ghcr.io/pasturestack/server:${{ inputs.release_tag }}
LAYERED_CANDIDATE_IMAGE: local/pasturestack/server-layered:${{ inputs.release_tag }}-${{ github.run_id }}
CANDIDATE_IMAGE: local/pasturestack/server:${{ inputs.release_tag }}-${{ github.run_id }}
CANDIDATE_NAME: pasturestack-server-release-${{ github.run_id }}
CATTLE_VOLUME: pasturestack-server-release-cattle-${{ github.run_id }}
MYSQL_VOLUME: pasturestack-server-release-mysql-${{ github.run_id }}
MYSQL_LOG_VOLUME: pasturestack-server-release-mysqllog-${{ github.run_id }}
TRIVY_IMAGE: ghcr.io/aquasecurity/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969
steps:
- name: Enable the containerd image store
shell: bash
run: |
set -Eeuo pipefail
if sudo test -f /etc/docker/daemon.json; then
sudo cat /etc/docker/daemon.json >"$RUNNER_TEMP/daemon-current.json"
else
printf '{}\n' >"$RUNNER_TEMP/daemon-current.json"
fi
jq '
.features = (.features // {})
| .features["containerd-snapshotter"] = true
' "$RUNNER_TEMP/daemon-current.json" >"$RUNNER_TEMP/daemon-containerd.json"
sudo install -o root -g root -m 0644 \
"$RUNNER_TEMP/daemon-containerd.json" /etc/docker/daemon.json
sudo systemctl restart docker
docker info --format '{{json .DriverStatus}}' |
grep -F 'io.containerd.snapshotter.v1' >/dev/null
- name: Check out exact clean source
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
set -Eeuo pipefail
[[ "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]
git clone --filter=blob:none "https://github.com/${GITHUB_REPOSITORY}.git" source
git -C source checkout --detach "$SOURCE_SHA"
test "$(git -C source rev-parse HEAD)" = "$SOURCE_SHA"
test -z "$(git -C source status --short)"
python3 source/server/security/test_scan_coverage.py
python3 source/server/security/test_web_console_sbom_scan.py
python3 source/scripts/test-render-server-release-notes.py
python3 source/scripts/test-publisher-safety-guards.py
python3 source/scripts/test-web-console-sbom.py
python3 source/scripts/test-server-backport-consumer-contract.py
python3 source/scripts/test-ai-resource-api-guards.py
declared_release="$(sed -n 's/^ARG SERVER_RELEASE_TAG=//p' \
source/server/Dockerfile.web-compose-release | sort -u)"
test "$declared_release" = "$RELEASE_TAG"
host_api_version="$(sed -n 's/^ARG HOST_API_VERSION=//p' \
source/server/Dockerfile.web-compose-release | sort -u)"
[[ "$host_api_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
printf 'HOST_API_VERSION=%s\n' "$host_api_version" >>"$GITHUB_ENV"
for web_pin in WEB_CONSOLE_RELEASE_TAG WEB_CONSOLE_ARTIFACT \
WEB_CONSOLE_ARTIFACT_SHA256 WEB_CONSOLE_COMMIT WEB_CONSOLE_LOCK_SHA256 \
WEB_CONSOLE_RUNTIME_SBOM_ARTIFACT WEB_CONSOLE_RUNTIME_SBOM_SHA256; do
value="$(sed -n "s/^ARG ${web_pin}=//p" \
source/server/Dockerfile.web-compose-release | sort -u)"
test -n "$value"
[[ "$value" != *$'\n'* ]]
printf '%s=%s\n' "$web_pin" "$value" >>"$GITHUB_ENV"
done
release_notes="source/docs/releases/server-${RELEASE_TAG#v}.md"
test -f "$release_notes"
test -s "$release_notes"
if git -C source ls-remote --exit-code --tags origin "refs/tags/${RELEASE_TAG}" \
>/dev/null 2>&1; then
echo "Release tag already exists: $RELEASE_TAG" >&2
exit 1
fi
if gh release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \
>/dev/null 2>&1; then
echo "GitHub Release already exists: $RELEASE_TAG" >&2
exit 1
fi
if docker manifest inspect "$TARGET_IMAGE" >/dev/null 2>&1; then
echo "Container tag already exists: $TARGET_IMAGE" >&2
exit 1
fi
mkdir -p "$RUNNER_TEMP/server-evidence"
- name: Run source gates
shell: bash
working-directory: source
run: |
set -Eeuo pipefail
bash scripts/check-server-source-gates.sh
- name: Build exact release image once
shell: bash
working-directory: source
run: |
set -Eeuo pipefail
PASTURESTACK_BUILD_NO_CACHE=1 IMAGE="$LAYERED_CANDIDATE_IMAGE" \
bash server/build-api-explorer-patch-image.sh |&
tee "$RUNNER_TEMP/server-evidence/build.log"
test "$(docker image inspect "$LAYERED_CANDIDATE_IMAGE" \
--format '{{index .Config.Labels "org.opencontainers.image.version"}}')" = \
"$RELEASE_TAG"
test "$(docker image inspect "$LAYERED_CANDIDATE_IMAGE" \
--format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = \
"$SOURCE_SHA"
base_ref='ghcr.io/pasturestack/server:v1.6.460@sha256:c855af8aea232dacc5bb6df68e2271d482c68b53c43ab0c108ec19118f5ab403'
base_count="$(docker buildx imagetools inspect --raw "$base_ref" |
jq -r '.layers | length')"
layered_count="$(docker image inspect "$LAYERED_CANDIDATE_IMAGE" |
jq -r '.[0].RootFS.Layers | length')"
test "$base_count" -eq 1
test "$layered_count" -gt 1
test "$layered_count" -le 32
printf 'SERVER_LAYERED_BUILD_OK base_layers=%s source_layers=%s maximum=32 base=v1.6.460\n' \
"$base_count" "$layered_count" |
tee "$RUNNER_TEMP/server-evidence/source-layers.txt"
- name: Flatten and compare the release image for classic overlay2
shell: bash
run: |
set -Eeuo pipefail
extract_name="pasturestack-server-layer-export-${GITHUB_RUN_ID}"
rootfs_tar="$RUNNER_TEMP/server-layered-rootfs.tar"
changes_file="$RUNNER_TEMP/server-flatten-changes.bin"
cleanup_extract() {
docker rm -v "$extract_name" >/dev/null 2>&1 || true
rm -f -- "$rootfs_tar" "$changes_file"
}
trap cleanup_extract EXIT
[[ "$LAYERED_CANDIDATE_IMAGE" =~ ^local/pasturestack/server-layered:v[0-9]+\.[0-9]+\.[0-9]+-[0-9]+$ ]]
[[ "$CANDIDATE_IMAGE" =~ ^local/pasturestack/server:v[0-9]+\.[0-9]+\.[0-9]+-[0-9]+$ ]]
docker create --name "$extract_name" "$LAYERED_CANDIDATE_IMAGE" >/dev/null
docker export --output "$rootfs_tar" "$extract_name"
test -f "$rootfs_tar"
test ! -L "$rootfs_tar"
docker image inspect "$LAYERED_CANDIDATE_IMAGE" |
python3 source/scripts/flatten-server-image.py changes >"$changes_file"
mapfile -d '' -t flatten_changes <"$changes_file"
test "${#flatten_changes[@]}" -gt 0
flatten_args=()
for change in "${flatten_changes[@]}"; do
flatten_args+=(--change "$change")
done
docker image import --platform linux/amd64 \
"${flatten_args[@]}" "$rootfs_tar" "$CANDIDATE_IMAGE" >/dev/null
docker image inspect "$LAYERED_CANDIDATE_IMAGE" "$CANDIDATE_IMAGE" |
python3 source/scripts/flatten-server-image.py verify |
tee "$RUNNER_TEMP/server-evidence/image-flatten.txt"
grep -F 'SERVER_IMAGE_FLATTEN_OK' \
"$RUNNER_TEMP/server-evidence/image-flatten.txt" >/dev/null
test "$(docker image inspect "$CANDIDATE_IMAGE" \
--format '{{index .Config.Labels "org.opencontainers.image.version"}}')" = \
"$RELEASE_TAG"
test "$(docker image inspect "$CANDIDATE_IMAGE" \
--format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = \
"$SOURCE_SHA"
cleanup_extract
trap - EXIT
- name: Verify the actual candidate JDK security runtime
shell: bash
working-directory: source
run: |
set -Eeuo pipefail
{
test "$(docker image inspect "$CANDIDATE_IMAGE" --format '{{range .Config.Env}}{{println .}}{{end}}' |
sed -n 's/^JAVA_VERSION=//p')" = jdk-25.0.5+8
test "$(docker image inspect "$CANDIDATE_IMAGE" --format '{{range .Config.Env}}{{println .}}{{end}}' |
sed -n 's/^PASTURESTACK_JAVA_ARCHIVE_SHA256=//p')" = \
68982529869257d0ee6b824ec5aa270ff8b118bb438670da0b97ebdab178d7fb
docker run --rm --network none --entrypoint /bin/bash "$CANDIDATE_IMAGE" -ec '
test "$(command -v java)" = /opt/java/openjdk/bin/java
test "$(readlink -f /usr/bin/java)" = /opt/java/openjdk/bin/java
test "$(readlink -f /usr/bin/javac)" = /opt/java/openjdk/bin/javac
java -version 2>&1 | tee /tmp/java-version
grep -F "25.0.5+8" /tmp/java-version
javac -version
test -s /opt/java/openjdk/lib/security/cacerts
test -s /opt/java/openjdk/legal/java.base/LICENSE
for lib in lib/server/libjvm.so lib/libnet.so lib/libnio.so; do
# The launcher loads libjvm before libnet/libnio. Model only
# those JDK-owned directories for this standalone ELF check.
deps=$(LD_LIBRARY_PATH=/opt/java/openjdk/lib:/opt/java/openjdk/lib/server ldd "/opt/java/openjdk/$lib")
if grep -F "not found" <<<"$deps"; then exit 1; fi
done
java --list-modules | grep -E "^(java.naming|java.security.jgss|jdk.crypto.ec)@25.0.5"
'
RC16_JDK25_CHECK_IMAGE="$CANDIDATE_IMAGE" bash scripts/check-server-java-patches-jdk25.sh
printf 'SERVER_JAVA_SECURITY_RUNTIME_OK version=25.0.5+8 official_archive_sha256=68982529869257d0ee6b824ec5aa270ff8b118bb438670da0b97ebdab178d7fb actual_candidate=1\n'
} |& tee "$RUNNER_TEMP/server-evidence/java-security-runtime.txt"
- name: Verify ordinary Brotli operations against the actual final image
shell: bash
run: |
set -Eeuo pipefail
evidence="$RUNNER_TEMP/server-evidence"
# Compile against pinned public declarations, without installing a
# decoder or relying on runner-provided development packages.
python3 - "$evidence/brotli-headers" <<'PY'
import hashlib, io, json, pathlib, re, sys, tarfile, urllib.request
lock = json.loads(pathlib.Path('source/server/security/ubuntu-backports/brotli/brotli.lock.json').read_text())
item, = (x for x in lock['sourceFiles'] if x['name'] == 'brotli_1.2.0.orig.tar.gz')
with urllib.request.urlopen(item['url'], timeout=60) as response:
data = response.read(item['size'] + 1)
assert len(data) == item['size'] and hashlib.sha256(data).hexdigest() == item['sha256']
output = pathlib.Path(sys.argv[1]) / 'brotli'
output.mkdir(parents=True)
with tarfile.open(fileobj=io.BytesIO(data)) as archive:
names = set()
for member in archive.getmembers():
match = re.fullmatch(r'google-brotli-028fb5a/c/include/brotli/([a-z_]+\.h)', member.name)
if not match:
continue
assert member.isfile() and match[1] not in names
names.add(match[1])
(output / match[1]).write_bytes(archive.extractfile(member).read())
assert {'decode.h', 'encode.h', 'types.h', 'port.h'} <= names
assert hashlib.sha256((output / 'decode.h').read_bytes()).hexdigest() == lock['baselineSha256']['c/include/brotli/decode.h']
PY
cc -O2 -Wall -Wextra -Werror \
-I"$evidence/brotli-headers" \
source/server/security/ubuntu-backports/bundle/installer/testdata/consumer-brotli-probe.c \
-ldl -o "$evidence/brotli-runtime-probe"
docker run --rm --network none \
--mount "type=bind,source=$evidence/brotli-runtime-probe,target=/tmp/brotli-runtime-probe,readonly" \
--entrypoint /tmp/brotli-runtime-probe "$CANDIDATE_IMAGE" |
tee "$evidence/brotli-runtime.txt"
grep -Fx 'PASS: three installed Brotli SONAMEs RTLD_NOW, ordinary compression/decompression' \
"$evidence/brotli-runtime.txt"
- name: Start, restart, and export the candidate
shell: bash
run: |
set -Eeuo pipefail
evidence="$RUNNER_TEMP/server-evidence"
rootfs="$RUNNER_TEMP/server-rootfs"
capture_candidate_evidence() {
if docker inspect "$CANDIDATE_NAME" >/dev/null 2>&1; then
docker logs "$CANDIDATE_NAME" >"$evidence/candidate.log" 2>&1 || true
docker inspect "$CANDIDATE_NAME" >"$evidence/candidate-inspect.json" \
2>/dev/null || true
fi
}
cleanup_candidate() {
capture_candidate_evidence
docker rm -f "$CANDIDATE_NAME" >/dev/null 2>&1 || true
docker volume rm "$CATTLE_VOLUME" "$MYSQL_VOLUME" "$MYSQL_LOG_VOLUME" \
>/dev/null 2>&1 || true
}
for object in "$CANDIDATE_NAME" "$CATTLE_VOLUME" "$MYSQL_VOLUME" "$MYSQL_LOG_VOLUME"; do
if docker inspect "$object" >/dev/null 2>&1; then echo 'Run-owned object already exists' >&2; exit 1; fi
done
# Only arm cleanup after proving all names are available. A collision
# must never remove another run's container or persistent volumes.
trap cleanup_candidate EXIT
# Exercise the opt-in protocol on this disposable runtime while
# retaining the image's required Java arguments. Image defaults and
# the published configuration remain unchanged.
candidate_java_opts="$(docker image inspect "$CANDIDATE_IMAGE" \
--format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^JAVA_OPTS=//p')"
docker run -d \
--name "$CANDIDATE_NAME" \
--restart no \
-p 127.0.0.1:18082:8080 \
-e PROXY_PLATFORM_PUBLIC_ORIGIN=https://stack.example.test \
-e "JAVA_OPTS=${candidate_java_opts} -Dapi.ai.bridge.enabled=true -Dapi.ai.service.upgrade.enabled=true -Dapi.ai.resource.change.enabled=true" \
-v "$CATTLE_VOLUME:/var/lib/cattle" \
-v "$MYSQL_VOLUME:/var/lib/mysql" \
-v "$MYSQL_LOG_VOLUME:/var/log/mysql" \
"$CANDIDATE_IMAGE" >/dev/null
wait_for_pong() {
local body attempt
for attempt in $(seq 1 180); do
if [[ "$(docker inspect "$CANDIDATE_NAME" \
--format '{{.State.Running}}' 2>/dev/null || true)" != true ]]; then
return 1
fi
body="$(curl -fsS --max-time 3 http://127.0.0.1:18082/ping 2>/dev/null || true)"
if [[ "$body" == pong ]]; then
printf '%s' "$attempt"
return 0
fi
sleep 2
done
return 1
}
header_value() {
local file=$1 name=$2
awk -v name="$name" '
BEGIN { IGNORECASE = 1 }
{
line = $0
sub(/\r$/, "", line)
colon = index(line, ":")
if (colon > 0 && tolower(substr(line, 1, colon - 1)) == tolower(name)) {
value = substr(line, colon + 1)
sub(/^[[:space:]]+/, "", value)
}
}
END { print value }
' "$file"
}
assert_public_proxy_contract() {
local phase=$1 attempt schema_code schema_value path slug response_code
local private_api_attempt cache_control pragma expires age
local schema_matched=false
local schema_headers="$RUNNER_TEMP/public-origin-${phase}-schema.headers"
local response_headers private_api_matched
local summary="$evidence/reverse-proxy-${phase}.txt"
public_proxy_contract_attempts=0
: >"$summary"
for attempt in $(seq 1 60); do
public_proxy_contract_attempts="$attempt"
: >"$schema_headers"
schema_code="$(curl -sS --max-time 5 -D "$schema_headers" -o /dev/null \
-w '%{http_code}' -H 'Host: stack.example.test' \
http://127.0.0.1:18082/v2-beta/schemas 2>/dev/null || true)"
schema_value="$(header_value "$schema_headers" X-Api-Schemas 2>/dev/null || true)"
if [[ "$schema_code" == 200 && \
"$schema_value" == https://stack.example.test/v2-beta/schemas ]]; then
schema_matched=true
break
fi
sleep 2
done
printf 'phase=%s path=/v2-beta/schemas status=%s x_api_schemas=%s\n' \
"$phase" "$schema_code" "$schema_value" >>"$summary"
if [[ "$schema_matched" != true ]]; then
echo "Public proxy contract did not converge during ${phase}: status=${schema_code} X-Api-Schemas=${schema_value}" >&2
return 1
fi
for path in /v2-beta/accounts /v1-auth/config; do
slug="${path#/}"
slug="${slug//\//-}"
response_headers="$RUNNER_TEMP/public-origin-${phase}-${slug}.headers"
private_api_matched=false
for private_api_attempt in $(seq 1 60); do
: >"$response_headers"
response_code="$(curl -sS --max-time 5 -D "$response_headers" -o /dev/null \
-w '%{http_code}' -H 'Host: stack.example.test' \
"http://127.0.0.1:18082${path}" 2>/dev/null || true)"
cache_control="$(header_value "$response_headers" Cache-Control 2>/dev/null || true)"
pragma="$(header_value "$response_headers" Pragma 2>/dev/null || true)"
expires="$(header_value "$response_headers" Expires 2>/dev/null || true)"
age="$(header_value "$response_headers" Age 2>/dev/null || true)"
if [[ "$response_code" =~ ^(200|401|403)$ && \
"$cache_control" == 'private, no-store' && \
"$pragma" == 'no-cache' && "$expires" == 0 && -z "$age" ]]; then
private_api_matched=true
break
fi
sleep 2
done
if (( private_api_attempt > public_proxy_contract_attempts )); then
public_proxy_contract_attempts="$private_api_attempt"
fi
printf 'phase=%s path=%s status=%s cache_control=%s pragma=%s expires=%s age=%s attempts=%s\n' \
"$phase" "$path" "$response_code" "$cache_control" "$pragma" \
"$expires" "${age:-absent}" "$private_api_attempt" >>"$summary"
if [[ "$private_api_matched" != true ]]; then
echo "Private API contract did not converge during ${phase}: path=${path} status=${response_code} cache_control=${cache_control} pragma=${pragma} expires=${expires} age=${age:-absent}" >&2
return 1
fi
done
}
first_attempts="$(wait_for_pong)"
for catalog_attempts in $(seq 1 90); do
docker logs "$CANDIDATE_NAME" >"$evidence/candidate.log" 2>&1
if grep -Fq 'PastureStack Catalog bootstrap verified a non-empty collection.' \
"$evidence/candidate.log"; then
break
fi
sleep 2
done
grep -Fq 'PastureStack Catalog bootstrap verified a non-empty collection.' \
"$evidence/candidate.log"
if grep -Fq "mysql_native_password': this user requires mysql native password authentication" "$evidence/candidate.log"; then echo 'Candidate database authentication failed' >&2; exit 1; fi
docker inspect "$CANDIDATE_NAME" --format '{{range .Config.Env}}{{println .}}{{end}}' |
grep -Fx "CATTLE_RANCHER_SERVER_VERSION=$RELEASE_TAG" >/dev/null
docker inspect "$CANDIDATE_NAME" --format '{{range .Config.Env}}{{println .}}{{end}}' |
grep -Fx 'PROXY_PLATFORM_PUBLIC_ORIGIN=https://stack.example.test' >/dev/null
assert_public_proxy_contract before-restart
first_contract_attempts="$public_proxy_contract_attempts"
docker restart "$CANDIDATE_NAME" >/dev/null
restart_attempts="$(wait_for_pong)"
assert_public_proxy_contract after-restart
restart_contract_attempts="$public_proxy_contract_attempts"
test "$(docker inspect "$CANDIDATE_NAME" --format '{{.State.Running}}')" = true
http_code="$(curl -sS --max-time 3 -o "$RUNNER_TEMP/ping-body" \
-w '%{http_code}' http://127.0.0.1:18082/ping)"
test "$http_code" = 200
test "$(cat "$RUNNER_TEMP/ping-body")" = pong
# Exercise the published API and real auth overlay on this fresh,
# disposable database before the image is allowed to be released.
python3 source/scripts/test-mfa-policy-api.py \
--port 18082 --disposable --registration-basic --ai-resource |
tee "$evidence/mfa-policy-api-smoke.txt"
grep -E '^AI_RESOURCE_API_OK checks=[0-9]+ scoped_create_read=true process_pool=true human_cookie=true no_hosts=true$' \
"$evidence/mfa-policy-api-smoke.txt" >/dev/null
grep -E '^REGISTRATION_BASIC_API_OK checks=7 ordinary_user=true entry_only=true node_active_verified=false script_sha256=[0-9a-f]{64}$' \
"$evidence/mfa-policy-api-smoke.txt" >"$evidence/registration-basic-api-smoke.txt"
docker stop "$CANDIDATE_NAME" >/dev/null
mkdir -p "$rootfs"
docker export --output "$RUNNER_TEMP/server-rootfs.tar" "$CANDIDATE_NAME"
tar --no-same-owner -xf "$RUNNER_TEMP/server-rootfs.tar" -C "$rootfs"
test "$(readlink "$rootfs/usr/share/cattle/artifacts/host-api.tar.gz")" = \
"host-api-${HOST_API_VERSION}.tar.gz"
grep -Fx \
"export CATTLE_AGENT_PACKAGE_HOST_API_URL=/usr/share/cattle/artifacts/host-api-${HOST_API_VERSION}.tar.gz" \
"$rootfs/usr/share/cattle/env_vars" >/dev/null
cp "$rootfs/usr/share/cattle/artifacts/host-api-${HOST_API_VERSION}.tar.gz" \
"$evidence/host-api-${HOST_API_VERSION}.tar.gz"
host_default() {
sed -n "s/^ARG ${1}=//p" source/server/Dockerfile.web-compose-release | sort -u
}
if [[ "$(host_default HOST_API_PACKAGE_MODE)" == producer ]]; then
printf '%s %s\n' "$(host_default HOST_API_ARCHIVE_SHA256)" \
"$evidence/host-api-${HOST_API_VERSION}.tar.gz" | sha256sum -c -
bash source/server/artifacts/verify-host-api-package.sh \
"$evidence/host-api-${HOST_API_VERSION}.tar.gz" \
"$(host_default HOST_API_PACKAGE_ID)" \
"$(host_default HOST_API_BINARY_SHA256)" \
"$(host_default HOST_API_APPLY_SHA256)" \
"$(host_default HOST_API_COMMIT)" |
tee "$evidence/host-api-package-check.txt"
else
test "$(host_default HOST_API_PACKAGE_MODE)" = legacy-repair
test "$HOST_API_VERSION" = 0.38.4
bash source/scripts/check-server-host-api-package.sh \
"$evidence/host-api-${HOST_API_VERSION}.tar.gz" |
tee "$evidence/host-api-package-check.txt"
fi
printf 'SERVER_CANDIDATE_SMOKE_OK release=%s first_attempts=%s restart_attempts=%s catalog_attempts=%s first_contract_attempts=%s restart_contract_attempts=%s http=200 body=pong public_origin=https://stack.example.test private_api_cache=no-store\n' \
"$RELEASE_TAG" "$first_attempts" "$restart_attempts" "$catalog_attempts" \
"$first_contract_attempts" "$restart_contract_attempts" |
tee "$evidence/candidate-smoke.txt"
cleanup_candidate
trap - EXIT
- name: Retain the independent browser runtime SBOM from the actual candidate
shell: bash
run: |
set -Eeuo pipefail
evidence="$RUNNER_TEMP/server-evidence"
image_sbom="$RUNNER_TEMP/server-rootfs/usr/share/licenses/pasturestack/web-console/runtime.cdx.json"
test -f "$image_sbom"
test ! -L "$image_sbom"
cp "$image_sbom" "$evidence/web-console-runtime.cdx.json"
cmp "$image_sbom" "$evidence/web-console-runtime.cdx.json"
python3 source/server/artifacts/verify-web-console-sbom.py \
--sbom "$evidence/web-console-runtime.cdx.json" \
--sbom-sha256 "$WEB_CONSOLE_RUNTIME_SBOM_SHA256" \
--version "$WEB_CONSOLE_RELEASE_TAG" --source-commit "$WEB_CONSOLE_COMMIT" \
--archive-name "$WEB_CONSOLE_ARTIFACT" --archive-sha256 "$WEB_CONSOLE_ARTIFACT_SHA256" \
--lock-sha256 "$WEB_CONSOLE_LOCK_SHA256" |
tee "$evidence/web-console-sbom-identity.txt"
(
cd "$evidence"
sha256sum web-console-runtime.cdx.json >web-console-runtime.cdx.json.sha256
)
- name: Scan the final merged runtime and create SBOM
shell: bash
run: |
set -Eeuo pipefail
evidence="$RUNNER_TEMP/server-evidence"
(
cd "$RUNNER_TEMP"
python3 "$GITHUB_WORKSPACE/source/server/artifacts/prepare-nested-agent-scan.py"
) >"$RUNNER_TEMP/server-nested-qualified.json"
mkdir -p "$RUNNER_TEMP/trivy-cache"
docker pull "$TRIVY_IMAGE" >/dev/null
docker image inspect "$TRIVY_IMAGE" --format '{{json .RepoDigests}}' |
jq -e --arg expected \
'ghcr.io/aquasecurity/trivy@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969' \
'index($expected) != null' >/dev/null
docker run --rm \
-v "$RUNNER_TEMP/trivy-cache:/root/.cache/trivy" \
"$TRIVY_IMAGE" image --download-db-only
# Browser component lookup is independent of the OS/rootfs reports.
# No OS VEX, ignorefile, build BOM or hosted SDK inventory is applied.
docker run --rm \
-v "$evidence:/evidence" \
-v "$RUNNER_TEMP/trivy-cache:/root/.cache/trivy" \
"$TRIVY_IMAGE" --config /dev/null sbom \
--scanners vuln --skip-db-update --skip-vex-repo-update \
--ignorefile /dev/null --list-all-pkgs --format json \
--output /evidence/web-console-runtime-scan-raw.json \
/evidence/web-console-runtime.cdx.json
python3 source/server/security/check_web_console_sbom_scan.py \
--sbom "$evidence/web-console-runtime.cdx.json" \
--scan "$evidence/web-console-runtime-scan-raw.json" \
--effective "$evidence/web-console-runtime-scan-effective.json" \
--coverage "$evidence/web-console-runtime-scan-coverage.json" \
--findings "$evidence/web-console-runtime-vulnerabilities.tsv" \
--db-metadata "$RUNNER_TEMP/trivy-cache/db/metadata.json" \
--db-file "$RUNNER_TEMP/trivy-cache/db/trivy.db" \
--scanner-image "$TRIVY_IMAGE"
cp source/server/security/openvex.json "$evidence/server.openvex.json"
cp source/server/security/vendor-pending.json \
"$evidence/server.vendor-pending.json"
# Compressed delivery bundles need a separate actual-binary scan.
# Preserve the raw rootfs report; publish the nested raw scan and SBOM.
docker run --rm \
-v "$RUNNER_TEMP/server-nested-payloads:/payloads:ro" \
-v "$evidence:/evidence" \
-v "$RUNNER_TEMP/trivy-cache:/root/.cache/trivy" \
"$TRIVY_IMAGE" rootfs --scanners vuln,secret --list-all-pkgs --format json \
--output /evidence/server-nested-scan-raw.json /payloads
jq -e '.SchemaVersion == 2 and ([.Results[]? | select(.Type == "gobinary")] | length) == 4
and ([.Results[]? | select(.Type == "gobinary") | .Target | split("/") | last] | sort)
== (["host-api-linux-amd64","node-agent-linux-amd64","node-agent-windows-amd64.exe","per-host-subnet-windows-amd64.exe"] | sort)' \
"$evidence/server-nested-scan-raw.json" >/dev/null
docker run --rm \
-v "$RUNNER_TEMP/server-nested-payloads:/payloads:ro" \
-v "$evidence:/evidence" \
-v "$RUNNER_TEMP/trivy-cache:/root/.cache/trivy" \
"$TRIVY_IMAGE" rootfs --format cyclonedx \
--output /evidence/server-nested-sbom.cdx.json /payloads
jq -e '.bomFormat == "CycloneDX" and (.components | length) > 0' \
"$evidence/server-nested-sbom.cdx.json" >/dev/null
jq -n --slurpfile qualified "$RUNNER_TEMP/server-nested-qualified.json" \
--slurpfile raw "$evidence/server-nested-scan-raw.json" \
--slurpfile sbom "$evidence/server-nested-sbom.cdx.json" \
'{qualification:$qualified[0],raw:$raw[0],sbom:$sbom[0]}' \
>"$evidence/server-nested-payloads.json"
docker run --rm \
-v "$RUNNER_TEMP/server-rootfs:/rootfs:ro" \
-v "$evidence:/evidence" \
-v "$RUNNER_TEMP/trivy-cache:/root/.cache/trivy" \
"$TRIVY_IMAGE" rootfs \
--scanners vuln,secret --list-all-pkgs \
--format json \
--output /evidence/server-security-scan-raw.json \
/rootfs
jq -e '.SchemaVersion == 2 and (.Results | type == "array")' \
"$evidence/server-security-scan-raw.json" >/dev/null
(
cd "$evidence"
python3 "$GITHUB_WORKSPACE/source/server/security/check_scan_coverage.py"
) | tee "$evidence/server-go-scan-coverage.txt"
jq -r '
.Results[]? | .Target as $target | .Vulnerabilities[]?
| [.Severity, .VulnerabilityID, .PkgName, .InstalledVersion,
(.FixedVersion // ""), ($target // "")] | @tsv
' "$evidence/server-security-scan-raw.json" | LC_ALL=C sort -u \
>"$evidence/server-vulnerabilities-raw.tsv"
jq -r '
.raw.Results[]? | .Target as $target | .Vulnerabilities[]?
| [.Severity, .VulnerabilityID, .PkgName, .InstalledVersion,
(.FixedVersion // ""), ("active-payload/" + $target)] | @tsv
' "$evidence/server-nested-payloads.json" | LC_ALL=C sort -u \
>>"$evidence/server-vulnerabilities-raw.tsv"
cat "$evidence/web-console-runtime-vulnerabilities.tsv" \
>>"$evidence/server-vulnerabilities-raw.tsv"
jq -r '
.Results[]? | .Target as $target | .Secrets[]?
| [.RuleID, .Category, .Severity, (.Title // ""), ($target // "")]
| @tsv
' "$evidence/server-security-scan-raw.json" | LC_ALL=C sort -u \
>"$evidence/server-secrets.tsv"
jq -r '
.raw.Results[]? | .Target as $target | .Secrets[]?
| [.RuleID, .Category, .Severity, (.Title // ""), ("active-payload/" + $target)] | @tsv
' "$evidence/server-nested-payloads.json" >>"$evidence/server-secrets.tsv"
test ! -s "$evidence/server-secrets.tsv"
docker run --rm \
-v "$RUNNER_TEMP/server-rootfs:/rootfs:ro" \
-v "$evidence:/evidence" \
-v "$RUNNER_TEMP/trivy-cache:/root/.cache/trivy" \
"$TRIVY_IMAGE" rootfs \
--format cyclonedx \
--output /evidence/server.cdx.json \
/rootfs
jq -e '
.bomFormat == "CycloneDX"
and ((.components | type) == "array")
and ((.components | length) > 0)
' "$evidence/server.cdx.json" >/dev/null
# Bind the candidate's retained metadata to the same immutable OCI
# before passing its exact-package VEX to native Trivy.
cp -a "$RUNNER_TEMP/server-rootfs/usr/share/pasturestack/security/ubuntu-backports" "$evidence/ubuntu-backports"
(cd "$evidence/ubuntu-backports" && sha256sum --check SHA256SUMS)
bundle_image="$(sed -n 's/^FROM \(ghcr.io\/pasturestack\/ubuntu-security-backports:v0\.1\.1@sha256:[0-9a-f]\{64\}\) AS ubuntu_security_backports$/\1/p' source/server/Dockerfile.web-compose-release)"
[[ "$bundle_image" =~ ^ghcr.io/pasturestack/ubuntu-security-backports:v0\.1\.1@sha256:[0-9a-f]{64}$ ]]
docker pull --platform linux/amd64 "$bundle_image"
mkdir -p "$evidence/bundle-tools"
bundle_container="$(docker create --entrypoint /unused "$bundle_image")"
trap 'docker rm "$bundle_container" >/dev/null' EXIT
docker cp "$bundle_container:/out/verify-consumer-evidence.py" "$evidence/bundle-tools/verify-consumer-evidence.py"
docker cp "$bundle_container:/out/SHA256SUMS" "$evidence/bundle-tools/SHA256SUMS"
docker rm "$bundle_container" >/dev/null
trap - EXIT
(cd "$evidence/bundle-tools" && \
test "$(grep -Ec '^[0-9a-f]{64} verify-consumer-evidence\.py$' SHA256SUMS)" -eq 1 && \
grep -E '^[0-9a-f]{64} verify-consumer-evidence\.py$' SHA256SUMS | sha256sum --check -)
python3 "$evidence/bundle-tools/verify-consumer-evidence.py" \
--evidence-dir "$evidence/ubuntu-backports" --sbom "$evidence/server.cdx.json" \
--expected-bundle-digest "${bundle_image##*@}" \
--bundle-checksums "$evidence/bundle-tools/SHA256SUMS" \
>"$evidence/ubuntu-backports-validation.json"
tar -cf "$evidence/ubuntu-backports-evidence.tar" -C "$evidence" \
ubuntu-backports bundle-tools ubuntu-backports-validation.json
docker run --rm \
-v "$RUNNER_TEMP/server-rootfs:/rootfs:ro" \
-v "$evidence:/evidence" \
-v "$RUNNER_TEMP/trivy-cache:/root/.cache/trivy" \
"$TRIVY_IMAGE" rootfs \
--scanners vuln \
--skip-vex-repo-update \
--vex /evidence/server.openvex.json \
--vex /evidence/ubuntu-backports/openvex.json \
--format json \
--output /evidence/server-security-scan.json \
/rootfs
jq -e '.SchemaVersion == 2 and (.Results | type == "array")' \
"$evidence/server-security-scan.json" >/dev/null
jq -r '
.Results[]? | .Target as $target | .Vulnerabilities[]?
| [.Severity, .VulnerabilityID, .PkgName, .InstalledVersion,
(.FixedVersion // ""), ($target // "")] | @tsv
' "$evidence/server-security-scan.json" | LC_ALL=C sort -u \
>"$evidence/server-vulnerabilities-unresolved.tsv"
jq -r '
.raw.Results[]? | .Target as $target | .Vulnerabilities[]?
| [.Severity, .VulnerabilityID, .PkgName, .InstalledVersion,
(.FixedVersion // ""), ("active-payload/" + $target)] | @tsv
' "$evidence/server-nested-payloads.json" | LC_ALL=C sort -u \
>>"$evidence/server-vulnerabilities-unresolved.tsv"
cat "$evidence/web-console-runtime-vulnerabilities.tsv" \
>>"$evidence/server-vulnerabilities-unresolved.tsv"
bash source/scripts/validate-vendor-pending-findings.sh \
"$evidence/server.vendor-pending.json" \
"$evidence/server-vulnerabilities-unresolved.tsv" \
"$RELEASE_TAG" | tee "$evidence/vendor-pending-summary.txt"
raw_count="$(wc -l <"$evidence/server-vulnerabilities-raw.tsv")"
vendor_pending_count="$(wc -l <"$evidence/server-vulnerabilities-unresolved.tsv" | tr -d ' ')"
browser_scan_coverage="$(jq -r '.coverage' "$evidence/web-console-runtime-scan-coverage.json")"
printf 'SERVER_SECURITY_GATE_OK untracked=0 vendor_pending=%s critical_high=0 fixed_available=0 secrets=0 raw_findings=%s vex=openvex-0.2.0 scan_target=merged-rootfs-and-active-payloads-and-browser-runtime-sbom trivy=0.74.0 browser_scan_coverage=%s\n' \
"$vendor_pending_count" "$raw_count" "$browser_scan_coverage" |
tee "$evidence/security-summary.txt"
- name: Publish semantic image
id: publish
shell: bash
env:
REGISTRY_TOKEN: ${{ github.token }}
run: |
set -Eeuo pipefail
printf '%s' "$REGISTRY_TOKEN" |
docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin >/dev/null
docker tag "$CANDIDATE_IMAGE" "$TARGET_IMAGE"
docker push "$TARGET_IMAGE" | tee "$RUNNER_TEMP/server-evidence/push.log"
digest="$(docker buildx imagetools inspect "$TARGET_IMAGE" \
--format '{{.Manifest.Digest}}')"
[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]
digest_sha256="${digest#sha256:}"
image_purl="pkg:oci/server@sha256%3A${digest_sha256}?repository_url=ghcr.io%2Fpasturestack%2Fserver&tag=${RELEASE_TAG}"
sbom="$RUNNER_TEMP/server-evidence/server.cdx.json"
sbom_tmp="$RUNNER_TEMP/server-evidence/server.cdx.tmp.json"
jq \
--arg name 'ghcr.io/pasturestack/server' \
--arg release "$RELEASE_TAG" \
--arg digest "$digest_sha256" \
--arg purl "$image_purl" \
--arg source "$SOURCE_SHA" '
(.metadata.component["bom-ref"] // "") as $old_ref
| .metadata = (.metadata // {})
| .metadata.component = {
"type": "container",
"bom-ref": $purl,
"name": $name,
"version": $release,
"purl": $purl,
"hashes": [{"alg": "SHA-256", "content": $digest}],
"properties": [
{
"name": "org.opencontainers.image.revision",
"value": $source
},
{
"name": "org.opencontainers.image.ref.name",
"value": ($name + ":" + $release)
}
]
}
| if (.dependencies | type) == "array" then
.dependencies |= map(
.ref = (if .ref == $old_ref then $purl else .ref end)
| if (.dependsOn | type) == "array" then
.dependsOn |= map(if . == $old_ref then $purl else . end)
else
.
end
)
else
.
end
' "$sbom" >"$sbom_tmp"
mv "$sbom_tmp" "$sbom"
jq -e \
--arg release "$RELEASE_TAG" \
--arg digest "$digest_sha256" \
--arg purl "$image_purl" '
.bomFormat == "CycloneDX"
and .metadata.component.type == "container"
and .metadata.component.name == "ghcr.io/pasturestack/server"
and .metadata.component.version == $release
and .metadata.component.purl == $purl
and .metadata.component["bom-ref"] == $purl
and (.metadata.component.hashes | length) == 1
and .metadata.component.hashes[0].alg == "SHA-256"
and .metadata.component.hashes[0].content == $digest
and ([.dependencies[]? | select(.ref == $purl)] | length) == 1
' "$sbom" >/dev/null
printf 'SERVER_SBOM_IDENTITY_OK release=%s image=%s digest=%s purl=%s\n' \
"$RELEASE_TAG" "$TARGET_IMAGE" "$digest" "$image_purl" |
tee "$RUNNER_TEMP/server-evidence/sbom-identity.txt"
docker buildx imagetools inspect --raw "$TARGET_IMAGE" |
jq -e \
'.schemaVersion == 2 and (.config.digest | test("^sha256:[0-9a-f]{64}$")) and (.layers | length) == 1' \
>"$RUNNER_TEMP/server-evidence/registry-layer-verification.json"
printf 'SERVER_REGISTRY_LAYER_OK release=%s layers=1\n' \
"$RELEASE_TAG" |
tee "$RUNNER_TEMP/server-evidence/registry-layer-summary.txt"
docker buildx imagetools inspect "$TARGET_IMAGE" \
>"$RUNNER_TEMP/server-evidence/image-inspect.txt"
printf 'SERVER_IMAGE_PUBLISHED release=%s source=%s reference=%s digest=%s\n' \
"$RELEASE_TAG" "$SOURCE_SHA" "$TARGET_IMAGE" "$digest" |
tee "$RUNNER_TEMP/server-evidence/published.txt"
printf 'digest=%s\n' "$digest" >>"$GITHUB_OUTPUT"
- name: Attest image provenance
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
with:
subject-name: ghcr.io/pasturestack/server
subject-digest: ${{ steps.publish.outputs.digest }}
push-to-registry: true
- name: Attest image SBOM
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
with:
subject-name: ghcr.io/pasturestack/server
subject-digest: ${{ steps.publish.outputs.digest }}
sbom-path: ${{ runner.temp }}/server-evidence/server.cdx.json
push-to-registry: true
- name: Create immutable GitHub Release
shell: bash
env:
GH_TOKEN: ${{ github.token }}
PUBLISHED_DIGEST: ${{ steps.publish.outputs.digest }}
run: |
set -Eeuo pipefail
evidence="$RUNNER_TEMP/server-evidence"
release_notes="source/docs/releases/server-${RELEASE_TAG#v}.md"
python3 source/scripts/render-server-release-notes.py \
"$release_notes" "$RELEASE_TAG" "$PUBLISHED_DIGEST" "$SOURCE_SHA" \
>"$evidence/release-notes.md"
# An empty internal findings table is valid, but Release asset upload
# requires a nonempty body. Keep gate input bytes unchanged and expose
# a deterministic, lossless representation of that exact report.
gzip -n -c "$evidence/web-console-runtime-vulnerabilities.tsv" \
>"$evidence/web-console-runtime-vulnerabilities.tsv.gz"
test -s "$evidence/web-console-runtime-vulnerabilities.tsv.gz"
# Recheck immutable producer bytes after report conversion. A valid
# gzip stream is not proof of a valid host package (an empty gzip is
# valid too). Never transform or re-compress this producer archive.
host_archive_sha="$(sed -n 's/^ARG HOST_API_ARCHIVE_SHA256=//p' \
source/server/Dockerfile.web-compose-release | sort -u)"
[[ "$host_archive_sha" =~ ^[0-9a-f]{64}$ ]]
printf '%s %s\n' "$host_archive_sha" \
"$evidence/host-api-${HOST_API_VERSION}.tar.gz" | sha256sum -c -
(
cd "$evidence"
sha256sum \
build.log \
brotli-runtime.txt \
candidate.log \
candidate-smoke.txt \
"host-api-${HOST_API_VERSION}.tar.gz" \
host-api-package-check.txt \
image-inspect.txt \
java-security-runtime.txt \
mfa-policy-api-smoke.txt \
registration-basic-api-smoke.txt \
published.txt \
push.log \
release-notes.md \
reverse-proxy-after-restart.txt \
reverse-proxy-before-restart.txt \
sbom-identity.txt \
security-summary.txt \
server.openvex.json \
server.vendor-pending.json \
ubuntu-backports-evidence.tar \
server-security-scan-raw.json \
server-security-scan.json \
server-nested-payloads.json \
server-vulnerabilities-raw.tsv \
server-vulnerabilities-unresolved.tsv \
server.cdx.json \
vendor-pending-summary.txt \
web-console-runtime.cdx.json \
web-console-runtime.cdx.json.sha256 \
web-console-sbom-identity.txt \
web-console-runtime-scan-raw.json \
web-console-runtime-scan-effective.json \
web-console-runtime-scan-coverage.json \
web-console-runtime-vulnerabilities.tsv.gz \
>SHA256SUMS
)
gh release create "$RELEASE_TAG" \
--repo "$GITHUB_REPOSITORY" \
--target "$SOURCE_SHA" \
--title "PastureStack Server $RELEASE_TAG" \
--notes-file "$evidence/release-notes.md" \
"$evidence/SHA256SUMS" \
"$evidence/build.log" \
"$evidence/brotli-runtime.txt" \
"$evidence/candidate.log" \
"$evidence/candidate-smoke.txt" \
"$evidence/host-api-${HOST_API_VERSION}.tar.gz" \
"$evidence/host-api-package-check.txt" \
"$evidence/image-inspect.txt" \
"$evidence/java-security-runtime.txt" \
"$evidence/mfa-policy-api-smoke.txt" \
"$evidence/registration-basic-api-smoke.txt" \
"$evidence/published.txt" \
"$evidence/push.log" \
"$evidence/release-notes.md" \
"$evidence/reverse-proxy-after-restart.txt" \
"$evidence/reverse-proxy-before-restart.txt" \
"$evidence/sbom-identity.txt" \
"$evidence/security-summary.txt" \
"$evidence/server.openvex.json" \
"$evidence/server.vendor-pending.json" \
"$evidence/ubuntu-backports-evidence.tar" \
"$evidence/server-security-scan-raw.json" \
"$evidence/server-security-scan.json" \
"$evidence/server-nested-payloads.json" \
"$evidence/server-vulnerabilities-raw.tsv" \
"$evidence/server-vulnerabilities-unresolved.tsv" \
"$evidence/server.cdx.json" \
"$evidence/vendor-pending-summary.txt" \
"$evidence/web-console-runtime.cdx.json" \
"$evidence/web-console-runtime.cdx.json.sha256" \
"$evidence/web-console-sbom-identity.txt" \
"$evidence/web-console-runtime-scan-raw.json" \
"$evidence/web-console-runtime-scan-effective.json" \
"$evidence/web-console-runtime-scan-coverage.json" \
"$evidence/web-console-runtime-vulnerabilities.tsv.gz"
{
printf 'SHA256SUMS\tsha256:%s\n' \
"$(sha256sum "$evidence/SHA256SUMS" | awk '{print $1}')"
awk '{ print $2 "\tsha256:" $1 }' "$evidence/SHA256SUMS"
} | LC_ALL=C sort >"$evidence/release-assets-expected.tsv"
gh release view "$RELEASE_TAG" \
--repo "$GITHUB_REPOSITORY" \
--json assets \
--jq '.assets[] | [.name, .digest] | @tsv' |
LC_ALL=C sort >"$evidence/release-assets-published.tsv"
diff -u \
"$evidence/release-assets-expected.tsv" \
"$evidence/release-assets-published.tsv"
- name: Retain workflow evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: server-${{ inputs.release_tag }}-publication-${{ github.run_id }}
path: ${{ runner.temp }}/server-evidence
if-no-files-found: warn
retention-days: 30
compression-level: 9
- name: Clean run-owned resources
if: always()
shell: bash
run: |
docker logout ghcr.io >/dev/null 2>&1 || true
docker rm -f "$CANDIDATE_NAME" >/dev/null 2>&1 || true
docker volume rm "$CATTLE_VOLUME" "$MYSQL_VOLUME" "$MYSQL_LOG_VOLUME" \
>/dev/null 2>&1 || true
docker image rm "$TARGET_IMAGE" "$CANDIDATE_IMAGE" "$TRIVY_IMAGE" \
>/dev/null 2>&1 || true
sudo rm -rf \
"$RUNNER_TEMP/server-rootfs" \
"$RUNNER_TEMP/server-rootfs.tar" \
"$RUNNER_TEMP/trivy-cache" \
"$RUNNER_TEMP/server-nested-payloads" \
"$RUNNER_TEMP/server-nested-qualified.json" \
"$RUNNER_TEMP/daemon-current.json" \
"$RUNNER_TEMP/daemon-containerd.json" \
"$RUNNER_TEMP/ping-body"