diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml index 1221f3a80be7a..fe455b70e415c 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml @@ -40,6 +40,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo - dockerfile_literal: | FROM registry.access.redhat.com/ubi9/openjdk-17:1.21 USER root @@ -129,6 +135,7 @@ tests: - ref: acm-fetch-managed-clusters - ref: acm-opp-app - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: interop-tests-opp-quay-smoke - ref: acm-tests-observability zz_generated_metadata: diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml index c3a92c8259122..9df9af594e67c 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml @@ -40,6 +40,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo - dockerfile_literal: | FROM registry.access.redhat.com/ubi9/openjdk-17:1.21 USER root @@ -123,10 +129,11 @@ tests: - chain: cucushift-installer-check-cluster-health - ref: stackrox-opp-readiness - ref: stackrox-opp-smoke - - ref: acm-tests-clc-create + - ref: acm-tests-clc-smoke - ref: acm-fetch-managed-clusters - ref: acm-opp-app - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: interop-tests-opp-quay-smoke - ref: acm-tests-observability - as: interop-opp-vsphere @@ -177,6 +184,7 @@ tests: - ref: acm-policies-openshift-plus - chain: cucushift-installer-check-cluster-health - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: acm-tests-observability - ref: acm-opp-app workflow: acm-ipi-vsphere diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml index e31b7716529a3..df9364f22d947 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml @@ -36,6 +36,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo releases: latest: candidate: @@ -77,6 +83,7 @@ tests: FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + IGNORE_SECONDARY_POLICIES: "true" OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"} @@ -104,10 +111,11 @@ tests: - ref: acm-policies-openshift-plus-setup - ref: acm-policies-openshift-plus - chain: cucushift-installer-check-cluster-health - - ref: acm-tests-clc-create + - ref: acm-tests-clc-smoke - ref: acm-fetch-managed-clusters - ref: acm-opp-app - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: interop-tests-opp-quay-smoke - ref: acm-tests-observability - as: interop-opp-vsphere @@ -135,6 +143,7 @@ tests: FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + IGNORE_SECONDARY_POLICIES: "true" OPENSHIFT_REQUIRED_CORES: "72" OPENSHIFT_REQUIRED_MEMORY: "288" OPERATORS: | @@ -156,6 +165,7 @@ tests: - ref: acm-policies-openshift-plus - chain: cucushift-installer-check-cluster-health - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: acm-tests-observability - ref: acm-opp-app workflow: acm-ipi-vsphere diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml index 8598841060b45..8fb2da67b580c 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml @@ -36,6 +36,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo releases: latest: candidate: diff --git a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh index 0f847d68d0cec..f04cd58e50fb6 100644 --- a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh +++ b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh @@ -18,9 +18,15 @@ if [[ -n "${QUAY_OPERATOR_CHANNEL}" ]]; then fi echo 'y' | ./deploy.sh -p policygenerator/policy-sets/stable/openshift-plus -n policies -u https://github.com/stolostron/policy-collection.git -a openshift-plus +# openshift-plus generates ~25 policies; require 4+ before oc wait to avoid +# racing the GitOps Subscription propagation (stolostron/policy-collection#174) +typeset -i expectedMinPolicies=4 typeset -i pollDeadline=$((SECONDS + 600)) -until (($(oc get policies -n policies -o name 2>/dev/null | wc -l))); do - ((SECONDS > pollDeadline)) && { : "Error: no policies appeared after 10 minutes"; exit 1; } +until (( $(oc get policies -n policies -o name 2>/dev/null | wc -l) >= expectedMinPolicies )); do + ((SECONDS > pollDeadline)) && { + printf '%s\n' "Error: fewer than ${expectedMinPolicies} policies after 10 minutes" >&2 + exit 1 + } sleep 5 done @@ -29,7 +35,7 @@ typeset -a quayNamespacesArr=(quay openshift-quay quay-enterprise) typeset quayFound=false for ns in "${quayNamespacesArr[@]}"; do if (($(oc get quayregistry -n "${ns}" -o name 2>/dev/null | wc -l))); then - : "Found Quay Operator deployment in namespace ${ns}, waiting for ready condition" + echo "Found Quay Operator deployment in namespace ${ns}, waiting for ready condition" oc wait quayregistry --all -n "${ns}" \ --for condition=Available=True \ --timeout=10m || true @@ -37,12 +43,27 @@ for ns in "${quayNamespacesArr[@]}"; do break fi done -[[ "${quayFound}" == "false" ]] && : "Warning: no QuayRegistry found in namespaces: ${quayNamespacesArr[*]}" +[[ "${quayFound}" == "false" ]] && echo "Warning: no QuayRegistry found in namespaces: ${quayNamespacesArr[*]}" >&2 typeset -a secondaryPoliciesArr=( policy-acs + policy-acs-monitor-certs + policy-acs-operator-central + policy-acs-sync-resources + policy-advanced-managed-cluster-security policy-advanced-managed-cluster-status + policy-compliance-operator-install + policy-config-quay policy-hub-quay-bridge + policy-install-quay + policy-observability-operator + policy-observability-storage + policy-observability-storage-status + policy-odf + policy-odf-cluster + policy-odf-noobaa + policy-odf-status + policy-quay-bridge policy-quay-status ) @@ -57,12 +78,12 @@ if [[ "${IGNORE_SECONDARY_POLICIES}" == "true" ]]; then --for jsonpath='{.status.compliant}'=Compliant \ --timeout=40m } || { - : "Critical policies failed to become compliant:" + echo "ERROR: Critical policies failed to become compliant:" >&2 oc get policies -n policies | grep -Ev "$(IFS='|'; echo "${secondaryPoliciesArr[*]}")" || true exit 1 } else - : "All policies are secondary (ignored), no critical policies to wait for" + echo "All policies are secondary (ignored), no critical policies to wait for" fi else { @@ -70,7 +91,7 @@ else --for jsonpath='{.status.compliant}'=Compliant \ --timeout=40m } || { - : "Policies failed to become compliant:" + echo "ERROR: Policies failed to become compliant:" >&2 oc get policies -n policies exit 1 } diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/OWNERS b/ci-operator/step-registry/acm/tests/clc-smoke/OWNERS new file mode 100644 index 0000000000000..76364ea3076e7 --- /dev/null +++ b/ci-operator/step-registry/acm/tests/clc-smoke/OWNERS @@ -0,0 +1,9 @@ +approvers: +- cspi-qe-ocp-lp +- dtthuynh +- vboulos +options: {} +reviewers: +- cspi-qe-ocp-lp +- dtthuynh +- vboulos diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/README.md b/ci-operator/step-registry/acm/tests/clc-smoke/README.md new file mode 100644 index 0000000000000..8819c7ae40c75 --- /dev/null +++ b/ci-operator/step-registry/acm/tests/clc-smoke/README.md @@ -0,0 +1,38 @@ +# acm-tests-clc-smoke-ref + +## Table of Contents +- [Purpose](#purpose) +- [Process](#process) +- [Requirements](#requirements) + - [Infrastructure](#infrastructure) + - [Environment Variables](#environment-variables) + +## Purpose + +Smoke-scoped variant of [acm-tests-clc-create](../clc-create/README.md) with a right-sized timeout and strict failure handling for OPP interop. + +The full `acm-tests-clc-create` step already creates only 1 AWS managed cluster (~50 min actual runtime) but carries a 28800s (8h) timeout and suppresses failures with `|| :`. This step: +- Reduces the timeout to 5400s (90 min), giving ~80% headroom over the observed average. +- Propagates failures so downstream steps (`acm-fetch-managed-clusters`, `acm-opp-app`) fail fast instead of running against a missing cluster. + +> **IMPORTANT** +> You must use the [acm-tests-clc-destroy-ref](../clc-destroy/README.md) as a post step when using this step. If you do not and succeed in running this step then you will leave clusters running on the ACM QE team's cloud. + +## Process + +- Copies secret options file needed for test execution. +- Injects AWS credentials from the cluster profile into options.yaml. +- Sets dynamic variables based on the provisioned hub cluster. +- Runs `execute_clc_interop_commands.sh` which invokes Cypress with tag filter `@create+aws+-sno+-@clusterpool` (controlled by `TEST_STAGE=OCPInterop-create` inside the image). + +## Requirements + +### Infrastructure + +- An existing OpenShift cluster to act as the target Hub. +- "advanced-cluster-management" operator installed (see [`install-operators`](../../../install-operators/README.md)). +- MCH custom resource installed (see [acm-mch step](../mch/README.md)). + +### Environment Variables + +- Please see [acm-tests-clc-smoke-ref.yaml](acm-tests-clc-smoke-ref.yaml) env section. diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh new file mode 100755 index 0000000000000..83661193bf2ac --- /dev/null +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -euxo pipefail; shopt -s inherit_errexit + +typeset secretsDir="/tmp/secrets" +typeset optionFile="./options.yaml" +typeset awsCredFile="${CLUSTER_PROFILE_DIR}/.awscred" + +if [[ "${SKIP_OCP_DEPLOY:-false}" == "true" ]]; then + cp "${secretsDir}/ci/kubeconfig" "${SHARED_DIR}/kubeconfig" + cp "${secretsDir}/ci/kubeadmin-password" "${SHARED_DIR}/kubeadmin-password" +fi + +cp "${secretsDir}/clc-interop/secret-options-yaml" "${optionFile}" + +if [[ -f "${awsCredFile}" ]]; then + typeset awsAccKeyID= + typeset awsAccKeyToken= + + set +x + awsAccKeyID="$(sed -nE 's/^\s*aws_access_key_id\s*=\s*//p;T;q' "${awsCredFile}")" + awsAccKeyToken="$(sed -nE 's/^\s*aws_secret_access_key\s*=\s*//p;T;q' "${awsCredFile}")" + + if [[ -z "${awsAccKeyID}" ]] || [[ -z "${awsAccKeyToken}" ]]; then + echo "ERROR: Failed to extract AWS credentials from ${awsCredFile}" 1>&2 + exit 1 + fi + + yq -o json eval . "${optionFile}" | + jq -c \ + --arg awsAccKeyID "${awsAccKeyID}" \ + --rawfile awsAccKeyToken <(printf '%s' "${awsAccKeyToken}") \ + ' + .options.connections.apiKeys.aws|=( + .awsAccessKeyID=$awsAccKeyID | + .awsSecretAccessKeyID=($awsAccKeyToken | rtrimstr("\n")) + ) + ' | + yq -p json -o yaml eval . > "${optionFile}.tmp" + mv -f "${optionFile}.tmp" "${optionFile}" + set -x + + unset awsAccKeyID awsAccKeyToken +fi + +set +x +export CYPRESS_OPTIONS_HUB_PASSWORD= +CYPRESS_OPTIONS_HUB_PASSWORD="$(cat "${SHARED_DIR}/kubeadmin-password")" + +typeset clcStatus=0 + +CYPRESS_BASE_URL="$(oc whoami --show-console)" \ +CYPRESS_HUB_API_URL="$(oc whoami --show-server)" \ +CYPRESS_CLC_OCP_IMAGE_VERSION="$(cat "${secretsDir}/clc/ocp_image_version")" \ +CLOUD_PROVIDERS="$(cat "${secretsDir}/clc/ocp_cloud_providers")" \ +bash +x ./execute_clc_interop_commands.sh || clcStatus=$? +set -x + +unset CYPRESS_OPTIONS_HUB_PASSWORD + +cp -r reports "${ARTIFACT_DIR}/" +exit "${clcStatus}" diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json new file mode 100644 index 0000000000000..bfcc856a27d44 --- /dev/null +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp", + "dtthuynh", + "vboulos" + ], + "reviewers": [ + "cspi-qe-ocp-lp", + "dtthuynh", + "vboulos" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml new file mode 100644 index 0000000000000..775b3cda884ae --- /dev/null +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml @@ -0,0 +1,74 @@ +ref: + as: acm-tests-clc-smoke + from: clc-ui-e2e + commands: acm-tests-clc-smoke-commands.sh + timeout: 5400s + best_effort: true + resources: + requests: + cpu: '2' + memory: 6Gi + credentials: + - namespace: test-credentials + name: opp-acm-clc-credentials + mount_path: /tmp/secrets/clc-interop + - namespace: test-credentials + name: acm-clc-credentials + mount_path: /tmp/secrets/clc + - namespace: test-credentials + name: acm-ci-credentials + mount_path: /tmp/secrets/ci + env: + - name: CYPRESS_OC_IDP + default: "kube:admin" + documentation: |- + Identity + - name: CYPRESS_OPTIONS_HUB_USER + default: "kubeadmin" + documentation: |- + Hub cluster username + - name: CYPRESS_SPOKE_CLUSTER + default: "" + documentation: |- + Identify spoke clusters + - name: BROWSER + default: "chrome" + documentation: |- + Set browser for cypress + - name: CUSTOMER_TAGS + default: "" + documentation: |- + Cypress grep tag filter (passed through to test execution) + - name: CYPRESS_CLC_OC_IDP + default: "clc-e2e-htpasswd" + documentation: |- + Serves tests RBAC settings + - name: CYPRESS_CLC_RBAC_PASS + default: "test-RBAC-4-e2e" + documentation: |- + Serves tests RBAC settings + - name: CYPRESS_CLC_OCP_IMAGE_REGISTRY + default: "quay.io/openshift-release-dev/ocp-release" + documentation: |- + Image registry + - name: CYPRESS_ACM_NAMESPACE + default: "ocm" + documentation: |- + Acm namespace + - name: CYPRESS_MCE_NAMESPACE + default: "multicluster-engine" + documentation: |- + Mce namespace + - name: IMPORT_KUBERNETES_CLUSTERS + default: "" + documentation: |- + Comma separated list of imports + - name: SKIP_OCP_DEPLOY + default: "false" + documentation: |- + When true, copies kubeconfig from CI secrets instead of using cluster profile + documentation: |- + Smoke-scoped ACM cluster lifecycle step that creates a single managed + cluster on AWS (~50 min). Runs with best_effort so that CLC failures + do not block independent downstream validations (ODF health, Quay smoke, + observability). JUnit results are still reported for failure visibility. diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index f726090036915..eaf164b557e59 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -1,18 +1,19 @@ #!/bin/bash -set -euo pipefail +set -eux -o pipefail shopt -s inherit_errexit ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}" mkdir -p "${ARTIFACT_DIR}" typeset junitFile="${ARTIFACT_DIR}/junit_quay_interop.xml" -typeset imageTag="${BUILD_ID:-$(date +%s)}" +typeset imageTag='' +imageTag="${BUILD_ID:-$(date +%s)}" typeset -A testStatus typeset -A testDuration typeset -A testFailureMsg typeset -a allTests=( "[sig-interop][Jira:INTEROP][Feature:Quay] Push and pull image via Quay route" - "[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF PVC backing Quay storage" + "[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF object storage integration" "[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" ) @@ -33,13 +34,15 @@ function RecordResult () { testStatus["${name}"]="${status}" testDuration["${name}"]="${dur}" testFailureMsg["${name}"]="${msg}" + true } # shellcheck disable=SC2329 function GenerateJunit () { typeset -i total=${#allTests[@]} typeset -i failures=0 skipped=0 - typeset -i elapsed=$(( $(date +%s) - suiteStart )) + typeset -i elapsed=0 + elapsed=$(( $(date +%s) - suiteStart )) for t in "${allTests[@]}"; do [[ "${testStatus[${t}]}" == "failed" ]] && failures=$((failures + 1)) @@ -53,94 +56,135 @@ function GenerateJunit () { EOF for t in "${allTests[@]}"; do - typeset escaped_name - escaped_name=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') - typeset escaped_msg - escaped_msg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escapedName + escapedName=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escapedMsg + escapedMsg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') if [[ "${testStatus[${t}]}" == "failed" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" elif [[ "${testStatus[${t}]}" == "skipped" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" else - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" fi done - cat >> "${junitFile}" <> "${junitFile}" <<'EOF' EOF cat "${junitFile}" + true } -trap GenerateJunit EXIT +trap '{ ( GenerateJunit; true ); }' EXIT function DiscoverQuay () { QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}') QUAY_REGISTRY=$(oc get quayregistry -n "${QUAY_NS}" -o jsonpath='{.items[0].metadata.name}') QUAY_HOST=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.status.registryEndpoint}') QUAY_HOST="${QUAY_HOST#https://}" + if [[ -z "${QUAY_HOST}" ]]; then + echo "ERROR: Quay registry route not ready (empty host)" >&2 + return 1 + fi export QUAY_NS QUAY_REGISTRY QUAY_HOST + true } function GetQuayAuth () { - typeset configSecret - configSecret=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.spec.configBundleSecret}') - if [[ -z "${configSecret}" ]]; then - configSecret="${QUAY_REGISTRY}-config-bundle" - fi - - QUAY_USER=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_EMAIL}' 2>/dev/null | base64 -d || echo "") - if [[ -z "${QUAY_USER}" ]]; then + QUAY_USER="" + QUAY_PASSWORD="" + QUAY_TOKEN="" + + set +x + if oc get secret quayadmin -n "${QUAY_NS}" 2>/dev/null; then + QUAY_TOKEN=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_TOKEN="" + QUAY_PASSWORD=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" QUAY_USER="quayadmin" + if [[ -n "${QUAY_TOKEN}" || -n "${QUAY_PASSWORD}" ]]; then + set -x + echo "INFO: Quay credentials obtained from quayadmin secret" + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 0 + fi fi - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_PASSWORD}' 2>/dev/null | base64 -d || echo "") - if [[ -z "${QUAY_PASSWORD}" ]]; then - typeset initSecret="${QUAY_REGISTRY}-init-config-bundle-secret" - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${initSecret}" -o jsonpath='{.data.superuser-password}' 2>/dev/null | base64 -d || echo "") + if oc get secret quaydevel -n "${QUAY_NS}" 2>/dev/null; then + QUAY_PASSWORD=$(oc get secret quaydevel -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" + QUAY_USER="quaydevel" + if [[ -n "${QUAY_PASSWORD}" ]]; then + set -x + echo "INFO: Quay credentials obtained from quaydevel secret" + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 0 + fi fi - if [[ -z "${QUAY_PASSWORD}" ]]; then - for secret in $(oc get secrets -n "${QUAY_NS}" -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep -i "quay.*config"); do - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${secret}" -o go-template='{{index .data "config.yaml"}}' 2>/dev/null | base64 -d | grep -oP "(?<=SUPER_USER_PASSWORD: ).*" || echo "") - [[ -n "${QUAY_PASSWORD}" ]] && break - done + typeset initPassword + initPassword=$(python3 -c "import secrets,string; print(''.join(secrets.choice(string.ascii_letters+string.digits) for _ in range(20)))") + typeset initResult + initResult=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/user/initialize" \ + -H "Content-Type: application/json" \ + -d "{\"username\":\"quayadmin\",\"password\":\"${initPassword}\",\"email\":\"quayadmin@example.com\",\"access_token\":true}" 2>/dev/null) || initResult="" + + QUAY_TOKEN=$(echo "${initResult}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('access_token',''))" 2>/dev/null) || QUAY_TOKEN="" + if [[ -n "${QUAY_TOKEN}" ]]; then + QUAY_USER="quayadmin" + QUAY_PASSWORD="${initPassword}" + set -x + echo "INFO: Quay admin user initialized via /api/v1/user/initialize" + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 0 fi + set -x - export QUAY_USER QUAY_PASSWORD + echo "ERROR: Could not obtain Quay credentials from any source" >&2 + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 1 } function PreflightCheck () { if ! curl -sk --connect-timeout 15 "https://${QUAY_HOST}/api/v1/discovery" | grep -qi "quay"; then - echo "ERROR: Quay route not reachable at ${QUAY_HOST}" >&2 + echo "ERROR: Quay registry endpoint not reachable" >&2 return 1 fi + true } function CreateTestOrg () { - typeset signinPayload - signinPayload=$(python3 -c "import json,sys; print(json.dumps({'user':sys.argv[1],'pass':sys.argv[2]}))" "${QUAY_USER}" "${QUAY_PASSWORD}") - typeset token - token=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \ - -H "Content-Type: application/json" \ - -d "${signinPayload}" | \ - python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || echo "") - - if [[ -z "${token}" ]]; then - token=$(curl -sk -H "Authorization: Basic $(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64)" \ - "https://${QUAY_HOST}/api/v1/user/" | \ - python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null || echo "") + if [[ -z "${QUAY_TOKEN}" && -n "${QUAY_PASSWORD}" ]]; then + typeset cookieFile="/tmp/quay-cookies.txt" + typeset csrf + csrf=$(curl -sk "https://${QUAY_HOST}/csrf_token" -c "${cookieFile}" | \ + python3 -c "import sys,json; print(json.load(sys.stdin).get('csrf_token',''))" 2>/dev/null) || csrf="" + + if [[ -n "${csrf}" ]]; then + typeset signinResult + set +x + signinResult=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \ + -H "Content-Type: application/json" \ + -H "X-CSRF-Token: ${csrf}" \ + -b "${cookieFile}" -c "${cookieFile}" \ + -d "{\"username\":\"${QUAY_USER}\",\"password\":\"${QUAY_PASSWORD}\"}" 2>/dev/null) || signinResult="" + QUAY_TOKEN=$(echo "${signinResult}" | \ + python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null) || QUAY_TOKEN="" + set -x + fi + rm -f "${cookieFile}" + export QUAY_TOKEN fi - QUAY_TOKEN="${token}" - export QUAY_TOKEN + if [[ -z "${QUAY_TOKEN}" ]]; then + echo "WARNING: No Quay token available; org creation may fail" >&2 + fi curl -sk -X POST "https://${QUAY_HOST}/api/v1/organization/" \ -H "Authorization: Bearer ${QUAY_TOKEN}" \ -H "Content-Type: application/json" \ -d '{"name":"interop-smoke-test","email":"interop-test@example.com"}' || true + true } ################################################################################ @@ -154,9 +198,24 @@ function RunPushPull () { typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" typeset authFile="/tmp/quay-auth.json" + if [[ -z "${QUAY_TOKEN}" && -z "${QUAY_PASSWORD}" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "No valid Quay authentication token or password available" "${elapsed}" + return 1 + fi + + typeset registryAuth + set +x + if [[ -n "${QUAY_TOKEN}" ]]; then + registryAuth=$(echo -n "\$oauthtoken:${QUAY_TOKEN}" | base64) + else + registryAuth=$(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64) + fi + cat > "${authFile}" </dev/null 2>&1; then + "docker://${pushTarget}"; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "Image not pullable from Quay after push" "${elapsed}" return 1 @@ -181,64 +240,56 @@ EOF } ################################################################################ -# Test Case 2: Verify ODF PVC backing Quay storage +# Test Case 2: Verify ODF object storage integration ################################################################################ -function RunOdfPvcCheck () { - typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF PVC backing Quay storage" +function RunOdfStorageCheck () { + typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF object storage integration" typeset -i start elapsed start=$(date +%s) - typeset pvcCount - pvcCount=$(oc get pvc -n "${QUAY_NS}" -l app=quay -o json 2>/dev/null | python3 -c " -import sys, json -data = json.load(sys.stdin) -items = data.get('items', []) -print(len(items)) -" 2>/dev/null || echo "0") + typeset noobaaPhase + noobaaPhase=$(oc get noobaa -n openshift-storage -o jsonpath='{.items[0].status.phase}' 2>/dev/null) || noobaaPhase="" + if [[ "${noobaaPhase}" != "Ready" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "NooBaa not Ready (phase: ${noobaaPhase:-not found})" "${elapsed}" + return 1 + fi - if [[ "${pvcCount}" == "0" ]]; then - pvcCount=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " -import sys, json -data = json.load(sys.stdin) -items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] -print(len(items)) -" 2>/dev/null || echo "0") + typeset obcCount + obcCount=$(oc get objectbucketclaim -n openshift-storage -o json 2>/dev/null | \ + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obcCount="0" + if [[ "${obcCount}" == "0" ]]; then + obcCount=$(oc get objectbucketclaim --all-namespaces -o json 2>/dev/null | \ + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obcCount="0" fi - if [[ "${pvcCount}" == "0" ]]; then + if [[ "${obcCount}" == "0" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "No Quay-related PVCs found in ${QUAY_NS}" "${elapsed}" + RecordResult "${testName}" "failed" "No ObjectBucketClaims found" "${elapsed}" return 1 fi - typeset unboundPvcs - unboundPvcs=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " -import sys, json -data = json.load(sys.stdin) -items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] -unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound'] -print(' '.join(unbound)) -" 2>/dev/null || echo "") - - if [[ -n "${unboundPvcs}" ]]; then + typeset obCount + obCount=$(oc get objectbucket -o json 2>/dev/null | \ + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obCount="0" + if [[ "${obCount}" == "0" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "Unbound PVCs: ${unboundPvcs}" "${elapsed}" + RecordResult "${testName}" "failed" "No ObjectBucket resources found for OBCs" "${elapsed}" return 1 fi - typeset odfBacked - odfBacked=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " + typeset unboundPvcs + unboundPvcs=$(oc get pvc -n "${QUAY_NS}" -o json 2>/dev/null | python3 -c " import sys, json data = json.load(sys.stdin) items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] -sc_names = set(i['spec'].get('storageClassName','') for i in items) -odf = any('ocs' in s or 'ceph' in s or 'odf' in s for s in sc_names) -print('true' if odf else 'false') -" 2>/dev/null || echo "false") +unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound'] +print(' '.join(unbound)) +" 2>/dev/null) || unboundPvcs="" - if [[ "${odfBacked}" != "true" ]]; then + if [[ -n "${unboundPvcs}" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "Quay PVCs not using ODF/Ceph storage class" "${elapsed}" + RecordResult "${testName}" "failed" "Unbound Quay PVCs: ${unboundPvcs}" "${elapsed}" return 1 fi @@ -250,20 +301,103 @@ print('true' if odf else 'false') ################################################################################ # Test Case 3: ACS scan of pushed Quay image ################################################################################ +function RegisterQuayInAcs () { + typeset acsHost="${1}" acsPassword="${2}" + set +x + + typeset existing + existing=$(curl -sk -u "admin:${acsPassword}" \ + "https://${acsHost}/v1/imageintegrations" 2>/dev/null | \ + python3 -c " +import sys, json, os +host = os.environ['QUAY_HOST'] +data = json.load(sys.stdin) +for i in data.get('integrations', []): + if host in i.get('docker', {}).get('endpoint', ''): + print(i['id']) + sys.exit(0) +sys.exit(1) +" 2>/dev/null) || existing="" + + if [[ -n "${existing}" ]]; then + echo "INFO: Quay integration already registered in ACS" + return 0 + fi + + typeset regUser regPass + if [[ -n "${QUAY_TOKEN}" ]]; then + regUser="\$oauthtoken" + regPass="${QUAY_TOKEN}" + else + regUser="${QUAY_USER}" + regPass="${QUAY_PASSWORD}" + fi + + typeset regPayload='' + regPayload=$(python3 -c " +import json, sys, os +payload = { + 'name': 'interop-quay-smoke', + 'type': 'docker', + 'categories': ['REGISTRY'], + 'docker': { + 'endpoint': os.environ['QUAY_HOST'], + 'username': sys.argv[1], + 'password': sys.argv[2], + 'insecure': True + }, + 'skipTestIntegration': True +} +print(json.dumps(payload)) +" "${regUser}" "${regPass}") + + curl -sk -X POST "https://${acsHost}/v1/imageintegrations" \ + -u "admin:${acsPassword}" \ + -H "Content-Type: application/json" \ + -d "${regPayload}" || true + set -x + + echo "INFO: Registered Quay registry endpoint as ACS image integration" + true +} + +function RequestAcsScan () { + typeset acsHost="${1}" acsPassword="${2}" imageName="${3}" + set +x + + typeset scanPayload='' + scanPayload=$(python3 -c " +import json, sys +payload = {'imageName': sys.argv[1], 'force': True} +print(json.dumps(payload)) +" "${imageName}") + + curl -sk -X POST "https://${acsHost}/v1/images/scan" \ + -u "admin:${acsPassword}" \ + -H "Content-Type: application/json" \ + -d "${scanPayload}" || true + set -x + + echo "INFO: Requested ACS scan of ${imageName}" + true +} + function RunAcsScan () { typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" typeset -i start elapsed start=$(date +%s) typeset acsHost acsPassword - acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null || echo "") + acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null) || acsHost="" if [[ -z "${acsHost}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS Central route not found" "${elapsed}" return 1 fi - acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "") + set +x + acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || acsPassword="" + set -x if [[ -z "${acsPassword}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS admin password not found" "${elapsed}" @@ -271,12 +405,20 @@ function RunAcsScan () { fi typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" - typeset -i attempts=0 maxAttempts=20 + + set +x + RegisterQuayInAcs "${acsHost}" "${acsPassword}" + RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" + set -x + + typeset -i attempts=0 maxAttempts=40 while (( attempts < maxAttempts )); do typeset scanResult + set +x scanResult=$(curl -sk -u "admin:${acsPassword}" \ - "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null || echo "") + "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null) || scanResult="" + set -x if echo "${scanResult}" | python3 -c " import sys, json @@ -289,12 +431,18 @@ sys.exit(0 if len(images) > 0 else 1) return 0 fi + if (( attempts % 4 == 3 )); then + set +x + RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" + set -x + fi + attempts=$((attempts + 1)) sleep 15 done elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "ACS did not detect pushed image within 5 minutes" "${elapsed}" + RecordResult "${testName}" "failed" "ACS did not detect pushed image within 10 minutes" "${elapsed}" return 1 } @@ -308,10 +456,14 @@ function Main () { PreflightCheck || { echo "FATAL: Quay not reachable; skipping all tests" >&2; exit 1; } CreateTestOrg - typeset -i status=0 - RunPushPull || status=1 - RunOdfPvcCheck || status=1 - RunAcsScan || status=1 + typeset -i status=0 pushPassed=0 + RunPushPull && pushPassed=1 || status=1 + RunOdfStorageCheck || status=1 + if (( pushPassed )); then + RunAcsScan || status=1 + else + RecordResult "[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" "skipped" "Skipped: push-pull test failed; no image available to scan" + fi rm -f /tmp/quay-auth.json @@ -322,7 +474,7 @@ function Main () { "${_fURL[@]}" \ https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/ci-operator/interop/common/ExitTrap--PostProcessPrep.sh )" || true - if type -t ExitTrap--PostProcessPrep 1>/dev/null; then + if type -t ExitTrap--PostProcessPrep; then LP_IO__ET_PPP__NEW_TS_NAME="${DR__RP__CR_COMP_NAME}--%s" \ ExitTrap--PostProcessPrep || true fi diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml index eb2e58149f440..5b0caae06170a 100644 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml @@ -1,6 +1,6 @@ ref: as: interop-tests-opp-quay-smoke - from: cli + from: cli-with-skopeo cli: latest commands: interop-tests-opp-quay-smoke-commands.sh timeout: 30m0s @@ -11,8 +11,9 @@ ref: memory: 256Mi documentation: |- Validates Quay as a cross-product registry within the OPP bundle. - Tests image push/pull via the Quay route, verifies ODF-backed PVC storage, - and confirms ACS detects and scans the pushed image. + Tests image push/pull via the Quay route, verifies ODF object storage + integration (OBCs and NooBaa health), and confirms ACS detects and scans + the pushed image. env: - name: DR__RP__CR_COMP_NAME default: "lp-interop--Quay" diff --git a/ci-operator/step-registry/interop/opp/observability-odf/OWNERS b/ci-operator/step-registry/interop/opp/observability-odf/OWNERS new file mode 100644 index 0000000000000..41d144d3728a2 --- /dev/null +++ b/ci-operator/step-registry/interop/opp/observability-odf/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh new file mode 100755 index 0000000000000..0d7fcf9504446 --- /dev/null +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -0,0 +1,629 @@ +#!/bin/bash +set -euo pipefail; shopt -s inherit_errexit + +# --------------------------------------------------------------------------- +# ACM Observability + ODF Interop Validation (6-point gate) +# +# Validates that ACM's observability stack (Thanos) correctly uses +# ODF-provided object storage (Ceph RGW or NooBaa S3) as its backend. +# This is a cross-product interop test exercising the ACM <-> ODF boundary. +# +# Produces JUnit XML consumed by Prow / Sippy / TestGrid. +# --------------------------------------------------------------------------- + +typeset ACM_NAMESPACE="${ACM_NAMESPACE:-open-cluster-management}" +typeset OBS_NAMESPACE="${OBS_NAMESPACE:-open-cluster-management-observability}" +typeset ODF_NAMESPACE="${ODF_NAMESPACE:-openshift-storage}" + +typeset junitFile="${ARTIFACT_DIR}/junit_observability_odf.xml" + +typeset -a tcNamesArr=() +typeset -a tcResultsArr=() +typeset -a tcMessagesArr=() + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +function AddResult () { + typeset name="${1:-}"; (($#)) && shift + typeset result="${1:-}"; (($#)) && shift + typeset message="${1:-}"; (($#)) && shift + tcNamesArr+=("${name}") + tcResultsArr+=("${result}") + tcMessagesArr+=("${message}") + true +} + +function XmlEscape () { + typeset text="${1:-}"; (($#)) && shift + text="${text//&/&}" + text="${text///>}" + text="${text//\"/"}" + text="${text//\'/'}" + printf '%s' "${text}" + true +} + +function WriteJunit () { + typeset -i total=${#tcNamesArr[@]} + typeset -i failCount=0 + typeset -i skipCount=0 + typeset r="" + for r in "${tcResultsArr[@]}"; do + if [[ "${r}" == "fail" ]]; then + (( ++failCount )) + elif [[ "${r}" == "skip" ]]; then + (( ++skipCount )) + fi + done + + { + echo '' + echo "" + typeset -i i=0 + for i in "${!tcNamesArr[@]}"; do + typeset name="" + name="$(XmlEscape "${tcNamesArr[$i]}")" + echo " " + if [[ "${tcResultsArr[$i]}" == "fail" ]]; then + typeset msg="" + msg="$(XmlEscape "${tcMessagesArr[$i]}")" + echo " " + elif [[ "${tcResultsArr[$i]}" == "skip" ]]; then + typeset msg="" + msg="$(XmlEscape "${tcMessagesArr[$i]}")" + echo " " + fi + echo " " + done + echo "" + } > "${junitFile}" + : "JUnit XML written to ${junitFile}" + true +} + +# shellcheck disable=SC2317,SC2329 +function CollectExitArtifacts () { + : "Collecting observability + ODF diagnostics..." + oc get multiclusterobservabilities.observability.open-cluster-management.io --all-namespaces -o yaml > "${ARTIFACT_DIR}/mco.yaml" || true + oc get pods -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-pods.yaml" || true + oc get obc -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-obc.yaml" || true + oc get secret -n "${OBS_NAMESPACE}" -o name > "${ARTIFACT_DIR}/obs-secrets-list.txt" || true + oc get cephobjectstore -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/cephobjectstore.yaml" || true + oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw -o yaml > "${ARTIFACT_DIR}/rgw-pods.yaml" || true + oc get noobaa -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/noobaa.yaml" || true + true +} + +trap '{( CollectExitArtifacts; true )}' EXIT + +# --------------------------------------------------------------------------- +# Check 1: ODF Ceph RGW infrastructure ready +# --------------------------------------------------------------------------- + +function CheckRgwReady () { + : "=== Check 1: ODF Ceph RGW infrastructure ===" + + typeset rgwPhase="" + if ! rgwPhase="$(oc get cephobjectstore -n "${ODF_NAMESPACE}" -o json | python3 -c " +import sys,json +d=json.load(sys.stdin) +items=d.get('items',[]) +if not items: + print('NotFound') +else: + print(items[0].get('status',{}).get('phase','Unknown')) +")"; then + AddResult "odf-storage-ready" "fail" "Failed to query CephObjectStore" + return + fi + + if [[ "${rgwPhase}" == "NotFound" ]]; then + typeset noobaaJson="" + noobaaJson="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json)" || true + typeset noobaaPhase="" + if [[ -n "${noobaaJson}" ]]; then + noobaaPhase="$(printf '%s' "${noobaaJson}" | python3 -c " +import sys,json +items=json.load(sys.stdin).get('items',[]) +print(items[0].get('status',{}).get('phase','') if items else '') +")" + fi + if [[ "${noobaaPhase}" == "Ready" ]]; then + AddResult "odf-storage-ready" "pass" "NooBaa Ready (RGW not deployed)" + elif [[ -n "${noobaaPhase}" ]]; then + AddResult "odf-storage-ready" "fail" "NooBaa phase=${noobaaPhase} (expected Ready); RGW not deployed" + else + AddResult "odf-storage-ready" "skip" "Neither CephObjectStore nor NooBaa found in ${ODF_NAMESPACE}" + fi + return + fi + + typeset failMsg="" + if [[ "${rgwPhase}" != "Ready" ]]; then + failMsg="CephObjectStore phase=${rgwPhase} (expected Ready)" + fi + + typeset rgwPods="" + rgwPods="$(oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw \ + --field-selector=status.phase=Running --no-headers)" || true + typeset rgwPodCount="" + rgwPodCount="$(printf '%s' "${rgwPods}" | awk 'END{print NR}')" + + if [[ "${rgwPodCount}" -eq 0 ]]; then + typeset rgwMsg="No rook-ceph-rgw pods Running in ${ODF_NAMESPACE}" + if [[ -n "${failMsg}" ]]; then + failMsg="${failMsg}; ${rgwMsg}" + else + failMsg="${rgwMsg}" + fi + fi + + typeset scExists="" + scExists="$(oc get sc ocs-storagecluster-ceph-rgw -o name)" || true + if [[ -z "${scExists}" ]]; then + typeset scMsg="StorageClass ocs-storagecluster-ceph-rgw not found" + if [[ -n "${failMsg}" ]]; then + failMsg="${failMsg}; ${scMsg}" + else + failMsg="${scMsg}" + fi + fi + + if [[ -z "${failMsg}" ]]; then + : "PASS: CephObjectStore Ready, RGW pods Running, StorageClass exists" + AddResult "odf-storage-ready" "pass" + else + AddResult "odf-storage-ready" "fail" "${failMsg}" + fi + true +} + +# --------------------------------------------------------------------------- +# Check 2: MultiClusterObservability CR exists and is Ready +# --------------------------------------------------------------------------- + +function CheckMcoReady () { + : "=== Check 2: MultiClusterObservability CR ===" + + typeset mcoStatus="" + if ! mcoStatus="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ + --all-namespaces -o json | python3 -c " +import sys,json +d=json.load(sys.stdin) +items=d.get('items',[]) +if not items: + print('NotFound') +else: + conds=items[0].get('status',{}).get('conditions',[]) + ready=[c for c in conds if c.get('type')=='Ready'] + print(ready[0].get('status','Unknown') if ready else 'NoCondition') +")"; then + AddResult "mco-ready" "fail" "Failed to query MultiClusterObservability CR" + return + fi + + if [[ "${mcoStatus}" == "True" ]]; then + : "PASS: MultiClusterObservability Ready=True" + AddResult "mco-ready" "pass" + elif [[ "${mcoStatus}" == "NotFound" ]]; then + AddResult "mco-ready" "skip" "MultiClusterObservability CR not found; observability not deployed" + else + AddResult "mco-ready" "fail" "MultiClusterObservability Ready=${mcoStatus} (expected True)" + fi + true +} + +# --------------------------------------------------------------------------- +# Check 3: Object storage secret references ODF-backed endpoint +# --------------------------------------------------------------------------- + +function CheckStorageEndpoint () { + : "=== Check 3: Object storage endpoint ===" + + typeset storageConfig="" + if ! storageConfig="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ + --all-namespaces -o json | python3 -c " +import sys,json +d=json.load(sys.stdin) +items=d.get('items',[]) +if not items: + print('') +else: + spec=items[0].get('spec',{}) + storage=spec.get('storageConfig',{}).get('metricObjectStorage',{}) + name=storage.get('name','') + key=storage.get('key','thanos.yaml') + print(f'{name}|{key}' if name else '') +")"; then + AddResult "storage-endpoint" "fail" "Failed to read MCO storage config" + return + fi + + if [[ -z "${storageConfig}" ]]; then + AddResult "storage-endpoint" "skip" "No metricObjectStorage secret configured in MCO" + return + fi + + typeset secretName="${storageConfig%%|*}" + typeset secretKey="${storageConfig#*|}" + + typeset secretJson="" + secretJson="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json)" || true + typeset endpointCheck="" + if [[ -n "${secretJson}" ]]; then + endpointCheck="$(printf '%s' "${secretJson}" | python3 -c " +import sys,json,base64,re +sys.tracebacklimit=0 +d=json.load(sys.stdin) +target_key=sys.argv[1] if len(sys.argv)>1 else 'thanos.yaml' +raw=d.get('data',{}).get(target_key,'') +if not raw: + print('no-endpoint') + sys.exit(0) +try: + content=base64.b64decode(raw).decode('utf-8','replace') +except Exception: + print('no-endpoint') + sys.exit(0) +endpoint='' +try: + import yaml + cfg=yaml.safe_load(content) + endpoint=cfg.get('config',{}).get('endpoint','') if isinstance(cfg,dict) else '' +except Exception: + m=re.search(r'endpoint:\s*(.+)',content) + endpoint=m.group(1).strip() if m else '' +del content +if not endpoint: + print('no-endpoint') + sys.exit(0) +odf_pat=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE) +print('odf-backed' if odf_pat.search(endpoint) else 'external') +" "${secretKey}")" + fi + + if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then + AddResult "storage-endpoint" "fail" "Secret ${secretName} exists but no endpoint config found in key ${secretKey}" + return + fi + + if [[ "${endpointCheck}" == "odf-backed" ]]; then + AddResult "storage-endpoint" "pass" + else + AddResult "storage-endpoint" "fail" "Storage endpoint does not reference ODF-backed service" + fi + true +} + +# --------------------------------------------------------------------------- +# Check 4: Thanos components healthy +# --------------------------------------------------------------------------- + +function CheckThanosHealth () { + : "=== Check 4: Thanos components healthy ===" + + if ! oc get namespace "${OBS_NAMESPACE}" -o name; then + AddResult "thanos-health" "skip" "Observability namespace ${OBS_NAMESPACE} does not exist" + return + fi + + typeset failMsg="" + typeset -i foundCount=0 + typeset -a missingComponents=() + + typeset -a componentNames=("thanos-receive" "thanos-compact" "thanos-store" "thanos-query" "alertmanager" "rbac-query-proxy") + typeset -a componentLabels=("app=thanos-receive" "app=thanos-compact" "app=thanos-store" "app=thanos-query" "alertmanager=observability" "app=rbac-query-proxy") + + typeset -i idx=0 + for idx in "${!componentNames[@]}"; do + typeset component="${componentNames[$idx]}" + typeset labelSelector="${componentLabels[$idx]}" + + typeset podList="" + podList="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \ + --no-headers)" || true + + if [[ -z "${podList}" ]]; then + typeset allPods="" + allPods="$(oc get pods -n "${OBS_NAMESPACE}" \ + --no-headers)" || true + podList="$(printf '%s' "${allPods}" | awk -v pat="^${component}" '$0 ~ pat')" + fi + + typeset podCount="" + podCount="$(printf '%s' "${podList}" | awk 'NF {c++} END{print c+0}')" + + if [[ "${podCount}" -eq 0 ]]; then + missingComponents+=("${component}") + continue + fi + + (( ++foundCount )) + + typeset notReady="" + notReady="$(printf '%s' "${podList}" \ + | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')" + + if [[ -n "${notReady}" ]]; then + typeset compMsg="${component}: ${notReady//$'\n'/, }" + if [[ -n "${failMsg}" ]]; then + failMsg="${failMsg}; ${compMsg}" + else + failMsg="${compMsg}" + fi + fi + done + + if (( foundCount == 0 )); then + AddResult "thanos-health" "skip" "No Thanos/observability components found in ${OBS_NAMESPACE}; observability not deployed" + elif [[ -n "${failMsg}" ]]; then + AddResult "thanos-health" "fail" "Unhealthy Thanos components: ${failMsg}" + elif (( ${#missingComponents[@]} > 0 )); then + AddResult "thanos-health" "fail" "Missing components: ${missingComponents[*]}" + else + AddResult "thanos-health" "pass" + fi + true +} + +# --------------------------------------------------------------------------- +# Check 5: ObjectBucketClaim bound (if used by observability) +# --------------------------------------------------------------------------- + +function CheckObcBound () { + : "=== Check 5: Observability ObjectBucketClaim ===" + + typeset obcList="" + obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true + + typeset obcItemCount=0 + obcItemCount="$(printf '%s' "${obcList}" | python3 -c " +import sys,json +try: + d=json.load(sys.stdin) + print(len(d.get('items',[]))) +except Exception: + print(0) +")" + + if [[ "${obcItemCount}" -eq 0 ]]; then + typeset odfObcJson="" + odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || true + obcList="$(printf '%s' "${odfObcJson}" | python3 -c " +import sys,json +try: + d=json.load(sys.stdin) + obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()] + print(json.dumps({'items':obs})) +except Exception: + print(json.dumps({'items':[]})) +")" + obcItemCount="$(printf '%s' "${obcList}" | python3 -c " +import sys,json +try: + d=json.load(sys.stdin) + print(len(d.get('items',[]))) +except Exception: + print(0) +")" + fi + + if [[ "${obcItemCount}" -eq 0 ]]; then + AddResult "obc-bound" "skip" "No ObjectBucketClaim found for observability" + return + fi + + typeset obcStatus="" + if ! obcStatus="$(echo "${obcList}" | python3 -c " +import sys,json +d=json.load(sys.stdin) +items=d.get('items',[]) +if not items: + print('NotFound') +else: + results=[] + for i in items: + name=i['metadata']['name'] + phase=i.get('status',{}).get('phase','Unknown') + results.append(f'{name}={phase}') + print(';'.join(results)) +")"; then + AddResult "obc-bound" "fail" "Failed to parse OBC status" + return + fi + + if [[ "${obcStatus}" == "NotFound" ]]; then + AddResult "obc-bound" "skip" "No ObjectBucketClaim found for observability" + return + fi + + typeset unboundObcs="" + unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | sed '/=Bound$/d')" + + if [[ -z "${unboundObcs}" ]]; then + : "PASS: All observability OBCs bound: ${obcStatus}" + AddResult "obc-bound" "pass" + else + AddResult "obc-bound" "fail" "Unbound OBCs: ${unboundObcs//$'\n'/, }" + fi + true +} + +# --------------------------------------------------------------------------- +# Check 6: Thanos metrics query functional (basic data flow) +# --------------------------------------------------------------------------- + +function ValidateThanosResponse () { + typeset body="${1:-}"; (($#)) && shift + typeset via="${1:-unknown}"; (($#)) && shift + + typeset validation="" + validation="$(echo "${body}" | python3 -c " +import sys,json +try: + d=json.load(sys.stdin) +except Exception: + print('parse-error') + sys.exit(0) +if d.get('status')!='success': + print('status=' + str(d.get('status',''))) + sys.exit(0) +data=d.get('data',{}) +if data.get('resultType')!='vector': + print('resultType=' + str(data.get('resultType',''))) + sys.exit(0) +result=data.get('result',[]) +if not isinstance(result,list) or len(result)==0: + print('empty-result') + sys.exit(0) +print('ok') +")" + + if [[ "${validation}" == "ok" ]]; then + AddResult "thanos-query" "pass" + elif [[ "${validation}" == "empty-result" ]]; then + AddResult "thanos-query" "fail" "Thanos query succeeded via ${via} but returned empty result vector" + elif [[ "${validation}" == "parse-error" || -z "${validation}" ]]; then + AddResult "thanos-query" "fail" "Thanos query via ${via} returned unparseable response" + else + AddResult "thanos-query" "fail" "Thanos query via ${via} returned ${validation}" + fi + true +} + +function CheckThanosQuery () { + : "=== Check 6: Thanos query functional ===" + + typeset routeJson="" + routeJson="$(oc get routes -n "${OBS_NAMESPACE}" -o json)" || true + typeset queryRoute="" + if [[ -n "${routeJson}" ]]; then + queryRoute="$(printf '%s' "${routeJson}" | python3 -c " +import sys,json +d=json.load(sys.stdin) +routes=d.get('items',[]) +exact=[r for r in routes if r['metadata']['name']=='observability-thanos-query'] +if exact: + print(exact[0]['spec']['host']) + sys.exit(0) +fuzzy=[r for r in routes if 'thanos' in r['metadata']['name'] and 'query' in r['metadata']['name']] +print(fuzzy[0]['spec']['host'] if fuzzy else '') +")" + fi + + if [[ -z "${queryRoute}" ]]; then + : "No external route found; trying internal query service" + + typeset queryFrontendJson='' + queryFrontendJson="$(oc get pods -n "${OBS_NAMESPACE}" \ + -l app.kubernetes.io/name=thanos-query-frontend -o json)" || true + typeset queryFrontendPod='' + if [[ -n "${queryFrontendJson}" ]]; then + queryFrontendPod="$(printf '%s' "${queryFrontendJson}" | python3 -c " +import sys,json +items=json.load(sys.stdin).get('items',[]) +print(items[0]['metadata']['name'] if items else '') +")" + fi + + typeset queryResult="" + if [[ -n "${queryFrontendPod}" ]]; then + queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryFrontendPod}" \ + -- curl -sk "http://localhost:9090/api/v1/query?query=up")" || true + fi + + if [[ -z "${queryResult}" ]]; then + typeset allQueryPods='' + allQueryPods="$(oc get pods -n "${OBS_NAMESPACE}" --no-headers)" || true + typeset queryPod='' + if [[ -n "${allQueryPods}" ]]; then + queryPod="$(printf '%s' "${allQueryPods}" \ + | awk '/thanos-query/ && !/frontend/ {print $1; exit}')" + fi + if [[ -n "${queryPod}" ]]; then + queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryPod}" \ + -- curl -sk "http://localhost:9090/api/v1/query?query=up")" || true + fi + fi + + if [[ -z "${queryResult}" ]]; then + AddResult "thanos-query" "skip" "Cannot reach Thanos query endpoint (no route, exec failed)" + return + fi + + ValidateThanosResponse "${queryResult}" "exec" + return + fi + + typeset token="" + token="$(oc whoami -t)" || true + + typeset responseBody="" + typeset httpCode="" + responseBody="$(curl -sk -w '\n%{http_code}' \ + -H "Authorization: Bearer ${token}" \ + "https://${queryRoute}/api/v1/query?query=up" \ + --max-time 30)" || true + + httpCode="$(echo "${responseBody}" | tail -1)" + responseBody="$(echo "${responseBody}" | sed '$d')" + + if [[ "${httpCode}" != "200" ]]; then + if [[ "${httpCode}" =~ ^(401|403)$ ]]; then + AddResult "thanos-query" "fail" "Thanos query route auth failed (HTTP ${httpCode}); no data flow verified" + else + AddResult "thanos-query" "fail" "Thanos query route unreachable (HTTP ${httpCode:-timeout})" + fi + return + fi + + ValidateThanosResponse "${responseBody}" "route" + true +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +function Main () { + if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then + export KUBECONFIG="${SHARED_DIR}/kubeconfig" + fi + + : "ACM Observability + ODF Interop Validation starting" + : "ACM namespace: ${ACM_NAMESPACE}" + : "Observability namespace: ${OBS_NAMESPACE}" + : "ODF namespace: ${ODF_NAMESPACE}" + : "Artifacts dir: ${ARTIFACT_DIR}" + + CheckRgwReady || true + CheckMcoReady || true + CheckStorageEndpoint || true + CheckThanosHealth || true + CheckObcBound || true + CheckThanosQuery || true + + WriteJunit + + typeset -i hasAnyFail=0 + typeset r="" + for r in "${tcResultsArr[@]}"; do + if [[ "${r}" == "fail" ]]; then + hasAnyFail=1 + break + fi + done + + if (( hasAnyFail )); then + : "ACM Observability + ODF Interop: SOME CHECKS FAILED" + exit 1 + fi + + : "ACM Observability + ODF Interop: ALL PASSED" + exit 0 +} + +Main "$@" diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json new file mode 100644 index 0000000000000..d2d17d0196b23 --- /dev/null +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml new file mode 100644 index 0000000000000..3276ddc73dbd1 --- /dev/null +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml @@ -0,0 +1,36 @@ +ref: + as: interop-opp-observability-odf + from: cli + grace_period: 30s + commands: interop-opp-observability-odf-commands.sh + timeout: 10m + env: + - name: ACM_NAMESPACE + default: "open-cluster-management" + documentation: Namespace where ACM is installed + - name: OBS_NAMESPACE + default: "open-cluster-management-observability" + documentation: Namespace where ACM Observability components run + - name: ODF_NAMESPACE + default: "openshift-storage" + documentation: Namespace where ODF is installed + resources: + requests: + cpu: 100m + memory: 200Mi + best_effort: true + documentation: |- + Validates the cross-product integration surface between ACM Observability + (Thanos) and ODF (Ceph RGW). This is a true interop test that exercises + the boundary where ACM consumes ODF-provided object storage. + + Checks performed: + 1. ODF Ceph RGW infrastructure ready (CephObjectStore, RGW pods, StorageClass) + 2. MultiClusterObservability CR exists and is Ready + 3. Object storage secret references an ODF-backed endpoint (not MinIO) + 4. Thanos and observability components healthy (receive, compact, store, + query, alertmanager, rbac-query-proxy) + 5. ObjectBucketClaim used by observability is Bound + 6. Thanos query endpoint functional (metrics data flow) + + Produces JUnit XML for Prow / Sippy / TestGrid consumption. diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh index 156a7f9b28caa..36441844fcd92 100755 --- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh +++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh @@ -1,6 +1,5 @@ #!/bin/bash -set -euo pipefail -shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit # --------------------------------------------------------------------------- # ODF Health Check (7-point gate) @@ -105,7 +104,7 @@ function CheckOdfCsv () { typeset csvPhase="" if ! csvPhase="$(oc get csv -n "${ODF_NAMESPACE}" -o json | python3 -c " import sys,json,re; d=json.load(sys.stdin) -m=[i for i in d.get('items',[]) if re.match(r'^(odf-|ocs-)operator',i['metadata']['name'])] +m=[i for i in d.get('items',[]) if re.match(r'^(odf-operator|ocs-operator)',i['metadata']['name'])] print((m[0].get('status',{}).get('phase','NotFound')) if m else 'NotFound') ")"; then AddResult "odf-csv-phase" "fail" "Failed to query ODF CSVs in ${ODF_NAMESPACE}" @@ -485,6 +484,32 @@ print(d['items'][0].get('status',{}).get('ceph',{}).get('health','unknown') if d # Main # --------------------------------------------------------------------------- +function CheckOdfInstalled () { + if ! oc get namespace "${ODF_NAMESPACE}" &>/dev/null; then + return 1 + fi + typeset csvJson="" + typeset -i ocExit=0 + csvJson="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || ocExit=$? + if (( ocExit != 0 )); then + printf '%s\n' "Error: oc get csv failed (exit ${ocExit}) in ${ODF_NAMESPACE}" >&2 + return 2 + fi + if [[ -z "${csvJson}" ]]; then + printf '%s\n' "Error: oc get csv returned empty output in ${ODF_NAMESPACE}" >&2 + return 2 + fi + typeset csvCount="" + if ! csvCount="$(printf '%s' "${csvJson}" | python3 -c " +import sys,json,re; d=json.load(sys.stdin) +print(len([i for i in d.get('items',[]) if re.match(r'^(odf-operator|ocs-operator)',i['metadata']['name'])])) +")"; then + printf '%s\n' "Error: failed to parse CSV JSON from ${ODF_NAMESPACE}" >&2 + return 2 + fi + [[ "${csvCount}" -gt 0 ]] +} + function Main () { if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then export KUBECONFIG="${SHARED_DIR}/kubeconfig" @@ -494,6 +519,26 @@ function Main () { : "Namespace: ${ODF_NAMESPACE}" : "Artifacts dir: ${ARTIFACT_DIR}" + typeset -i odfProbeResult=0 + CheckOdfInstalled || odfProbeResult=$? + if (( odfProbeResult == 2 )); then + : "ODF Health Check: PROBE ERROR (cannot determine ODF state)" + exit 1 + fi + if (( odfProbeResult == 1 )); then + typeset skipMsg="ODF is not installed (no ODF/OCS CSV in ${ODF_NAMESPACE})" + typeset -a checkNames=("odf-csv-phase" "storagecluster-ready" "cephcluster-health" + "storageclasses-available" "pvc-provision-rbd" "pvc-provision-cephfs" + "noobaa-s3-functional" "ceph-health-detail") + typeset name="" + for name in "${checkNames[@]}"; do + AddResult "${name}" "skip" "${skipMsg}" + done + WriteJunit + : "ODF Health Check: ALL SKIPPED (ODF not installed)" + exit 0 + fi + CheckOdfCsv || true CheckStorageCluster || true CheckCephCluster || true diff --git a/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh b/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh index 5a78660adc402..45dcea495679c 100755 --- a/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh +++ b/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eux -o pipefail +set -euo pipefail shopt -s inherit_errexit # --------------------------------------------------------------------------- @@ -84,9 +84,20 @@ function CheckCentralRoute () { WaitFor "Central route" CheckCentralRoute -set +x -centralUrl="$(oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}')" -set -x +typeset -i routeElapsed=0 +centralUrl="" +while [[ -z "${centralUrl}" ]]; do + centralUrl="$(oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}' 2>/dev/null)" || true + if [[ -n "${centralUrl}" ]]; then + break + fi + if (( routeElapsed >= 30 )); then + echo "[readiness] FATAL: Central route host empty after 30s" + exit 1 + fi + sleep 5 + (( routeElapsed += 5 )) || true +done echo "[readiness] Central route discovered" # --------------------------------------------------------------------------- @@ -94,10 +105,8 @@ echo "[readiness] Central route discovered" # --------------------------------------------------------------------------- typeset roxAdminPassword="" echo "[readiness] Extracting roxAdminPassword..." -set +x roxAdminPassword="$(oc get secret -n "${centralNs}" central-htpasswd \ -o jsonpath='{.data.password}' | base64 -d)" -set -x if [[ -z "${roxAdminPassword}" ]]; then echo "[readiness] FATAL: could not extract roxAdminPassword" @@ -109,12 +118,10 @@ echo "[readiness] roxAdminPassword extracted successfully" # Check 2: Central API health (authenticated v1/metadata) # --------------------------------------------------------------------------- function CheckCentralApi () { - set +x typeset httpCode="" httpCode="$(curl -sk -o /dev/null -w '%{http_code}' \ -u "admin:${roxAdminPassword}" \ - "https://${centralUrl}/v1/metadata" --max-time 10)" || { set -x; return 1; } - set -x + "https://${centralUrl}/v1/metadata" --max-time 10)" || return 1 [[ "${httpCode}" == "200" ]] } @@ -124,12 +131,10 @@ WaitFor "Central API health (v1/metadata)" CheckCentralApi # Check 3: At least 1 secured cluster connected # --------------------------------------------------------------------------- function CheckClustersConnected () { - set +x typeset clusterCount="" clusterCount="$(curl -sk -u "admin:${roxAdminPassword}" \ "https://${centralUrl}/v1/clusters" --max-time 10 \ - | JsonLength clusters)" || { set -x; return 1; } - set -x + | JsonLength clusters)" || return 1 [[ "${clusterCount}" -ge 1 ]] } @@ -184,12 +189,10 @@ WaitFor "sensor pods Running in ${scNs}" CheckSensorPods # Check 5: Default policies loaded (count > 80) # --------------------------------------------------------------------------- function CheckPoliciesLoaded () { - set +x typeset policyCount="" policyCount="$(curl -sk -u "admin:${roxAdminPassword}" \ "https://${centralUrl}/v1/policies?query=" --max-time 10 \ - | JsonLength policies)" || { set -x; return 1; } - set -x + | JsonLength policies)" || return 1 echo "[readiness] policy count: ${policyCount}" [[ "${policyCount}" -gt 80 ]] } @@ -198,10 +201,8 @@ WaitFor "default policies loaded (>80)" CheckPoliciesLoaded echo "[readiness] Writing connection details to SHARED_DIR..." -set +x echo "${roxAdminPassword}" > "${SHARED_DIR}/ROX_ADMIN_PASSWORD" echo "${centralUrl}" > "${SHARED_DIR}/CENTRAL_URL" -set -x echo "${centralNs}" > "${SHARED_DIR}/CENTRAL_NS" echo "${scNs}" > "${SHARED_DIR}/SC_NS" diff --git a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh index 3907a7d723dab..972ab22b7f8a0 100755 --- a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh +++ b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eux -o pipefail +set -euo pipefail shopt -s inherit_errexit if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then @@ -8,10 +8,8 @@ fi echo "[smoke] Reading connection details from SHARED_DIR..." -set +x CENTRAL_URL="$(cat "${SHARED_DIR}/CENTRAL_URL")" ROX_ADMIN_PASSWORD="$(cat "${SHARED_DIR}/ROX_ADMIN_PASSWORD")" -set -x echo "[smoke] Connection details loaded from SHARED_DIR" @@ -51,7 +49,6 @@ grep -q 'DEFAULT_CLUSTER_NAME = "local-cluster"' \ /tmp/stackrox/qa-tests-backend/src/main/groovy/services/ClusterService.groovy \ || { echo "[smoke] FATAL: DEFAULT_CLUSTER_NAME patch failed"; exit 1; } -set +x export API_HOSTNAME="${CENTRAL_URL}" export API_PORT="443" export ROX_USERNAME="admin" @@ -72,7 +69,6 @@ if [[ -f /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2)" export GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2 fi -set -x cd /tmp/stackrox/qa-tests-backend @@ -103,13 +99,4 @@ if [[ -d build/reports/tests/testSMOKE ]]; then fi echo "[smoke] Test run finished with exit code: ${testExit}" -if [[ "${testExit}" -ne 0 ]] && [[ -d build/test-results/testSMOKE ]]; then - typeset total="" - total="$(find build/test-results/testSMOKE -name '*.xml' -exec grep -l 'testcase' {} \; | wc -l)" - if [[ "${total}" -gt 0 ]]; then - echo "[smoke] Tests executed and results captured; treating as informational (exit 0)." - echo "[smoke] Review JUnit XML in ARTIFACT_DIR for individual test failures." - exit 0 - fi -fi exit "${testExit}" diff --git a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml index 74d11f0815aa9..329a34c9d7a71 100644 --- a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml +++ b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml @@ -11,6 +11,7 @@ ref: memory: 4Gi from: acs-smoke-runner timeout: 1h0m0s + best_effort: true documentation: |- Run the ACS qa-tests-backend SMOKE suite against a live ACS instance. Reads connection credentials from SHARED_DIR