diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml
index 1221f3a80be7a..fe455b70e415c 100644
--- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml
+++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml
@@ -40,6 +40,12 @@ images:
from: cli
optional: true
to: cli-with-git
+ - dockerfile_literal: |
+ FROM this-is-ignored
+ RUN dnf install -y skopeo && dnf clean all
+ from: cli
+ optional: true
+ to: cli-with-skopeo
- dockerfile_literal: |
FROM registry.access.redhat.com/ubi9/openjdk-17:1.21
USER root
@@ -129,6 +135,7 @@ tests:
- ref: acm-fetch-managed-clusters
- ref: acm-opp-app
- ref: interop-opp-odf-health
+ - ref: interop-opp-observability-odf
- ref: interop-tests-opp-quay-smoke
- ref: acm-tests-observability
zz_generated_metadata:
diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml
index c3a92c8259122..9df9af594e67c 100644
--- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml
+++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml
@@ -40,6 +40,12 @@ images:
from: cli
optional: true
to: cli-with-git
+ - dockerfile_literal: |
+ FROM this-is-ignored
+ RUN dnf install -y skopeo && dnf clean all
+ from: cli
+ optional: true
+ to: cli-with-skopeo
- dockerfile_literal: |
FROM registry.access.redhat.com/ubi9/openjdk-17:1.21
USER root
@@ -123,10 +129,11 @@ tests:
- chain: cucushift-installer-check-cluster-health
- ref: stackrox-opp-readiness
- ref: stackrox-opp-smoke
- - ref: acm-tests-clc-create
+ - ref: acm-tests-clc-smoke
- ref: acm-fetch-managed-clusters
- ref: acm-opp-app
- ref: interop-opp-odf-health
+ - ref: interop-opp-observability-odf
- ref: interop-tests-opp-quay-smoke
- ref: acm-tests-observability
- as: interop-opp-vsphere
@@ -177,6 +184,7 @@ tests:
- ref: acm-policies-openshift-plus
- chain: cucushift-installer-check-cluster-health
- ref: interop-opp-odf-health
+ - ref: interop-opp-observability-odf
- ref: acm-tests-observability
- ref: acm-opp-app
workflow: acm-ipi-vsphere
diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml
index e31b7716529a3..df9364f22d947 100644
--- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml
+++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml
@@ -36,6 +36,12 @@ images:
from: cli
optional: true
to: cli-with-git
+ - dockerfile_literal: |
+ FROM this-is-ignored
+ RUN dnf install -y skopeo && dnf clean all
+ from: cli
+ optional: true
+ to: cli-with-skopeo
releases:
latest:
candidate:
@@ -77,6 +83,7 @@ tests:
FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323
FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP
FIREWATCH_FAIL_WITH_TEST_FAILURES: "true"
+ IGNORE_SECONDARY_POLICIES: "true"
OPERATORS: |
[
{"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}
@@ -104,10 +111,11 @@ tests:
- ref: acm-policies-openshift-plus-setup
- ref: acm-policies-openshift-plus
- chain: cucushift-installer-check-cluster-health
- - ref: acm-tests-clc-create
+ - ref: acm-tests-clc-smoke
- ref: acm-fetch-managed-clusters
- ref: acm-opp-app
- ref: interop-opp-odf-health
+ - ref: interop-opp-observability-odf
- ref: interop-tests-opp-quay-smoke
- ref: acm-tests-observability
- as: interop-opp-vsphere
@@ -135,6 +143,7 @@ tests:
FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323
FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP
FIREWATCH_FAIL_WITH_TEST_FAILURES: "true"
+ IGNORE_SECONDARY_POLICIES: "true"
OPENSHIFT_REQUIRED_CORES: "72"
OPENSHIFT_REQUIRED_MEMORY: "288"
OPERATORS: |
@@ -156,6 +165,7 @@ tests:
- ref: acm-policies-openshift-plus
- chain: cucushift-installer-check-cluster-health
- ref: interop-opp-odf-health
+ - ref: interop-opp-observability-odf
- ref: acm-tests-observability
- ref: acm-opp-app
workflow: acm-ipi-vsphere
diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml
index 8598841060b45..8fb2da67b580c 100644
--- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml
+++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml
@@ -36,6 +36,12 @@ images:
from: cli
optional: true
to: cli-with-git
+ - dockerfile_literal: |
+ FROM this-is-ignored
+ RUN dnf install -y skopeo && dnf clean all
+ from: cli
+ optional: true
+ to: cli-with-skopeo
releases:
latest:
candidate:
diff --git a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh
index 0f847d68d0cec..f04cd58e50fb6 100644
--- a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh
+++ b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh
@@ -18,9 +18,15 @@ if [[ -n "${QUAY_OPERATOR_CHANNEL}" ]]; then
fi
echo 'y' | ./deploy.sh -p policygenerator/policy-sets/stable/openshift-plus -n policies -u https://github.com/stolostron/policy-collection.git -a openshift-plus
+# openshift-plus generates ~25 policies; require 4+ before oc wait to avoid
+# racing the GitOps Subscription propagation (stolostron/policy-collection#174)
+typeset -i expectedMinPolicies=4
typeset -i pollDeadline=$((SECONDS + 600))
-until (($(oc get policies -n policies -o name 2>/dev/null | wc -l))); do
- ((SECONDS > pollDeadline)) && { : "Error: no policies appeared after 10 minutes"; exit 1; }
+until (( $(oc get policies -n policies -o name 2>/dev/null | wc -l) >= expectedMinPolicies )); do
+ ((SECONDS > pollDeadline)) && {
+ printf '%s\n' "Error: fewer than ${expectedMinPolicies} policies after 10 minutes" >&2
+ exit 1
+ }
sleep 5
done
@@ -29,7 +35,7 @@ typeset -a quayNamespacesArr=(quay openshift-quay quay-enterprise)
typeset quayFound=false
for ns in "${quayNamespacesArr[@]}"; do
if (($(oc get quayregistry -n "${ns}" -o name 2>/dev/null | wc -l))); then
- : "Found Quay Operator deployment in namespace ${ns}, waiting for ready condition"
+ echo "Found Quay Operator deployment in namespace ${ns}, waiting for ready condition"
oc wait quayregistry --all -n "${ns}" \
--for condition=Available=True \
--timeout=10m || true
@@ -37,12 +43,27 @@ for ns in "${quayNamespacesArr[@]}"; do
break
fi
done
-[[ "${quayFound}" == "false" ]] && : "Warning: no QuayRegistry found in namespaces: ${quayNamespacesArr[*]}"
+[[ "${quayFound}" == "false" ]] && echo "Warning: no QuayRegistry found in namespaces: ${quayNamespacesArr[*]}" >&2
typeset -a secondaryPoliciesArr=(
policy-acs
+ policy-acs-monitor-certs
+ policy-acs-operator-central
+ policy-acs-sync-resources
+ policy-advanced-managed-cluster-security
policy-advanced-managed-cluster-status
+ policy-compliance-operator-install
+ policy-config-quay
policy-hub-quay-bridge
+ policy-install-quay
+ policy-observability-operator
+ policy-observability-storage
+ policy-observability-storage-status
+ policy-odf
+ policy-odf-cluster
+ policy-odf-noobaa
+ policy-odf-status
+ policy-quay-bridge
policy-quay-status
)
@@ -57,12 +78,12 @@ if [[ "${IGNORE_SECONDARY_POLICIES}" == "true" ]]; then
--for jsonpath='{.status.compliant}'=Compliant \
--timeout=40m
} || {
- : "Critical policies failed to become compliant:"
+ echo "ERROR: Critical policies failed to become compliant:" >&2
oc get policies -n policies | grep -Ev "$(IFS='|'; echo "${secondaryPoliciesArr[*]}")" || true
exit 1
}
else
- : "All policies are secondary (ignored), no critical policies to wait for"
+ echo "All policies are secondary (ignored), no critical policies to wait for"
fi
else
{
@@ -70,7 +91,7 @@ else
--for jsonpath='{.status.compliant}'=Compliant \
--timeout=40m
} || {
- : "Policies failed to become compliant:"
+ echo "ERROR: Policies failed to become compliant:" >&2
oc get policies -n policies
exit 1
}
diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/OWNERS b/ci-operator/step-registry/acm/tests/clc-smoke/OWNERS
new file mode 100644
index 0000000000000..76364ea3076e7
--- /dev/null
+++ b/ci-operator/step-registry/acm/tests/clc-smoke/OWNERS
@@ -0,0 +1,9 @@
+approvers:
+- cspi-qe-ocp-lp
+- dtthuynh
+- vboulos
+options: {}
+reviewers:
+- cspi-qe-ocp-lp
+- dtthuynh
+- vboulos
diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/README.md b/ci-operator/step-registry/acm/tests/clc-smoke/README.md
new file mode 100644
index 0000000000000..8819c7ae40c75
--- /dev/null
+++ b/ci-operator/step-registry/acm/tests/clc-smoke/README.md
@@ -0,0 +1,38 @@
+# acm-tests-clc-smoke-ref
+
+## Table of Contents
+- [Purpose](#purpose)
+- [Process](#process)
+- [Requirements](#requirements)
+ - [Infrastructure](#infrastructure)
+ - [Environment Variables](#environment-variables)
+
+## Purpose
+
+Smoke-scoped variant of [acm-tests-clc-create](../clc-create/README.md) with a right-sized timeout and strict failure handling for OPP interop.
+
+The full `acm-tests-clc-create` step already creates only 1 AWS managed cluster (~50 min actual runtime) but carries a 28800s (8h) timeout and suppresses failures with `|| :`. This step:
+- Reduces the timeout to 5400s (90 min), giving ~80% headroom over the observed average.
+- Propagates failures so downstream steps (`acm-fetch-managed-clusters`, `acm-opp-app`) fail fast instead of running against a missing cluster.
+
+> **IMPORTANT**
+> You must use the [acm-tests-clc-destroy-ref](../clc-destroy/README.md) as a post step when using this step. If you do not and succeed in running this step then you will leave clusters running on the ACM QE team's cloud.
+
+## Process
+
+- Copies secret options file needed for test execution.
+- Injects AWS credentials from the cluster profile into options.yaml.
+- Sets dynamic variables based on the provisioned hub cluster.
+- Runs `execute_clc_interop_commands.sh` which invokes Cypress with tag filter `@create+aws+-sno+-@clusterpool` (controlled by `TEST_STAGE=OCPInterop-create` inside the image).
+
+## Requirements
+
+### Infrastructure
+
+- An existing OpenShift cluster to act as the target Hub.
+- "advanced-cluster-management" operator installed (see [`install-operators`](../../../install-operators/README.md)).
+- MCH custom resource installed (see [acm-mch step](../mch/README.md)).
+
+### Environment Variables
+
+- Please see [acm-tests-clc-smoke-ref.yaml](acm-tests-clc-smoke-ref.yaml) env section.
diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh
new file mode 100755
index 0000000000000..83661193bf2ac
--- /dev/null
+++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+set -euxo pipefail; shopt -s inherit_errexit
+
+typeset secretsDir="/tmp/secrets"
+typeset optionFile="./options.yaml"
+typeset awsCredFile="${CLUSTER_PROFILE_DIR}/.awscred"
+
+if [[ "${SKIP_OCP_DEPLOY:-false}" == "true" ]]; then
+ cp "${secretsDir}/ci/kubeconfig" "${SHARED_DIR}/kubeconfig"
+ cp "${secretsDir}/ci/kubeadmin-password" "${SHARED_DIR}/kubeadmin-password"
+fi
+
+cp "${secretsDir}/clc-interop/secret-options-yaml" "${optionFile}"
+
+if [[ -f "${awsCredFile}" ]]; then
+ typeset awsAccKeyID=
+ typeset awsAccKeyToken=
+
+ set +x
+ awsAccKeyID="$(sed -nE 's/^\s*aws_access_key_id\s*=\s*//p;T;q' "${awsCredFile}")"
+ awsAccKeyToken="$(sed -nE 's/^\s*aws_secret_access_key\s*=\s*//p;T;q' "${awsCredFile}")"
+
+ if [[ -z "${awsAccKeyID}" ]] || [[ -z "${awsAccKeyToken}" ]]; then
+ echo "ERROR: Failed to extract AWS credentials from ${awsCredFile}" 1>&2
+ exit 1
+ fi
+
+ yq -o json eval . "${optionFile}" |
+ jq -c \
+ --arg awsAccKeyID "${awsAccKeyID}" \
+ --rawfile awsAccKeyToken <(printf '%s' "${awsAccKeyToken}") \
+ '
+ .options.connections.apiKeys.aws|=(
+ .awsAccessKeyID=$awsAccKeyID |
+ .awsSecretAccessKeyID=($awsAccKeyToken | rtrimstr("\n"))
+ )
+ ' |
+ yq -p json -o yaml eval . > "${optionFile}.tmp"
+ mv -f "${optionFile}.tmp" "${optionFile}"
+ set -x
+
+ unset awsAccKeyID awsAccKeyToken
+fi
+
+set +x
+export CYPRESS_OPTIONS_HUB_PASSWORD=
+CYPRESS_OPTIONS_HUB_PASSWORD="$(cat "${SHARED_DIR}/kubeadmin-password")"
+
+typeset clcStatus=0
+
+CYPRESS_BASE_URL="$(oc whoami --show-console)" \
+CYPRESS_HUB_API_URL="$(oc whoami --show-server)" \
+CYPRESS_CLC_OCP_IMAGE_VERSION="$(cat "${secretsDir}/clc/ocp_image_version")" \
+CLOUD_PROVIDERS="$(cat "${secretsDir}/clc/ocp_cloud_providers")" \
+bash +x ./execute_clc_interop_commands.sh || clcStatus=$?
+set -x
+
+unset CYPRESS_OPTIONS_HUB_PASSWORD
+
+cp -r reports "${ARTIFACT_DIR}/"
+exit "${clcStatus}"
diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json
new file mode 100644
index 0000000000000..bfcc856a27d44
--- /dev/null
+++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json
@@ -0,0 +1,15 @@
+{
+ "path": "acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml",
+ "owners": {
+ "approvers": [
+ "cspi-qe-ocp-lp",
+ "dtthuynh",
+ "vboulos"
+ ],
+ "reviewers": [
+ "cspi-qe-ocp-lp",
+ "dtthuynh",
+ "vboulos"
+ ]
+ }
+}
\ No newline at end of file
diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml
new file mode 100644
index 0000000000000..775b3cda884ae
--- /dev/null
+++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml
@@ -0,0 +1,74 @@
+ref:
+ as: acm-tests-clc-smoke
+ from: clc-ui-e2e
+ commands: acm-tests-clc-smoke-commands.sh
+ timeout: 5400s
+ best_effort: true
+ resources:
+ requests:
+ cpu: '2'
+ memory: 6Gi
+ credentials:
+ - namespace: test-credentials
+ name: opp-acm-clc-credentials
+ mount_path: /tmp/secrets/clc-interop
+ - namespace: test-credentials
+ name: acm-clc-credentials
+ mount_path: /tmp/secrets/clc
+ - namespace: test-credentials
+ name: acm-ci-credentials
+ mount_path: /tmp/secrets/ci
+ env:
+ - name: CYPRESS_OC_IDP
+ default: "kube:admin"
+ documentation: |-
+ Identity
+ - name: CYPRESS_OPTIONS_HUB_USER
+ default: "kubeadmin"
+ documentation: |-
+ Hub cluster username
+ - name: CYPRESS_SPOKE_CLUSTER
+ default: ""
+ documentation: |-
+ Identify spoke clusters
+ - name: BROWSER
+ default: "chrome"
+ documentation: |-
+ Set browser for cypress
+ - name: CUSTOMER_TAGS
+ default: ""
+ documentation: |-
+ Cypress grep tag filter (passed through to test execution)
+ - name: CYPRESS_CLC_OC_IDP
+ default: "clc-e2e-htpasswd"
+ documentation: |-
+ Serves tests RBAC settings
+ - name: CYPRESS_CLC_RBAC_PASS
+ default: "test-RBAC-4-e2e"
+ documentation: |-
+ Serves tests RBAC settings
+ - name: CYPRESS_CLC_OCP_IMAGE_REGISTRY
+ default: "quay.io/openshift-release-dev/ocp-release"
+ documentation: |-
+ Image registry
+ - name: CYPRESS_ACM_NAMESPACE
+ default: "ocm"
+ documentation: |-
+ Acm namespace
+ - name: CYPRESS_MCE_NAMESPACE
+ default: "multicluster-engine"
+ documentation: |-
+ Mce namespace
+ - name: IMPORT_KUBERNETES_CLUSTERS
+ default: ""
+ documentation: |-
+ Comma separated list of imports
+ - name: SKIP_OCP_DEPLOY
+ default: "false"
+ documentation: |-
+ When true, copies kubeconfig from CI secrets instead of using cluster profile
+ documentation: |-
+ Smoke-scoped ACM cluster lifecycle step that creates a single managed
+ cluster on AWS (~50 min). Runs with best_effort so that CLC failures
+ do not block independent downstream validations (ODF health, Quay smoke,
+ observability). JUnit results are still reported for failure visibility.
diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh
index f726090036915..eaf164b557e59 100755
--- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh
+++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh
@@ -1,18 +1,19 @@
#!/bin/bash
-set -euo pipefail
+set -eux -o pipefail
shopt -s inherit_errexit
ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}"
mkdir -p "${ARTIFACT_DIR}"
typeset junitFile="${ARTIFACT_DIR}/junit_quay_interop.xml"
-typeset imageTag="${BUILD_ID:-$(date +%s)}"
+typeset imageTag=''
+imageTag="${BUILD_ID:-$(date +%s)}"
typeset -A testStatus
typeset -A testDuration
typeset -A testFailureMsg
typeset -a allTests=(
"[sig-interop][Jira:INTEROP][Feature:Quay] Push and pull image via Quay route"
- "[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF PVC backing Quay storage"
+ "[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF object storage integration"
"[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image"
)
@@ -33,13 +34,15 @@ function RecordResult () {
testStatus["${name}"]="${status}"
testDuration["${name}"]="${dur}"
testFailureMsg["${name}"]="${msg}"
+ true
}
# shellcheck disable=SC2329
function GenerateJunit () {
typeset -i total=${#allTests[@]}
typeset -i failures=0 skipped=0
- typeset -i elapsed=$(( $(date +%s) - suiteStart ))
+ typeset -i elapsed=0
+ elapsed=$(( $(date +%s) - suiteStart ))
for t in "${allTests[@]}"; do
[[ "${testStatus[${t}]}" == "failed" ]] && failures=$((failures + 1))
@@ -53,94 +56,135 @@ function GenerateJunit () {
EOF
for t in "${allTests[@]}"; do
- typeset escaped_name
- escaped_name=$(printf '%s' "${t}" | sed 's/&/\&/g; s/\</g; s/>/\>/g; s/"/\"/g')
- typeset escaped_msg
- escaped_msg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s/\</g; s/>/\>/g; s/"/\"/g')
+ typeset escapedName
+ escapedName=$(printf '%s' "${t}" | sed 's/&/\&/g; s/\</g; s/>/\>/g; s/"/\"/g')
+ typeset escapedMsg
+ escapedMsg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s/\</g; s/>/\>/g; s/"/\"/g')
if [[ "${testStatus[${t}]}" == "failed" ]]; then
- echo " " >> "${junitFile}"
+ echo " " >> "${junitFile}"
elif [[ "${testStatus[${t}]}" == "skipped" ]]; then
- echo " " >> "${junitFile}"
+ echo " " >> "${junitFile}"
else
- echo " " >> "${junitFile}"
+ echo " " >> "${junitFile}"
fi
done
- cat >> "${junitFile}" <> "${junitFile}" <<'EOF'
EOF
cat "${junitFile}"
+ true
}
-trap GenerateJunit EXIT
+trap '{ ( GenerateJunit; true ); }' EXIT
function DiscoverQuay () {
QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}')
QUAY_REGISTRY=$(oc get quayregistry -n "${QUAY_NS}" -o jsonpath='{.items[0].metadata.name}')
QUAY_HOST=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.status.registryEndpoint}')
QUAY_HOST="${QUAY_HOST#https://}"
+ if [[ -z "${QUAY_HOST}" ]]; then
+ echo "ERROR: Quay registry route not ready (empty host)" >&2
+ return 1
+ fi
export QUAY_NS QUAY_REGISTRY QUAY_HOST
+ true
}
function GetQuayAuth () {
- typeset configSecret
- configSecret=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.spec.configBundleSecret}')
- if [[ -z "${configSecret}" ]]; then
- configSecret="${QUAY_REGISTRY}-config-bundle"
- fi
-
- QUAY_USER=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_EMAIL}' 2>/dev/null | base64 -d || echo "")
- if [[ -z "${QUAY_USER}" ]]; then
+ QUAY_USER=""
+ QUAY_PASSWORD=""
+ QUAY_TOKEN=""
+
+ set +x
+ if oc get secret quayadmin -n "${QUAY_NS}" 2>/dev/null; then
+ QUAY_TOKEN=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_TOKEN=""
+ QUAY_PASSWORD=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD=""
QUAY_USER="quayadmin"
+ if [[ -n "${QUAY_TOKEN}" || -n "${QUAY_PASSWORD}" ]]; then
+ set -x
+ echo "INFO: Quay credentials obtained from quayadmin secret"
+ export QUAY_USER QUAY_PASSWORD QUAY_TOKEN
+ return 0
+ fi
fi
- QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_PASSWORD}' 2>/dev/null | base64 -d || echo "")
- if [[ -z "${QUAY_PASSWORD}" ]]; then
- typeset initSecret="${QUAY_REGISTRY}-init-config-bundle-secret"
- QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${initSecret}" -o jsonpath='{.data.superuser-password}' 2>/dev/null | base64 -d || echo "")
+ if oc get secret quaydevel -n "${QUAY_NS}" 2>/dev/null; then
+ QUAY_PASSWORD=$(oc get secret quaydevel -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD=""
+ QUAY_USER="quaydevel"
+ if [[ -n "${QUAY_PASSWORD}" ]]; then
+ set -x
+ echo "INFO: Quay credentials obtained from quaydevel secret"
+ export QUAY_USER QUAY_PASSWORD QUAY_TOKEN
+ return 0
+ fi
fi
- if [[ -z "${QUAY_PASSWORD}" ]]; then
- for secret in $(oc get secrets -n "${QUAY_NS}" -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep -i "quay.*config"); do
- QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${secret}" -o go-template='{{index .data "config.yaml"}}' 2>/dev/null | base64 -d | grep -oP "(?<=SUPER_USER_PASSWORD: ).*" || echo "")
- [[ -n "${QUAY_PASSWORD}" ]] && break
- done
+ typeset initPassword
+ initPassword=$(python3 -c "import secrets,string; print(''.join(secrets.choice(string.ascii_letters+string.digits) for _ in range(20)))")
+ typeset initResult
+ initResult=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/user/initialize" \
+ -H "Content-Type: application/json" \
+ -d "{\"username\":\"quayadmin\",\"password\":\"${initPassword}\",\"email\":\"quayadmin@example.com\",\"access_token\":true}" 2>/dev/null) || initResult=""
+
+ QUAY_TOKEN=$(echo "${initResult}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('access_token',''))" 2>/dev/null) || QUAY_TOKEN=""
+ if [[ -n "${QUAY_TOKEN}" ]]; then
+ QUAY_USER="quayadmin"
+ QUAY_PASSWORD="${initPassword}"
+ set -x
+ echo "INFO: Quay admin user initialized via /api/v1/user/initialize"
+ export QUAY_USER QUAY_PASSWORD QUAY_TOKEN
+ return 0
fi
+ set -x
- export QUAY_USER QUAY_PASSWORD
+ echo "ERROR: Could not obtain Quay credentials from any source" >&2
+ export QUAY_USER QUAY_PASSWORD QUAY_TOKEN
+ return 1
}
function PreflightCheck () {
if ! curl -sk --connect-timeout 15 "https://${QUAY_HOST}/api/v1/discovery" | grep -qi "quay"; then
- echo "ERROR: Quay route not reachable at ${QUAY_HOST}" >&2
+ echo "ERROR: Quay registry endpoint not reachable" >&2
return 1
fi
+ true
}
function CreateTestOrg () {
- typeset signinPayload
- signinPayload=$(python3 -c "import json,sys; print(json.dumps({'user':sys.argv[1],'pass':sys.argv[2]}))" "${QUAY_USER}" "${QUAY_PASSWORD}")
- typeset token
- token=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \
- -H "Content-Type: application/json" \
- -d "${signinPayload}" | \
- python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || echo "")
-
- if [[ -z "${token}" ]]; then
- token=$(curl -sk -H "Authorization: Basic $(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64)" \
- "https://${QUAY_HOST}/api/v1/user/" | \
- python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null || echo "")
+ if [[ -z "${QUAY_TOKEN}" && -n "${QUAY_PASSWORD}" ]]; then
+ typeset cookieFile="/tmp/quay-cookies.txt"
+ typeset csrf
+ csrf=$(curl -sk "https://${QUAY_HOST}/csrf_token" -c "${cookieFile}" | \
+ python3 -c "import sys,json; print(json.load(sys.stdin).get('csrf_token',''))" 2>/dev/null) || csrf=""
+
+ if [[ -n "${csrf}" ]]; then
+ typeset signinResult
+ set +x
+ signinResult=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \
+ -H "Content-Type: application/json" \
+ -H "X-CSRF-Token: ${csrf}" \
+ -b "${cookieFile}" -c "${cookieFile}" \
+ -d "{\"username\":\"${QUAY_USER}\",\"password\":\"${QUAY_PASSWORD}\"}" 2>/dev/null) || signinResult=""
+ QUAY_TOKEN=$(echo "${signinResult}" | \
+ python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null) || QUAY_TOKEN=""
+ set -x
+ fi
+ rm -f "${cookieFile}"
+ export QUAY_TOKEN
fi
- QUAY_TOKEN="${token}"
- export QUAY_TOKEN
+ if [[ -z "${QUAY_TOKEN}" ]]; then
+ echo "WARNING: No Quay token available; org creation may fail" >&2
+ fi
curl -sk -X POST "https://${QUAY_HOST}/api/v1/organization/" \
-H "Authorization: Bearer ${QUAY_TOKEN}" \
-H "Content-Type: application/json" \
-d '{"name":"interop-smoke-test","email":"interop-test@example.com"}' || true
+ true
}
################################################################################
@@ -154,9 +198,24 @@ function RunPushPull () {
typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}"
typeset authFile="/tmp/quay-auth.json"
+ if [[ -z "${QUAY_TOKEN}" && -z "${QUAY_PASSWORD}" ]]; then
+ elapsed=$(( $(date +%s) - start ))
+ RecordResult "${testName}" "failed" "No valid Quay authentication token or password available" "${elapsed}"
+ return 1
+ fi
+
+ typeset registryAuth
+ set +x
+ if [[ -n "${QUAY_TOKEN}" ]]; then
+ registryAuth=$(echo -n "\$oauthtoken:${QUAY_TOKEN}" | base64)
+ else
+ registryAuth=$(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64)
+ fi
+
cat > "${authFile}" </dev/null 2>&1; then
+ "docker://${pushTarget}"; then
elapsed=$(( $(date +%s) - start ))
RecordResult "${testName}" "failed" "Image not pullable from Quay after push" "${elapsed}"
return 1
@@ -181,64 +240,56 @@ EOF
}
################################################################################
-# Test Case 2: Verify ODF PVC backing Quay storage
+# Test Case 2: Verify ODF object storage integration
################################################################################
-function RunOdfPvcCheck () {
- typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF PVC backing Quay storage"
+function RunOdfStorageCheck () {
+ typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF object storage integration"
typeset -i start elapsed
start=$(date +%s)
- typeset pvcCount
- pvcCount=$(oc get pvc -n "${QUAY_NS}" -l app=quay -o json 2>/dev/null | python3 -c "
-import sys, json
-data = json.load(sys.stdin)
-items = data.get('items', [])
-print(len(items))
-" 2>/dev/null || echo "0")
+ typeset noobaaPhase
+ noobaaPhase=$(oc get noobaa -n openshift-storage -o jsonpath='{.items[0].status.phase}' 2>/dev/null) || noobaaPhase=""
+ if [[ "${noobaaPhase}" != "Ready" ]]; then
+ elapsed=$(( $(date +%s) - start ))
+ RecordResult "${testName}" "failed" "NooBaa not Ready (phase: ${noobaaPhase:-not found})" "${elapsed}"
+ return 1
+ fi
- if [[ "${pvcCount}" == "0" ]]; then
- pvcCount=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c "
-import sys, json
-data = json.load(sys.stdin)
-items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()]
-print(len(items))
-" 2>/dev/null || echo "0")
+ typeset obcCount
+ obcCount=$(oc get objectbucketclaim -n openshift-storage -o json 2>/dev/null | \
+ python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obcCount="0"
+ if [[ "${obcCount}" == "0" ]]; then
+ obcCount=$(oc get objectbucketclaim --all-namespaces -o json 2>/dev/null | \
+ python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obcCount="0"
fi
- if [[ "${pvcCount}" == "0" ]]; then
+ if [[ "${obcCount}" == "0" ]]; then
elapsed=$(( $(date +%s) - start ))
- RecordResult "${testName}" "failed" "No Quay-related PVCs found in ${QUAY_NS}" "${elapsed}"
+ RecordResult "${testName}" "failed" "No ObjectBucketClaims found" "${elapsed}"
return 1
fi
- typeset unboundPvcs
- unboundPvcs=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c "
-import sys, json
-data = json.load(sys.stdin)
-items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()]
-unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound']
-print(' '.join(unbound))
-" 2>/dev/null || echo "")
-
- if [[ -n "${unboundPvcs}" ]]; then
+ typeset obCount
+ obCount=$(oc get objectbucket -o json 2>/dev/null | \
+ python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obCount="0"
+ if [[ "${obCount}" == "0" ]]; then
elapsed=$(( $(date +%s) - start ))
- RecordResult "${testName}" "failed" "Unbound PVCs: ${unboundPvcs}" "${elapsed}"
+ RecordResult "${testName}" "failed" "No ObjectBucket resources found for OBCs" "${elapsed}"
return 1
fi
- typeset odfBacked
- odfBacked=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c "
+ typeset unboundPvcs
+ unboundPvcs=$(oc get pvc -n "${QUAY_NS}" -o json 2>/dev/null | python3 -c "
import sys, json
data = json.load(sys.stdin)
items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()]
-sc_names = set(i['spec'].get('storageClassName','') for i in items)
-odf = any('ocs' in s or 'ceph' in s or 'odf' in s for s in sc_names)
-print('true' if odf else 'false')
-" 2>/dev/null || echo "false")
+unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound']
+print(' '.join(unbound))
+" 2>/dev/null) || unboundPvcs=""
- if [[ "${odfBacked}" != "true" ]]; then
+ if [[ -n "${unboundPvcs}" ]]; then
elapsed=$(( $(date +%s) - start ))
- RecordResult "${testName}" "failed" "Quay PVCs not using ODF/Ceph storage class" "${elapsed}"
+ RecordResult "${testName}" "failed" "Unbound Quay PVCs: ${unboundPvcs}" "${elapsed}"
return 1
fi
@@ -250,20 +301,103 @@ print('true' if odf else 'false')
################################################################################
# Test Case 3: ACS scan of pushed Quay image
################################################################################
+function RegisterQuayInAcs () {
+ typeset acsHost="${1}" acsPassword="${2}"
+ set +x
+
+ typeset existing
+ existing=$(curl -sk -u "admin:${acsPassword}" \
+ "https://${acsHost}/v1/imageintegrations" 2>/dev/null | \
+ python3 -c "
+import sys, json, os
+host = os.environ['QUAY_HOST']
+data = json.load(sys.stdin)
+for i in data.get('integrations', []):
+ if host in i.get('docker', {}).get('endpoint', ''):
+ print(i['id'])
+ sys.exit(0)
+sys.exit(1)
+" 2>/dev/null) || existing=""
+
+ if [[ -n "${existing}" ]]; then
+ echo "INFO: Quay integration already registered in ACS"
+ return 0
+ fi
+
+ typeset regUser regPass
+ if [[ -n "${QUAY_TOKEN}" ]]; then
+ regUser="\$oauthtoken"
+ regPass="${QUAY_TOKEN}"
+ else
+ regUser="${QUAY_USER}"
+ regPass="${QUAY_PASSWORD}"
+ fi
+
+ typeset regPayload=''
+ regPayload=$(python3 -c "
+import json, sys, os
+payload = {
+ 'name': 'interop-quay-smoke',
+ 'type': 'docker',
+ 'categories': ['REGISTRY'],
+ 'docker': {
+ 'endpoint': os.environ['QUAY_HOST'],
+ 'username': sys.argv[1],
+ 'password': sys.argv[2],
+ 'insecure': True
+ },
+ 'skipTestIntegration': True
+}
+print(json.dumps(payload))
+" "${regUser}" "${regPass}")
+
+ curl -sk -X POST "https://${acsHost}/v1/imageintegrations" \
+ -u "admin:${acsPassword}" \
+ -H "Content-Type: application/json" \
+ -d "${regPayload}" || true
+ set -x
+
+ echo "INFO: Registered Quay registry endpoint as ACS image integration"
+ true
+}
+
+function RequestAcsScan () {
+ typeset acsHost="${1}" acsPassword="${2}" imageName="${3}"
+ set +x
+
+ typeset scanPayload=''
+ scanPayload=$(python3 -c "
+import json, sys
+payload = {'imageName': sys.argv[1], 'force': True}
+print(json.dumps(payload))
+" "${imageName}")
+
+ curl -sk -X POST "https://${acsHost}/v1/images/scan" \
+ -u "admin:${acsPassword}" \
+ -H "Content-Type: application/json" \
+ -d "${scanPayload}" || true
+ set -x
+
+ echo "INFO: Requested ACS scan of ${imageName}"
+ true
+}
+
function RunAcsScan () {
typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image"
typeset -i start elapsed
start=$(date +%s)
typeset acsHost acsPassword
- acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null || echo "")
+ acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null) || acsHost=""
if [[ -z "${acsHost}" ]]; then
elapsed=$(( $(date +%s) - start ))
RecordResult "${testName}" "failed" "ACS Central route not found" "${elapsed}"
return 1
fi
- acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "")
+ set +x
+ acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || acsPassword=""
+ set -x
if [[ -z "${acsPassword}" ]]; then
elapsed=$(( $(date +%s) - start ))
RecordResult "${testName}" "failed" "ACS admin password not found" "${elapsed}"
@@ -271,12 +405,20 @@ function RunAcsScan () {
fi
typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}"
- typeset -i attempts=0 maxAttempts=20
+
+ set +x
+ RegisterQuayInAcs "${acsHost}" "${acsPassword}"
+ RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}"
+ set -x
+
+ typeset -i attempts=0 maxAttempts=40
while (( attempts < maxAttempts )); do
typeset scanResult
+ set +x
scanResult=$(curl -sk -u "admin:${acsPassword}" \
- "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null || echo "")
+ "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null) || scanResult=""
+ set -x
if echo "${scanResult}" | python3 -c "
import sys, json
@@ -289,12 +431,18 @@ sys.exit(0 if len(images) > 0 else 1)
return 0
fi
+ if (( attempts % 4 == 3 )); then
+ set +x
+ RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}"
+ set -x
+ fi
+
attempts=$((attempts + 1))
sleep 15
done
elapsed=$(( $(date +%s) - start ))
- RecordResult "${testName}" "failed" "ACS did not detect pushed image within 5 minutes" "${elapsed}"
+ RecordResult "${testName}" "failed" "ACS did not detect pushed image within 10 minutes" "${elapsed}"
return 1
}
@@ -308,10 +456,14 @@ function Main () {
PreflightCheck || { echo "FATAL: Quay not reachable; skipping all tests" >&2; exit 1; }
CreateTestOrg
- typeset -i status=0
- RunPushPull || status=1
- RunOdfPvcCheck || status=1
- RunAcsScan || status=1
+ typeset -i status=0 pushPassed=0
+ RunPushPull && pushPassed=1 || status=1
+ RunOdfStorageCheck || status=1
+ if (( pushPassed )); then
+ RunAcsScan || status=1
+ else
+ RecordResult "[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" "skipped" "Skipped: push-pull test failed; no image available to scan"
+ fi
rm -f /tmp/quay-auth.json
@@ -322,7 +474,7 @@ function Main () {
"${_fURL[@]}" \
https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/ci-operator/interop/common/ExitTrap--PostProcessPrep.sh
)" || true
- if type -t ExitTrap--PostProcessPrep 1>/dev/null; then
+ if type -t ExitTrap--PostProcessPrep; then
LP_IO__ET_PPP__NEW_TS_NAME="${DR__RP__CR_COMP_NAME}--%s" \
ExitTrap--PostProcessPrep || true
fi
diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml
index eb2e58149f440..5b0caae06170a 100644
--- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml
+++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml
@@ -1,6 +1,6 @@
ref:
as: interop-tests-opp-quay-smoke
- from: cli
+ from: cli-with-skopeo
cli: latest
commands: interop-tests-opp-quay-smoke-commands.sh
timeout: 30m0s
@@ -11,8 +11,9 @@ ref:
memory: 256Mi
documentation: |-
Validates Quay as a cross-product registry within the OPP bundle.
- Tests image push/pull via the Quay route, verifies ODF-backed PVC storage,
- and confirms ACS detects and scans the pushed image.
+ Tests image push/pull via the Quay route, verifies ODF object storage
+ integration (OBCs and NooBaa health), and confirms ACS detects and scans
+ the pushed image.
env:
- name: DR__RP__CR_COMP_NAME
default: "lp-interop--Quay"
diff --git a/ci-operator/step-registry/interop/opp/observability-odf/OWNERS b/ci-operator/step-registry/interop/opp/observability-odf/OWNERS
new file mode 100644
index 0000000000000..41d144d3728a2
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/observability-odf/OWNERS
@@ -0,0 +1,3 @@
+approvers: &owners
+- cspi-qe-ocp-lp
+reviewers: *owners
diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
new file mode 100755
index 0000000000000..0d7fcf9504446
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
@@ -0,0 +1,629 @@
+#!/bin/bash
+set -euo pipefail; shopt -s inherit_errexit
+
+# ---------------------------------------------------------------------------
+# ACM Observability + ODF Interop Validation (6-point gate)
+#
+# Validates that ACM's observability stack (Thanos) correctly uses
+# ODF-provided object storage (Ceph RGW or NooBaa S3) as its backend.
+# This is a cross-product interop test exercising the ACM <-> ODF boundary.
+#
+# Produces JUnit XML consumed by Prow / Sippy / TestGrid.
+# ---------------------------------------------------------------------------
+
+typeset ACM_NAMESPACE="${ACM_NAMESPACE:-open-cluster-management}"
+typeset OBS_NAMESPACE="${OBS_NAMESPACE:-open-cluster-management-observability}"
+typeset ODF_NAMESPACE="${ODF_NAMESPACE:-openshift-storage}"
+
+typeset junitFile="${ARTIFACT_DIR}/junit_observability_odf.xml"
+
+typeset -a tcNamesArr=()
+typeset -a tcResultsArr=()
+typeset -a tcMessagesArr=()
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+function AddResult () {
+ typeset name="${1:-}"; (($#)) && shift
+ typeset result="${1:-}"; (($#)) && shift
+ typeset message="${1:-}"; (($#)) && shift
+ tcNamesArr+=("${name}")
+ tcResultsArr+=("${result}")
+ tcMessagesArr+=("${message}")
+ true
+}
+
+function XmlEscape () {
+ typeset text="${1:-}"; (($#)) && shift
+ text="${text//&/&}"
+ text="${text//<}"
+ text="${text//>/>}"
+ text="${text//\"/"}"
+ text="${text//\'/'}"
+ printf '%s' "${text}"
+ true
+}
+
+function WriteJunit () {
+ typeset -i total=${#tcNamesArr[@]}
+ typeset -i failCount=0
+ typeset -i skipCount=0
+ typeset r=""
+ for r in "${tcResultsArr[@]}"; do
+ if [[ "${r}" == "fail" ]]; then
+ (( ++failCount ))
+ elif [[ "${r}" == "skip" ]]; then
+ (( ++skipCount ))
+ fi
+ done
+
+ {
+ echo ''
+ echo ""
+ typeset -i i=0
+ for i in "${!tcNamesArr[@]}"; do
+ typeset name=""
+ name="$(XmlEscape "${tcNamesArr[$i]}")"
+ echo " "
+ if [[ "${tcResultsArr[$i]}" == "fail" ]]; then
+ typeset msg=""
+ msg="$(XmlEscape "${tcMessagesArr[$i]}")"
+ echo " "
+ elif [[ "${tcResultsArr[$i]}" == "skip" ]]; then
+ typeset msg=""
+ msg="$(XmlEscape "${tcMessagesArr[$i]}")"
+ echo " "
+ fi
+ echo " "
+ done
+ echo ""
+ } > "${junitFile}"
+ : "JUnit XML written to ${junitFile}"
+ true
+}
+
+# shellcheck disable=SC2317,SC2329
+function CollectExitArtifacts () {
+ : "Collecting observability + ODF diagnostics..."
+ oc get multiclusterobservabilities.observability.open-cluster-management.io --all-namespaces -o yaml > "${ARTIFACT_DIR}/mco.yaml" || true
+ oc get pods -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-pods.yaml" || true
+ oc get obc -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-obc.yaml" || true
+ oc get secret -n "${OBS_NAMESPACE}" -o name > "${ARTIFACT_DIR}/obs-secrets-list.txt" || true
+ oc get cephobjectstore -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/cephobjectstore.yaml" || true
+ oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw -o yaml > "${ARTIFACT_DIR}/rgw-pods.yaml" || true
+ oc get noobaa -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/noobaa.yaml" || true
+ true
+}
+
+trap '{( CollectExitArtifacts; true )}' EXIT
+
+# ---------------------------------------------------------------------------
+# Check 1: ODF Ceph RGW infrastructure ready
+# ---------------------------------------------------------------------------
+
+function CheckRgwReady () {
+ : "=== Check 1: ODF Ceph RGW infrastructure ==="
+
+ typeset rgwPhase=""
+ if ! rgwPhase="$(oc get cephobjectstore -n "${ODF_NAMESPACE}" -o json | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+items=d.get('items',[])
+if not items:
+ print('NotFound')
+else:
+ print(items[0].get('status',{}).get('phase','Unknown'))
+")"; then
+ AddResult "odf-storage-ready" "fail" "Failed to query CephObjectStore"
+ return
+ fi
+
+ if [[ "${rgwPhase}" == "NotFound" ]]; then
+ typeset noobaaJson=""
+ noobaaJson="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json)" || true
+ typeset noobaaPhase=""
+ if [[ -n "${noobaaJson}" ]]; then
+ noobaaPhase="$(printf '%s' "${noobaaJson}" | python3 -c "
+import sys,json
+items=json.load(sys.stdin).get('items',[])
+print(items[0].get('status',{}).get('phase','') if items else '')
+")"
+ fi
+ if [[ "${noobaaPhase}" == "Ready" ]]; then
+ AddResult "odf-storage-ready" "pass" "NooBaa Ready (RGW not deployed)"
+ elif [[ -n "${noobaaPhase}" ]]; then
+ AddResult "odf-storage-ready" "fail" "NooBaa phase=${noobaaPhase} (expected Ready); RGW not deployed"
+ else
+ AddResult "odf-storage-ready" "skip" "Neither CephObjectStore nor NooBaa found in ${ODF_NAMESPACE}"
+ fi
+ return
+ fi
+
+ typeset failMsg=""
+ if [[ "${rgwPhase}" != "Ready" ]]; then
+ failMsg="CephObjectStore phase=${rgwPhase} (expected Ready)"
+ fi
+
+ typeset rgwPods=""
+ rgwPods="$(oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw \
+ --field-selector=status.phase=Running --no-headers)" || true
+ typeset rgwPodCount=""
+ rgwPodCount="$(printf '%s' "${rgwPods}" | awk 'END{print NR}')"
+
+ if [[ "${rgwPodCount}" -eq 0 ]]; then
+ typeset rgwMsg="No rook-ceph-rgw pods Running in ${ODF_NAMESPACE}"
+ if [[ -n "${failMsg}" ]]; then
+ failMsg="${failMsg}; ${rgwMsg}"
+ else
+ failMsg="${rgwMsg}"
+ fi
+ fi
+
+ typeset scExists=""
+ scExists="$(oc get sc ocs-storagecluster-ceph-rgw -o name)" || true
+ if [[ -z "${scExists}" ]]; then
+ typeset scMsg="StorageClass ocs-storagecluster-ceph-rgw not found"
+ if [[ -n "${failMsg}" ]]; then
+ failMsg="${failMsg}; ${scMsg}"
+ else
+ failMsg="${scMsg}"
+ fi
+ fi
+
+ if [[ -z "${failMsg}" ]]; then
+ : "PASS: CephObjectStore Ready, RGW pods Running, StorageClass exists"
+ AddResult "odf-storage-ready" "pass"
+ else
+ AddResult "odf-storage-ready" "fail" "${failMsg}"
+ fi
+ true
+}
+
+# ---------------------------------------------------------------------------
+# Check 2: MultiClusterObservability CR exists and is Ready
+# ---------------------------------------------------------------------------
+
+function CheckMcoReady () {
+ : "=== Check 2: MultiClusterObservability CR ==="
+
+ typeset mcoStatus=""
+ if ! mcoStatus="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \
+ --all-namespaces -o json | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+items=d.get('items',[])
+if not items:
+ print('NotFound')
+else:
+ conds=items[0].get('status',{}).get('conditions',[])
+ ready=[c for c in conds if c.get('type')=='Ready']
+ print(ready[0].get('status','Unknown') if ready else 'NoCondition')
+")"; then
+ AddResult "mco-ready" "fail" "Failed to query MultiClusterObservability CR"
+ return
+ fi
+
+ if [[ "${mcoStatus}" == "True" ]]; then
+ : "PASS: MultiClusterObservability Ready=True"
+ AddResult "mco-ready" "pass"
+ elif [[ "${mcoStatus}" == "NotFound" ]]; then
+ AddResult "mco-ready" "skip" "MultiClusterObservability CR not found; observability not deployed"
+ else
+ AddResult "mco-ready" "fail" "MultiClusterObservability Ready=${mcoStatus} (expected True)"
+ fi
+ true
+}
+
+# ---------------------------------------------------------------------------
+# Check 3: Object storage secret references ODF-backed endpoint
+# ---------------------------------------------------------------------------
+
+function CheckStorageEndpoint () {
+ : "=== Check 3: Object storage endpoint ==="
+
+ typeset storageConfig=""
+ if ! storageConfig="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \
+ --all-namespaces -o json | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+items=d.get('items',[])
+if not items:
+ print('')
+else:
+ spec=items[0].get('spec',{})
+ storage=spec.get('storageConfig',{}).get('metricObjectStorage',{})
+ name=storage.get('name','')
+ key=storage.get('key','thanos.yaml')
+ print(f'{name}|{key}' if name else '')
+")"; then
+ AddResult "storage-endpoint" "fail" "Failed to read MCO storage config"
+ return
+ fi
+
+ if [[ -z "${storageConfig}" ]]; then
+ AddResult "storage-endpoint" "skip" "No metricObjectStorage secret configured in MCO"
+ return
+ fi
+
+ typeset secretName="${storageConfig%%|*}"
+ typeset secretKey="${storageConfig#*|}"
+
+ typeset secretJson=""
+ secretJson="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json)" || true
+ typeset endpointCheck=""
+ if [[ -n "${secretJson}" ]]; then
+ endpointCheck="$(printf '%s' "${secretJson}" | python3 -c "
+import sys,json,base64,re
+sys.tracebacklimit=0
+d=json.load(sys.stdin)
+target_key=sys.argv[1] if len(sys.argv)>1 else 'thanos.yaml'
+raw=d.get('data',{}).get(target_key,'')
+if not raw:
+ print('no-endpoint')
+ sys.exit(0)
+try:
+ content=base64.b64decode(raw).decode('utf-8','replace')
+except Exception:
+ print('no-endpoint')
+ sys.exit(0)
+endpoint=''
+try:
+ import yaml
+ cfg=yaml.safe_load(content)
+ endpoint=cfg.get('config',{}).get('endpoint','') if isinstance(cfg,dict) else ''
+except Exception:
+ m=re.search(r'endpoint:\s*(.+)',content)
+ endpoint=m.group(1).strip() if m else ''
+del content
+if not endpoint:
+ print('no-endpoint')
+ sys.exit(0)
+odf_pat=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE)
+print('odf-backed' if odf_pat.search(endpoint) else 'external')
+" "${secretKey}")"
+ fi
+
+ if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then
+ AddResult "storage-endpoint" "fail" "Secret ${secretName} exists but no endpoint config found in key ${secretKey}"
+ return
+ fi
+
+ if [[ "${endpointCheck}" == "odf-backed" ]]; then
+ AddResult "storage-endpoint" "pass"
+ else
+ AddResult "storage-endpoint" "fail" "Storage endpoint does not reference ODF-backed service"
+ fi
+ true
+}
+
+# ---------------------------------------------------------------------------
+# Check 4: Thanos components healthy
+# ---------------------------------------------------------------------------
+
+function CheckThanosHealth () {
+ : "=== Check 4: Thanos components healthy ==="
+
+ if ! oc get namespace "${OBS_NAMESPACE}" -o name; then
+ AddResult "thanos-health" "skip" "Observability namespace ${OBS_NAMESPACE} does not exist"
+ return
+ fi
+
+ typeset failMsg=""
+ typeset -i foundCount=0
+ typeset -a missingComponents=()
+
+ typeset -a componentNames=("thanos-receive" "thanos-compact" "thanos-store" "thanos-query" "alertmanager" "rbac-query-proxy")
+ typeset -a componentLabels=("app=thanos-receive" "app=thanos-compact" "app=thanos-store" "app=thanos-query" "alertmanager=observability" "app=rbac-query-proxy")
+
+ typeset -i idx=0
+ for idx in "${!componentNames[@]}"; do
+ typeset component="${componentNames[$idx]}"
+ typeset labelSelector="${componentLabels[$idx]}"
+
+ typeset podList=""
+ podList="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \
+ --no-headers)" || true
+
+ if [[ -z "${podList}" ]]; then
+ typeset allPods=""
+ allPods="$(oc get pods -n "${OBS_NAMESPACE}" \
+ --no-headers)" || true
+ podList="$(printf '%s' "${allPods}" | awk -v pat="^${component}" '$0 ~ pat')"
+ fi
+
+ typeset podCount=""
+ podCount="$(printf '%s' "${podList}" | awk 'NF {c++} END{print c+0}')"
+
+ if [[ "${podCount}" -eq 0 ]]; then
+ missingComponents+=("${component}")
+ continue
+ fi
+
+ (( ++foundCount ))
+
+ typeset notReady=""
+ notReady="$(printf '%s' "${podList}" \
+ | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')"
+
+ if [[ -n "${notReady}" ]]; then
+ typeset compMsg="${component}: ${notReady//$'\n'/, }"
+ if [[ -n "${failMsg}" ]]; then
+ failMsg="${failMsg}; ${compMsg}"
+ else
+ failMsg="${compMsg}"
+ fi
+ fi
+ done
+
+ if (( foundCount == 0 )); then
+ AddResult "thanos-health" "skip" "No Thanos/observability components found in ${OBS_NAMESPACE}; observability not deployed"
+ elif [[ -n "${failMsg}" ]]; then
+ AddResult "thanos-health" "fail" "Unhealthy Thanos components: ${failMsg}"
+ elif (( ${#missingComponents[@]} > 0 )); then
+ AddResult "thanos-health" "fail" "Missing components: ${missingComponents[*]}"
+ else
+ AddResult "thanos-health" "pass"
+ fi
+ true
+}
+
+# ---------------------------------------------------------------------------
+# Check 5: ObjectBucketClaim bound (if used by observability)
+# ---------------------------------------------------------------------------
+
+function CheckObcBound () {
+ : "=== Check 5: Observability ObjectBucketClaim ==="
+
+ typeset obcList=""
+ obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true
+
+ typeset obcItemCount=0
+ obcItemCount="$(printf '%s' "${obcList}" | python3 -c "
+import sys,json
+try:
+ d=json.load(sys.stdin)
+ print(len(d.get('items',[])))
+except Exception:
+ print(0)
+")"
+
+ if [[ "${obcItemCount}" -eq 0 ]]; then
+ typeset odfObcJson=""
+ odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || true
+ obcList="$(printf '%s' "${odfObcJson}" | python3 -c "
+import sys,json
+try:
+ d=json.load(sys.stdin)
+ obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()]
+ print(json.dumps({'items':obs}))
+except Exception:
+ print(json.dumps({'items':[]}))
+")"
+ obcItemCount="$(printf '%s' "${obcList}" | python3 -c "
+import sys,json
+try:
+ d=json.load(sys.stdin)
+ print(len(d.get('items',[])))
+except Exception:
+ print(0)
+")"
+ fi
+
+ if [[ "${obcItemCount}" -eq 0 ]]; then
+ AddResult "obc-bound" "skip" "No ObjectBucketClaim found for observability"
+ return
+ fi
+
+ typeset obcStatus=""
+ if ! obcStatus="$(echo "${obcList}" | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+items=d.get('items',[])
+if not items:
+ print('NotFound')
+else:
+ results=[]
+ for i in items:
+ name=i['metadata']['name']
+ phase=i.get('status',{}).get('phase','Unknown')
+ results.append(f'{name}={phase}')
+ print(';'.join(results))
+")"; then
+ AddResult "obc-bound" "fail" "Failed to parse OBC status"
+ return
+ fi
+
+ if [[ "${obcStatus}" == "NotFound" ]]; then
+ AddResult "obc-bound" "skip" "No ObjectBucketClaim found for observability"
+ return
+ fi
+
+ typeset unboundObcs=""
+ unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | sed '/=Bound$/d')"
+
+ if [[ -z "${unboundObcs}" ]]; then
+ : "PASS: All observability OBCs bound: ${obcStatus}"
+ AddResult "obc-bound" "pass"
+ else
+ AddResult "obc-bound" "fail" "Unbound OBCs: ${unboundObcs//$'\n'/, }"
+ fi
+ true
+}
+
+# ---------------------------------------------------------------------------
+# Check 6: Thanos metrics query functional (basic data flow)
+# ---------------------------------------------------------------------------
+
+function ValidateThanosResponse () {
+ typeset body="${1:-}"; (($#)) && shift
+ typeset via="${1:-unknown}"; (($#)) && shift
+
+ typeset validation=""
+ validation="$(echo "${body}" | python3 -c "
+import sys,json
+try:
+ d=json.load(sys.stdin)
+except Exception:
+ print('parse-error')
+ sys.exit(0)
+if d.get('status')!='success':
+ print('status=' + str(d.get('status','')))
+ sys.exit(0)
+data=d.get('data',{})
+if data.get('resultType')!='vector':
+ print('resultType=' + str(data.get('resultType','')))
+ sys.exit(0)
+result=data.get('result',[])
+if not isinstance(result,list) or len(result)==0:
+ print('empty-result')
+ sys.exit(0)
+print('ok')
+")"
+
+ if [[ "${validation}" == "ok" ]]; then
+ AddResult "thanos-query" "pass"
+ elif [[ "${validation}" == "empty-result" ]]; then
+ AddResult "thanos-query" "fail" "Thanos query succeeded via ${via} but returned empty result vector"
+ elif [[ "${validation}" == "parse-error" || -z "${validation}" ]]; then
+ AddResult "thanos-query" "fail" "Thanos query via ${via} returned unparseable response"
+ else
+ AddResult "thanos-query" "fail" "Thanos query via ${via} returned ${validation}"
+ fi
+ true
+}
+
+function CheckThanosQuery () {
+ : "=== Check 6: Thanos query functional ==="
+
+ typeset routeJson=""
+ routeJson="$(oc get routes -n "${OBS_NAMESPACE}" -o json)" || true
+ typeset queryRoute=""
+ if [[ -n "${routeJson}" ]]; then
+ queryRoute="$(printf '%s' "${routeJson}" | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+routes=d.get('items',[])
+exact=[r for r in routes if r['metadata']['name']=='observability-thanos-query']
+if exact:
+ print(exact[0]['spec']['host'])
+ sys.exit(0)
+fuzzy=[r for r in routes if 'thanos' in r['metadata']['name'] and 'query' in r['metadata']['name']]
+print(fuzzy[0]['spec']['host'] if fuzzy else '')
+")"
+ fi
+
+ if [[ -z "${queryRoute}" ]]; then
+ : "No external route found; trying internal query service"
+
+ typeset queryFrontendJson=''
+ queryFrontendJson="$(oc get pods -n "${OBS_NAMESPACE}" \
+ -l app.kubernetes.io/name=thanos-query-frontend -o json)" || true
+ typeset queryFrontendPod=''
+ if [[ -n "${queryFrontendJson}" ]]; then
+ queryFrontendPod="$(printf '%s' "${queryFrontendJson}" | python3 -c "
+import sys,json
+items=json.load(sys.stdin).get('items',[])
+print(items[0]['metadata']['name'] if items else '')
+")"
+ fi
+
+ typeset queryResult=""
+ if [[ -n "${queryFrontendPod}" ]]; then
+ queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryFrontendPod}" \
+ -- curl -sk "http://localhost:9090/api/v1/query?query=up")" || true
+ fi
+
+ if [[ -z "${queryResult}" ]]; then
+ typeset allQueryPods=''
+ allQueryPods="$(oc get pods -n "${OBS_NAMESPACE}" --no-headers)" || true
+ typeset queryPod=''
+ if [[ -n "${allQueryPods}" ]]; then
+ queryPod="$(printf '%s' "${allQueryPods}" \
+ | awk '/thanos-query/ && !/frontend/ {print $1; exit}')"
+ fi
+ if [[ -n "${queryPod}" ]]; then
+ queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryPod}" \
+ -- curl -sk "http://localhost:9090/api/v1/query?query=up")" || true
+ fi
+ fi
+
+ if [[ -z "${queryResult}" ]]; then
+ AddResult "thanos-query" "skip" "Cannot reach Thanos query endpoint (no route, exec failed)"
+ return
+ fi
+
+ ValidateThanosResponse "${queryResult}" "exec"
+ return
+ fi
+
+ typeset token=""
+ token="$(oc whoami -t)" || true
+
+ typeset responseBody=""
+ typeset httpCode=""
+ responseBody="$(curl -sk -w '\n%{http_code}' \
+ -H "Authorization: Bearer ${token}" \
+ "https://${queryRoute}/api/v1/query?query=up" \
+ --max-time 30)" || true
+
+ httpCode="$(echo "${responseBody}" | tail -1)"
+ responseBody="$(echo "${responseBody}" | sed '$d')"
+
+ if [[ "${httpCode}" != "200" ]]; then
+ if [[ "${httpCode}" =~ ^(401|403)$ ]]; then
+ AddResult "thanos-query" "fail" "Thanos query route auth failed (HTTP ${httpCode}); no data flow verified"
+ else
+ AddResult "thanos-query" "fail" "Thanos query route unreachable (HTTP ${httpCode:-timeout})"
+ fi
+ return
+ fi
+
+ ValidateThanosResponse "${responseBody}" "route"
+ true
+}
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+function Main () {
+ if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then
+ export KUBECONFIG="${SHARED_DIR}/kubeconfig"
+ fi
+
+ : "ACM Observability + ODF Interop Validation starting"
+ : "ACM namespace: ${ACM_NAMESPACE}"
+ : "Observability namespace: ${OBS_NAMESPACE}"
+ : "ODF namespace: ${ODF_NAMESPACE}"
+ : "Artifacts dir: ${ARTIFACT_DIR}"
+
+ CheckRgwReady || true
+ CheckMcoReady || true
+ CheckStorageEndpoint || true
+ CheckThanosHealth || true
+ CheckObcBound || true
+ CheckThanosQuery || true
+
+ WriteJunit
+
+ typeset -i hasAnyFail=0
+ typeset r=""
+ for r in "${tcResultsArr[@]}"; do
+ if [[ "${r}" == "fail" ]]; then
+ hasAnyFail=1
+ break
+ fi
+ done
+
+ if (( hasAnyFail )); then
+ : "ACM Observability + ODF Interop: SOME CHECKS FAILED"
+ exit 1
+ fi
+
+ : "ACM Observability + ODF Interop: ALL PASSED"
+ exit 0
+}
+
+Main "$@"
diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json
new file mode 100644
index 0000000000000..d2d17d0196b23
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json
@@ -0,0 +1,11 @@
+{
+ "path": "interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml",
+ "owners": {
+ "approvers": [
+ "cspi-qe-ocp-lp"
+ ],
+ "reviewers": [
+ "cspi-qe-ocp-lp"
+ ]
+ }
+}
\ No newline at end of file
diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml
new file mode 100644
index 0000000000000..3276ddc73dbd1
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml
@@ -0,0 +1,36 @@
+ref:
+ as: interop-opp-observability-odf
+ from: cli
+ grace_period: 30s
+ commands: interop-opp-observability-odf-commands.sh
+ timeout: 10m
+ env:
+ - name: ACM_NAMESPACE
+ default: "open-cluster-management"
+ documentation: Namespace where ACM is installed
+ - name: OBS_NAMESPACE
+ default: "open-cluster-management-observability"
+ documentation: Namespace where ACM Observability components run
+ - name: ODF_NAMESPACE
+ default: "openshift-storage"
+ documentation: Namespace where ODF is installed
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ best_effort: true
+ documentation: |-
+ Validates the cross-product integration surface between ACM Observability
+ (Thanos) and ODF (Ceph RGW). This is a true interop test that exercises
+ the boundary where ACM consumes ODF-provided object storage.
+
+ Checks performed:
+ 1. ODF Ceph RGW infrastructure ready (CephObjectStore, RGW pods, StorageClass)
+ 2. MultiClusterObservability CR exists and is Ready
+ 3. Object storage secret references an ODF-backed endpoint (not MinIO)
+ 4. Thanos and observability components healthy (receive, compact, store,
+ query, alertmanager, rbac-query-proxy)
+ 5. ObjectBucketClaim used by observability is Bound
+ 6. Thanos query endpoint functional (metrics data flow)
+
+ Produces JUnit XML for Prow / Sippy / TestGrid consumption.
diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
index 156a7f9b28caa..36441844fcd92 100755
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
@@ -1,6 +1,5 @@
#!/bin/bash
-set -euo pipefail
-shopt -s inherit_errexit
+set -euo pipefail; shopt -s inherit_errexit
# ---------------------------------------------------------------------------
# ODF Health Check (7-point gate)
@@ -105,7 +104,7 @@ function CheckOdfCsv () {
typeset csvPhase=""
if ! csvPhase="$(oc get csv -n "${ODF_NAMESPACE}" -o json | python3 -c "
import sys,json,re; d=json.load(sys.stdin)
-m=[i for i in d.get('items',[]) if re.match(r'^(odf-|ocs-)operator',i['metadata']['name'])]
+m=[i for i in d.get('items',[]) if re.match(r'^(odf-operator|ocs-operator)',i['metadata']['name'])]
print((m[0].get('status',{}).get('phase','NotFound')) if m else 'NotFound')
")"; then
AddResult "odf-csv-phase" "fail" "Failed to query ODF CSVs in ${ODF_NAMESPACE}"
@@ -485,6 +484,32 @@ print(d['items'][0].get('status',{}).get('ceph',{}).get('health','unknown') if d
# Main
# ---------------------------------------------------------------------------
+function CheckOdfInstalled () {
+ if ! oc get namespace "${ODF_NAMESPACE}" &>/dev/null; then
+ return 1
+ fi
+ typeset csvJson=""
+ typeset -i ocExit=0
+ csvJson="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || ocExit=$?
+ if (( ocExit != 0 )); then
+ printf '%s\n' "Error: oc get csv failed (exit ${ocExit}) in ${ODF_NAMESPACE}" >&2
+ return 2
+ fi
+ if [[ -z "${csvJson}" ]]; then
+ printf '%s\n' "Error: oc get csv returned empty output in ${ODF_NAMESPACE}" >&2
+ return 2
+ fi
+ typeset csvCount=""
+ if ! csvCount="$(printf '%s' "${csvJson}" | python3 -c "
+import sys,json,re; d=json.load(sys.stdin)
+print(len([i for i in d.get('items',[]) if re.match(r'^(odf-operator|ocs-operator)',i['metadata']['name'])]))
+")"; then
+ printf '%s\n' "Error: failed to parse CSV JSON from ${ODF_NAMESPACE}" >&2
+ return 2
+ fi
+ [[ "${csvCount}" -gt 0 ]]
+}
+
function Main () {
if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then
export KUBECONFIG="${SHARED_DIR}/kubeconfig"
@@ -494,6 +519,26 @@ function Main () {
: "Namespace: ${ODF_NAMESPACE}"
: "Artifacts dir: ${ARTIFACT_DIR}"
+ typeset -i odfProbeResult=0
+ CheckOdfInstalled || odfProbeResult=$?
+ if (( odfProbeResult == 2 )); then
+ : "ODF Health Check: PROBE ERROR (cannot determine ODF state)"
+ exit 1
+ fi
+ if (( odfProbeResult == 1 )); then
+ typeset skipMsg="ODF is not installed (no ODF/OCS CSV in ${ODF_NAMESPACE})"
+ typeset -a checkNames=("odf-csv-phase" "storagecluster-ready" "cephcluster-health"
+ "storageclasses-available" "pvc-provision-rbd" "pvc-provision-cephfs"
+ "noobaa-s3-functional" "ceph-health-detail")
+ typeset name=""
+ for name in "${checkNames[@]}"; do
+ AddResult "${name}" "skip" "${skipMsg}"
+ done
+ WriteJunit
+ : "ODF Health Check: ALL SKIPPED (ODF not installed)"
+ exit 0
+ fi
+
CheckOdfCsv || true
CheckStorageCluster || true
CheckCephCluster || true
diff --git a/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh b/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh
index 5a78660adc402..45dcea495679c 100755
--- a/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh
+++ b/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -eux -o pipefail
+set -euo pipefail
shopt -s inherit_errexit
# ---------------------------------------------------------------------------
@@ -84,9 +84,20 @@ function CheckCentralRoute () {
WaitFor "Central route" CheckCentralRoute
-set +x
-centralUrl="$(oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}')"
-set -x
+typeset -i routeElapsed=0
+centralUrl=""
+while [[ -z "${centralUrl}" ]]; do
+ centralUrl="$(oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}' 2>/dev/null)" || true
+ if [[ -n "${centralUrl}" ]]; then
+ break
+ fi
+ if (( routeElapsed >= 30 )); then
+ echo "[readiness] FATAL: Central route host empty after 30s"
+ exit 1
+ fi
+ sleep 5
+ (( routeElapsed += 5 )) || true
+done
echo "[readiness] Central route discovered"
# ---------------------------------------------------------------------------
@@ -94,10 +105,8 @@ echo "[readiness] Central route discovered"
# ---------------------------------------------------------------------------
typeset roxAdminPassword=""
echo "[readiness] Extracting roxAdminPassword..."
-set +x
roxAdminPassword="$(oc get secret -n "${centralNs}" central-htpasswd \
-o jsonpath='{.data.password}' | base64 -d)"
-set -x
if [[ -z "${roxAdminPassword}" ]]; then
echo "[readiness] FATAL: could not extract roxAdminPassword"
@@ -109,12 +118,10 @@ echo "[readiness] roxAdminPassword extracted successfully"
# Check 2: Central API health (authenticated v1/metadata)
# ---------------------------------------------------------------------------
function CheckCentralApi () {
- set +x
typeset httpCode=""
httpCode="$(curl -sk -o /dev/null -w '%{http_code}' \
-u "admin:${roxAdminPassword}" \
- "https://${centralUrl}/v1/metadata" --max-time 10)" || { set -x; return 1; }
- set -x
+ "https://${centralUrl}/v1/metadata" --max-time 10)" || return 1
[[ "${httpCode}" == "200" ]]
}
@@ -124,12 +131,10 @@ WaitFor "Central API health (v1/metadata)" CheckCentralApi
# Check 3: At least 1 secured cluster connected
# ---------------------------------------------------------------------------
function CheckClustersConnected () {
- set +x
typeset clusterCount=""
clusterCount="$(curl -sk -u "admin:${roxAdminPassword}" \
"https://${centralUrl}/v1/clusters" --max-time 10 \
- | JsonLength clusters)" || { set -x; return 1; }
- set -x
+ | JsonLength clusters)" || return 1
[[ "${clusterCount}" -ge 1 ]]
}
@@ -184,12 +189,10 @@ WaitFor "sensor pods Running in ${scNs}" CheckSensorPods
# Check 5: Default policies loaded (count > 80)
# ---------------------------------------------------------------------------
function CheckPoliciesLoaded () {
- set +x
typeset policyCount=""
policyCount="$(curl -sk -u "admin:${roxAdminPassword}" \
"https://${centralUrl}/v1/policies?query=" --max-time 10 \
- | JsonLength policies)" || { set -x; return 1; }
- set -x
+ | JsonLength policies)" || return 1
echo "[readiness] policy count: ${policyCount}"
[[ "${policyCount}" -gt 80 ]]
}
@@ -198,10 +201,8 @@ WaitFor "default policies loaded (>80)" CheckPoliciesLoaded
echo "[readiness] Writing connection details to SHARED_DIR..."
-set +x
echo "${roxAdminPassword}" > "${SHARED_DIR}/ROX_ADMIN_PASSWORD"
echo "${centralUrl}" > "${SHARED_DIR}/CENTRAL_URL"
-set -x
echo "${centralNs}" > "${SHARED_DIR}/CENTRAL_NS"
echo "${scNs}" > "${SHARED_DIR}/SC_NS"
diff --git a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh
index 3907a7d723dab..972ab22b7f8a0 100755
--- a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh
+++ b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -eux -o pipefail
+set -euo pipefail
shopt -s inherit_errexit
if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then
@@ -8,10 +8,8 @@ fi
echo "[smoke] Reading connection details from SHARED_DIR..."
-set +x
CENTRAL_URL="$(cat "${SHARED_DIR}/CENTRAL_URL")"
ROX_ADMIN_PASSWORD="$(cat "${SHARED_DIR}/ROX_ADMIN_PASSWORD")"
-set -x
echo "[smoke] Connection details loaded from SHARED_DIR"
@@ -51,7 +49,6 @@ grep -q 'DEFAULT_CLUSTER_NAME = "local-cluster"' \
/tmp/stackrox/qa-tests-backend/src/main/groovy/services/ClusterService.groovy \
|| { echo "[smoke] FATAL: DEFAULT_CLUSTER_NAME patch failed"; exit 1; }
-set +x
export API_HOSTNAME="${CENTRAL_URL}"
export API_PORT="443"
export ROX_USERNAME="admin"
@@ -72,7 +69,6 @@ if [[ -f /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE
GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2)"
export GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2
fi
-set -x
cd /tmp/stackrox/qa-tests-backend
@@ -103,13 +99,4 @@ if [[ -d build/reports/tests/testSMOKE ]]; then
fi
echo "[smoke] Test run finished with exit code: ${testExit}"
-if [[ "${testExit}" -ne 0 ]] && [[ -d build/test-results/testSMOKE ]]; then
- typeset total=""
- total="$(find build/test-results/testSMOKE -name '*.xml' -exec grep -l 'testcase' {} \; | wc -l)"
- if [[ "${total}" -gt 0 ]]; then
- echo "[smoke] Tests executed and results captured; treating as informational (exit 0)."
- echo "[smoke] Review JUnit XML in ARTIFACT_DIR for individual test failures."
- exit 0
- fi
-fi
exit "${testExit}"
diff --git a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml
index 74d11f0815aa9..329a34c9d7a71 100644
--- a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml
+++ b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml
@@ -11,6 +11,7 @@ ref:
memory: 4Gi
from: acs-smoke-runner
timeout: 1h0m0s
+ best_effort: true
documentation: |-
Run the ACS qa-tests-backend SMOKE suite against a live ACS
instance. Reads connection credentials from SHARED_DIR