From 2ef52bbc28d354183dd76bc5d65e96bf7bc6f52f Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 19 Aug 2026 09:53:33 -0500 Subject: [PATCH 01/29] INTEROP-9416: Add opp-observability-odf step for ACM+ODF interop validation Validates the cross-product integration between ACM Observability (Thanos) and ODF (Ceph RGW/NooBaa) object storage with a 6-point gate: 1. ODF Ceph RGW infrastructure ready 2. MultiClusterObservability CR exists and is Ready 3. Object storage secret references ODF-backed endpoint 4. Thanos components healthy (with missing-component detection) 5. ObjectBucketClaim bound 6. Thanos query endpoint functional (strict HTTP 200 only) Addresses CodeRabbit review findings from v1: - Do not leak decoded secret content into JUnit artifacts - Detect missing Thanos components instead of silently passing - Fix unreachable OBC fallback by checking parsed item count - Only accept HTTP 200 for functional query check (not 401/403) - Use only metricObjectStorage.name for secret lookup (not key) --- .../interop/opp/observability-odf/OWNERS | 3 + .../interop-opp-observability-odf-commands.sh | 524 ++++++++++++++++++ ...op-opp-observability-odf-ref.metadata.json | 11 + .../interop-opp-observability-odf-ref.yaml | 36 ++ 4 files changed, 574 insertions(+) create mode 100644 ci-operator/step-registry/interop/opp/observability-odf/OWNERS create mode 100755 ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh create mode 100644 ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json create mode 100644 ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml diff --git a/ci-operator/step-registry/interop/opp/observability-odf/OWNERS b/ci-operator/step-registry/interop/opp/observability-odf/OWNERS new file mode 100644 index 0000000000000..41d144d3728a2 --- /dev/null +++ b/ci-operator/step-registry/interop/opp/observability-odf/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh new file mode 100755 index 0000000000000..3510619b51e83 --- /dev/null +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -0,0 +1,524 @@ +#!/bin/bash +set -euo pipefail +shopt -s inherit_errexit + +# --------------------------------------------------------------------------- +# ACM Observability + ODF Interop Validation (6-point gate) +# +# Validates that ACM's observability stack (Thanos) correctly uses +# ODF-provided object storage (Ceph RGW or NooBaa S3) as its backend. +# This is a cross-product interop test exercising the ACM <-> ODF boundary. +# +# Produces JUnit XML consumed by Prow / Sippy / TestGrid. +# --------------------------------------------------------------------------- + +ACM_NAMESPACE="${ACM_NAMESPACE:-open-cluster-management}" +OBS_NAMESPACE="${OBS_NAMESPACE:-open-cluster-management-observability}" +ODF_NAMESPACE="${ODF_NAMESPACE:-openshift-storage}" + +typeset junitFile="${ARTIFACT_DIR}/junit_observability_odf.xml" + +typeset -a tcNamesArr=() +typeset -a tcResultsArr=() +typeset -a tcMessagesArr=() + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +function AddResult () { + typeset name="${1:-}"; (($#)) && shift + typeset result="${1:-}"; (($#)) && shift + typeset message="${1:-}"; (($#)) && shift + tcNamesArr+=("${name}") + tcResultsArr+=("${result}") + tcMessagesArr+=("${message}") + true +} + +function XmlEscape () { + typeset text="${1:-}"; (($#)) && shift + text="${text//&/&}" + text="${text///>}" + text="${text//\"/"}" + text="${text//\'/'}" + printf '%s' "${text}" + true +} + +function WriteJunit () { + typeset -i total=${#tcNamesArr[@]} + typeset -i failCount=0 + typeset -i skipCount=0 + typeset r="" + for r in "${tcResultsArr[@]}"; do + if [[ "${r}" == "fail" ]]; then + (( failCount++ )) || true + elif [[ "${r}" == "skip" ]]; then + (( skipCount++ )) || true + fi + done + + { + echo '' + echo "" + typeset -i i=0 + for i in "${!tcNamesArr[@]}"; do + typeset name="" + name="$(XmlEscape "${tcNamesArr[$i]}")" + echo " " + if [[ "${tcResultsArr[$i]}" == "fail" ]]; then + typeset msg="" + msg="$(XmlEscape "${tcMessagesArr[$i]}")" + echo " " + elif [[ "${tcResultsArr[$i]}" == "skip" ]]; then + typeset msg="" + msg="$(XmlEscape "${tcMessagesArr[$i]}")" + echo " " + fi + echo " " + done + echo "" + } > "${junitFile}" + : "JUnit XML written to ${junitFile}" +} + +# shellcheck disable=SC2317 +function CollectExitArtifacts () { + : "Collecting observability + ODF diagnostics..." + oc get multiclusterobservabilities.observability.open-cluster-management.io --all-namespaces -o yaml > "${ARTIFACT_DIR}/mco.yaml" 2>/dev/null || true + oc get pods -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-pods.yaml" 2>/dev/null || true + oc get obc -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-obc.yaml" 2>/dev/null || true + oc get secret -n "${OBS_NAMESPACE}" -o name > "${ARTIFACT_DIR}/obs-secrets-list.txt" 2>/dev/null || true + oc get cephobjectstore -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/cephobjectstore.yaml" 2>/dev/null || true + oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw -o yaml > "${ARTIFACT_DIR}/rgw-pods.yaml" 2>/dev/null || true + oc get noobaa -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/noobaa.yaml" 2>/dev/null || true +} + +trap CollectExitArtifacts EXIT + +# --------------------------------------------------------------------------- +# Check 1: ODF Ceph RGW infrastructure ready +# --------------------------------------------------------------------------- + +function CheckRgwReady () { + : "=== Check 1: ODF Ceph RGW infrastructure ===" + + typeset rgwPhase="" + if ! rgwPhase="$(oc get cephobjectstore -n "${ODF_NAMESPACE}" -o json 2>/dev/null | python3 -c " +import sys,json +d=json.load(sys.stdin) +items=d.get('items',[]) +if not items: + print('NotFound') +else: + print(items[0].get('status',{}).get('phase','Unknown')) +")"; then + AddResult "rgw-ready" "fail" "Failed to query CephObjectStore" + return + fi + + if [[ "${rgwPhase}" == "NotFound" ]]; then + AddResult "rgw-ready" "skip" "No CephObjectStore found; ODF RGW not deployed" + return + fi + + typeset failMsg="" + if [[ "${rgwPhase}" != "Ready" ]]; then + failMsg="CephObjectStore phase=${rgwPhase} (expected Ready)" + fi + + typeset rgwPodCount="" + rgwPodCount="$(oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw \ + --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l)" || true + + if [[ "${rgwPodCount}" -eq 0 ]]; then + typeset rgwMsg="No rook-ceph-rgw pods Running in ${ODF_NAMESPACE}" + if [[ -n "${failMsg}" ]]; then + failMsg="${failMsg}; ${rgwMsg}" + else + failMsg="${rgwMsg}" + fi + fi + + typeset scExists="" + scExists="$(oc get sc ocs-storagecluster-ceph-rgw -o name 2>/dev/null)" || true + if [[ -z "${scExists}" ]]; then + typeset scMsg="StorageClass ocs-storagecluster-ceph-rgw not found" + if [[ -n "${failMsg}" ]]; then + failMsg="${failMsg}; ${scMsg}" + else + failMsg="${scMsg}" + fi + fi + + if [[ -z "${failMsg}" ]]; then + : "PASS: CephObjectStore Ready, RGW pods Running, StorageClass exists" + AddResult "rgw-ready" "pass" + else + AddResult "rgw-ready" "fail" "${failMsg}" + fi + true +} + +# --------------------------------------------------------------------------- +# Check 2: MultiClusterObservability CR exists and is Ready +# --------------------------------------------------------------------------- + +function CheckMcoReady () { + : "=== Check 2: MultiClusterObservability CR ===" + + typeset mcoStatus="" + if ! mcoStatus="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ + --all-namespaces -o json 2>/dev/null | python3 -c " +import sys,json +d=json.load(sys.stdin) +items=d.get('items',[]) +if not items: + print('NotFound') +else: + conds=items[0].get('status',{}).get('conditions',[]) + ready=[c for c in conds if c.get('type')=='Ready'] + print(ready[0].get('status','Unknown') if ready else 'NoCondition') +")"; then + AddResult "mco-ready" "fail" "Failed to query MultiClusterObservability CR" + return + fi + + if [[ "${mcoStatus}" == "True" ]]; then + : "PASS: MultiClusterObservability Ready=True" + AddResult "mco-ready" "pass" + elif [[ "${mcoStatus}" == "NotFound" ]]; then + AddResult "mco-ready" "skip" "MultiClusterObservability CR not found; observability not deployed" + else + AddResult "mco-ready" "fail" "MultiClusterObservability Ready=${mcoStatus} (expected True)" + fi + true +} + +# --------------------------------------------------------------------------- +# Check 3: Object storage secret references ODF-backed endpoint +# --------------------------------------------------------------------------- + +function CheckStorageEndpoint () { + : "=== Check 3: Object storage endpoint ===" + + typeset secretName="" + if ! secretName="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ + --all-namespaces -o json 2>/dev/null | python3 -c " +import sys,json +d=json.load(sys.stdin) +items=d.get('items',[]) +if not items: + print('') +else: + spec=items[0].get('spec',{}) + storage=spec.get('storageConfig',{}).get('metricObjectStorage',{}) + print(storage.get('name','')) +")"; then + AddResult "storage-endpoint" "fail" "Failed to read MCO storage config" + return + fi + + if [[ -z "${secretName}" ]]; then + AddResult "storage-endpoint" "skip" "No metricObjectStorage secret configured in MCO" + return + fi + + typeset endpointCheck="" + endpointCheck="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json 2>/dev/null \ + | python3 -c " +import sys,json,base64,re +d=json.load(sys.stdin) +odf_pattern=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs)', re.IGNORECASE) +for k,v in d.get('data',{}).items(): + decoded=base64.b64decode(v).decode('utf-8','replace') + if 'endpoint' in decoded.lower() or 'bucket' in decoded.lower(): + if odf_pattern.search(decoded): + print('odf-backed') + else: + print('external') + sys.exit(0) +print('no-endpoint') +" 2>/dev/null)" || true + + if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then + AddResult "storage-endpoint" "fail" "Secret ${secretName} exists but no endpoint/bucket config found" + return + fi + + if [[ "${endpointCheck}" == "odf-backed" ]]; then + AddResult "storage-endpoint" "pass" + else + AddResult "storage-endpoint" "fail" "Storage endpoint does not reference ODF-backed service" + fi + true +} + +# --------------------------------------------------------------------------- +# Check 4: Thanos components healthy +# --------------------------------------------------------------------------- + +function CheckThanosHealth () { + : "=== Check 4: Thanos components healthy ===" + + if ! oc get namespace "${OBS_NAMESPACE}" &>/dev/null; then + AddResult "thanos-health" "skip" "Observability namespace ${OBS_NAMESPACE} does not exist" + return + fi + + typeset failMsg="" + typeset -i foundCount=0 + typeset -a missingComponents=() + + typeset -a componentNames=("thanos-receive" "thanos-compact" "thanos-store" "thanos-query" "alertmanager" "rbac-query-proxy") + typeset -a componentLabels=("app=thanos-receive" "app=thanos-compact" "app=thanos-store" "app=thanos-query" "alertmanager=observability" "app=rbac-query-proxy") + + typeset -i idx=0 + for idx in "${!componentNames[@]}"; do + typeset component="${componentNames[$idx]}" + typeset labelSelector="${componentLabels[$idx]}" + + typeset podCount="" + podCount="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \ + --no-headers 2>/dev/null | wc -l)" || true + + if [[ "${podCount}" -eq 0 ]]; then + podCount="$(oc get pods -n "${OBS_NAMESPACE}" \ + --no-headers 2>/dev/null | grep -c "^${component}")" || true + fi + + if [[ "${podCount}" -eq 0 ]]; then + missingComponents+=("${component}") + continue + fi + + (( foundCount++ )) || true + + typeset notReady="" + notReady="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \ + --no-headers 2>/dev/null \ + | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')" || true + + if [[ -n "${notReady}" ]]; then + typeset compMsg="${component}: ${notReady//$'\n'/, }" + if [[ -n "${failMsg}" ]]; then + failMsg="${failMsg}; ${compMsg}" + else + failMsg="${compMsg}" + fi + fi + done + + if (( foundCount == 0 )); then + AddResult "thanos-health" "fail" "No Thanos/observability components found in ${OBS_NAMESPACE}" + elif [[ -n "${failMsg}" ]]; then + AddResult "thanos-health" "fail" "Unhealthy Thanos components: ${failMsg}" + elif (( ${#missingComponents[@]} > 0 )); then + AddResult "thanos-health" "pass" "Running (missing: ${missingComponents[*]})" + else + AddResult "thanos-health" "pass" + fi + true +} + +# --------------------------------------------------------------------------- +# Check 5: ObjectBucketClaim bound (if used by observability) +# --------------------------------------------------------------------------- + +function CheckObcBound () { + : "=== Check 5: Observability ObjectBucketClaim ===" + + typeset obcList="" + obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true + + typeset obcItemCount="" + if [[ -n "${obcList}" ]]; then + obcItemCount="$(echo "${obcList}" | python3 -c " +import sys,json +d=json.load(sys.stdin) +print(len(d.get('items',[]))) +" 2>/dev/null)" || true + fi + + if [[ "${obcItemCount:-0}" -eq 0 ]]; then + obcList="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null \ + | python3 -c " +import sys,json +d=json.load(sys.stdin) +obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()] +print(json.dumps({'items':obs})) +" 2>/dev/null)" || true + obcItemCount="$(echo "${obcList}" | python3 -c " +import sys,json +d=json.load(sys.stdin) +print(len(d.get('items',[]))) +" 2>/dev/null)" || true + fi + + if [[ "${obcItemCount:-0}" -eq 0 ]]; then + AddResult "obc-bound" "skip" "No ObjectBucketClaim found for observability" + return + fi + + typeset obcStatus="" + if ! obcStatus="$(echo "${obcList}" | python3 -c " +import sys,json +d=json.load(sys.stdin) +items=d.get('items',[]) +if not items: + print('NotFound') +else: + results=[] + for i in items: + name=i['metadata']['name'] + phase=i.get('status',{}).get('phase','Unknown') + results.append(f'{name}={phase}') + print(';'.join(results)) +")"; then + AddResult "obc-bound" "fail" "Failed to parse OBC status" + return + fi + + if [[ "${obcStatus}" == "NotFound" ]]; then + AddResult "obc-bound" "skip" "No ObjectBucketClaim found for observability" + return + fi + + typeset unboundObcs="" + unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | grep -v '=Bound$' || true)" + + if [[ -z "${unboundObcs}" ]]; then + : "PASS: All observability OBCs bound: ${obcStatus}" + AddResult "obc-bound" "pass" + else + AddResult "obc-bound" "fail" "Unbound OBCs: ${unboundObcs//$'\n'/, }" + fi + true +} + +# --------------------------------------------------------------------------- +# Check 6: Thanos metrics query functional (basic data flow) +# --------------------------------------------------------------------------- + +function CheckThanosQuery () { + : "=== Check 6: Thanos query functional ===" + + typeset queryRoute="" + queryRoute="$(oc get routes -n "${OBS_NAMESPACE}" -o json 2>/dev/null \ + | python3 -c " +import sys,json +d=json.load(sys.stdin) +routes=[i for i in d.get('items',[]) if 'query' in i['metadata'].get('name','').lower() or 'observ' in i['metadata'].get('name','').lower()] +if routes: + print(routes[0]['spec']['host']) +else: + print('') +" 2>/dev/null)" || true + + if [[ -z "${queryRoute}" ]]; then + typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090" + : "No external route found; trying internal service: ${svcHost}" + + typeset token="" + token="$(oc whoami -t 2>/dev/null)" || true + + typeset queryResult="" + queryResult="$(oc exec -n "${OBS_NAMESPACE}" \ + "$(oc get pods -n "${OBS_NAMESPACE}" -l app.kubernetes.io/name=thanos-query-frontend \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo '')" \ + -- wget -qO- --no-check-certificate \ + "http://localhost:9090/api/v1/query?query=up" 2>/dev/null)" || true + + if [[ -z "${queryResult}" ]]; then + queryResult="$(oc exec -n "${OBS_NAMESPACE}" \ + "$(oc get pods -n "${OBS_NAMESPACE}" --no-headers 2>/dev/null \ + | grep 'thanos-query' | grep -v 'frontend' | head -1 | awk '{print $1}')" \ + -- wget -qO- --no-check-certificate \ + "http://localhost:9090/api/v1/query?query=up" 2>/dev/null)" || true + fi + + if [[ -z "${queryResult}" ]]; then + AddResult "thanos-query" "skip" "Cannot reach Thanos query endpoint (no route, exec failed)" + return + fi + + typeset queryStatus="" + queryStatus="$(echo "${queryResult}" | python3 -c " +import sys,json +d=json.load(sys.stdin) +print(d.get('status','')) +" 2>/dev/null)" || true + + if [[ "${queryStatus}" == "success" ]]; then + : "PASS: Thanos query returned success via exec" + AddResult "thanos-query" "pass" + else + AddResult "thanos-query" "fail" "Thanos query returned status=${queryStatus}" + fi + return + fi + + typeset token="" + token="$(oc whoami -t 2>/dev/null)" || true + + typeset httpCode="" + httpCode="$(curl -sk -o /dev/null -w '%{http_code}' \ + -H "Authorization: Bearer ${token}" \ + "https://${queryRoute}/api/v1/query?query=up" \ + --max-time 30)" || true + + if [[ "${httpCode}" == "200" ]]; then + AddResult "thanos-query" "pass" + elif [[ "${httpCode}" =~ ^(401|403)$ ]]; then + AddResult "thanos-query" "fail" "Thanos query route reachable but auth failed (HTTP ${httpCode}); no data flow verified" + else + AddResult "thanos-query" "fail" "Thanos query unreachable at ${queryRoute} (HTTP ${httpCode:-timeout})" + fi + true +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +function Main () { + if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then + export KUBECONFIG="${SHARED_DIR}/kubeconfig" + fi + + : "ACM Observability + ODF Interop Validation starting" + : "ACM namespace: ${ACM_NAMESPACE}" + : "Observability namespace: ${OBS_NAMESPACE}" + : "ODF namespace: ${ODF_NAMESPACE}" + : "Artifacts dir: ${ARTIFACT_DIR}" + + CheckRgwReady || true + CheckMcoReady || true + CheckStorageEndpoint || true + CheckThanosHealth || true + CheckObcBound || true + CheckThanosQuery || true + + WriteJunit + + typeset -i hasAnyFail=0 + typeset r="" + for r in "${tcResultsArr[@]}"; do + if [[ "${r}" == "fail" ]]; then + hasAnyFail=1 + break + fi + done + + if (( hasAnyFail )); then + : "ACM Observability + ODF Interop: SOME CHECKS FAILED" + exit 1 + fi + + : "ACM Observability + ODF Interop: ALL PASSED" + exit 0 +} + +Main "$@" diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json new file mode 100644 index 0000000000000..d2d17d0196b23 --- /dev/null +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml new file mode 100644 index 0000000000000..3276ddc73dbd1 --- /dev/null +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml @@ -0,0 +1,36 @@ +ref: + as: interop-opp-observability-odf + from: cli + grace_period: 30s + commands: interop-opp-observability-odf-commands.sh + timeout: 10m + env: + - name: ACM_NAMESPACE + default: "open-cluster-management" + documentation: Namespace where ACM is installed + - name: OBS_NAMESPACE + default: "open-cluster-management-observability" + documentation: Namespace where ACM Observability components run + - name: ODF_NAMESPACE + default: "openshift-storage" + documentation: Namespace where ODF is installed + resources: + requests: + cpu: 100m + memory: 200Mi + best_effort: true + documentation: |- + Validates the cross-product integration surface between ACM Observability + (Thanos) and ODF (Ceph RGW). This is a true interop test that exercises + the boundary where ACM consumes ODF-provided object storage. + + Checks performed: + 1. ODF Ceph RGW infrastructure ready (CephObjectStore, RGW pods, StorageClass) + 2. MultiClusterObservability CR exists and is Ready + 3. Object storage secret references an ODF-backed endpoint (not MinIO) + 4. Thanos and observability components healthy (receive, compact, store, + query, alertmanager, rbac-query-proxy) + 5. ObjectBucketClaim used by observability is Bound + 6. Thanos query endpoint functional (metrics data flow) + + Produces JUnit XML for Prow / Sippy / TestGrid consumption. From 248c9c1fc40f726bc3548074bbcf6a57ff95a27f Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 19 Aug 2026 10:14:42 -0500 Subject: [PATCH 02/29] INTEROP-9416: Validate Thanos query response body, not just HTTP status Address CodeRabbit v2 finding: both the exec and route code paths now parse the Thanos response and require status=success, resultType=vector, and a non-empty result array before reporting pass. Shared validation extracted into ValidateThanosResponse. --- .../interop-opp-observability-odf-commands.sh | 72 +++++++++++++------ 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh index 3510619b51e83..3955a9e7adec3 100755 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -402,6 +402,43 @@ else: # Check 6: Thanos metrics query functional (basic data flow) # --------------------------------------------------------------------------- +function ValidateThanosResponse () { + typeset body="${1:-}"; (($#)) && shift + typeset via="${1:-unknown}"; (($#)) && shift + + typeset validation="" + validation="$(echo "${body}" | python3 -c " +import sys,json +try: + d=json.load(sys.stdin) +except Exception: + print('parse-error') + sys.exit(0) +if d.get('status')!='success': + print('status=' + str(d.get('status',''))) + sys.exit(0) +data=d.get('data',{}) +if data.get('resultType')!='vector': + print('resultType=' + str(data.get('resultType',''))) + sys.exit(0) +result=data.get('result',[]) +if not isinstance(result,list) or len(result)==0: + print('empty-result') + sys.exit(0) +print('ok') +" 2>/dev/null)" || true + + if [[ "${validation}" == "ok" ]]; then + AddResult "thanos-query" "pass" + elif [[ "${validation}" == "empty-result" ]]; then + AddResult "thanos-query" "fail" "Thanos query succeeded via ${via} but returned empty result vector" + elif [[ "${validation}" == "parse-error" || -z "${validation}" ]]; then + AddResult "thanos-query" "fail" "Thanos query via ${via} returned unparseable response" + else + AddResult "thanos-query" "fail" "Thanos query via ${via} returned ${validation}" + fi +} + function CheckThanosQuery () { : "=== Check 6: Thanos query functional ===" @@ -444,38 +481,33 @@ else: return fi - typeset queryStatus="" - queryStatus="$(echo "${queryResult}" | python3 -c " -import sys,json -d=json.load(sys.stdin) -print(d.get('status','')) -" 2>/dev/null)" || true - - if [[ "${queryStatus}" == "success" ]]; then - : "PASS: Thanos query returned success via exec" - AddResult "thanos-query" "pass" - else - AddResult "thanos-query" "fail" "Thanos query returned status=${queryStatus}" - fi + ValidateThanosResponse "${queryResult}" "exec" return fi typeset token="" token="$(oc whoami -t 2>/dev/null)" || true + typeset responseBody="" typeset httpCode="" - httpCode="$(curl -sk -o /dev/null -w '%{http_code}' \ + responseBody="$(curl -sk -w '\n%{http_code}' \ -H "Authorization: Bearer ${token}" \ "https://${queryRoute}/api/v1/query?query=up" \ --max-time 30)" || true - if [[ "${httpCode}" == "200" ]]; then - AddResult "thanos-query" "pass" - elif [[ "${httpCode}" =~ ^(401|403)$ ]]; then - AddResult "thanos-query" "fail" "Thanos query route reachable but auth failed (HTTP ${httpCode}); no data flow verified" - else - AddResult "thanos-query" "fail" "Thanos query unreachable at ${queryRoute} (HTTP ${httpCode:-timeout})" + httpCode="$(echo "${responseBody}" | tail -1)" + responseBody="$(echo "${responseBody}" | sed '$d')" + + if [[ "${httpCode}" != "200" ]]; then + if [[ "${httpCode}" =~ ^(401|403)$ ]]; then + AddResult "thanos-query" "fail" "Thanos query route auth failed (HTTP ${httpCode}); no data flow verified" + else + AddResult "thanos-query" "fail" "Thanos query unreachable at ${queryRoute} (HTTP ${httpCode:-timeout})" + fi + return fi + + ValidateThanosResponse "${responseBody}" "route" true } From 11354986b2c1fefc6c6fdf9e11b042934e173384 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 19 Aug 2026 10:28:16 -0500 Subject: [PATCH 03/29] INTEROP-9416: Remove cluster route from JUnit failure message Avoids exposing internal cluster URLs in published CI artifacts. --- .../observability-odf/interop-opp-observability-odf-commands.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh index 3955a9e7adec3..10741fafd48bd 100755 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -502,7 +502,7 @@ else: if [[ "${httpCode}" =~ ^(401|403)$ ]]; then AddResult "thanos-query" "fail" "Thanos query route auth failed (HTTP ${httpCode}); no data flow verified" else - AddResult "thanos-query" "fail" "Thanos query unreachable at ${queryRoute} (HTTP ${httpCode:-timeout})" + AddResult "thanos-query" "fail" "Thanos query route unreachable (HTTP ${httpCode:-timeout})" fi return fi From 6d69f25a9e88c2df90beeea8b4ac2dd6fd76d521 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 19 Aug 2026 10:53:40 -0500 Subject: [PATCH 04/29] INTEROP-9416: apply mpitt best-practice fixes - Enable xtrace (set -euxo pipefail) for CI debugging - Split pipelines to avoid masking oc failures with || true - Add xtrace bracketing around bearer token curl - Add terminal true to WriteJunit, CollectExitArtifacts, ValidateThanosResponse - Use subshell trap form for EXIT handler - Remove unused token variable in exec path - Change &>/dev/null to 2>/dev/null for namespace check --- .../interop-opp-observability-odf-commands.sh | 60 ++++++++++++------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh index 10741fafd48bd..d7c08762a5177 100755 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -euo pipefail +set -euxo pipefail shopt -s inherit_errexit # --------------------------------------------------------------------------- @@ -82,9 +82,10 @@ function WriteJunit () { echo "" } > "${junitFile}" : "JUnit XML written to ${junitFile}" + true } -# shellcheck disable=SC2317 +# shellcheck disable=SC2317,SC2329 function CollectExitArtifacts () { : "Collecting observability + ODF diagnostics..." oc get multiclusterobservabilities.observability.open-cluster-management.io --all-namespaces -o yaml > "${ARTIFACT_DIR}/mco.yaml" 2>/dev/null || true @@ -94,9 +95,10 @@ function CollectExitArtifacts () { oc get cephobjectstore -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/cephobjectstore.yaml" 2>/dev/null || true oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw -o yaml > "${ARTIFACT_DIR}/rgw-pods.yaml" 2>/dev/null || true oc get noobaa -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/noobaa.yaml" 2>/dev/null || true + true } -trap CollectExitArtifacts EXIT +trap '{( CollectExitArtifacts; true )}' EXIT # --------------------------------------------------------------------------- # Check 1: ODF Ceph RGW infrastructure ready @@ -129,9 +131,11 @@ else: failMsg="CephObjectStore phase=${rgwPhase} (expected Ready)" fi + typeset rgwPods="" + rgwPods="$(oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw \ + --field-selector=status.phase=Running --no-headers 2>/dev/null)" || true typeset rgwPodCount="" - rgwPodCount="$(oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw \ - --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l)" || true + rgwPodCount="$(printf '%s' "${rgwPods}" | grep -c . || true)" if [[ "${rgwPodCount}" -eq 0 ]]; then typeset rgwMsg="No rook-ceph-rgw pods Running in ${ODF_NAMESPACE}" @@ -226,9 +230,11 @@ else: return fi + typeset secretJson="" + secretJson="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true typeset endpointCheck="" - endpointCheck="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json 2>/dev/null \ - | python3 -c " + if [[ -n "${secretJson}" ]]; then + endpointCheck="$(printf '%s' "${secretJson}" | python3 -c " import sys,json,base64,re d=json.load(sys.stdin) odf_pattern=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs)', re.IGNORECASE) @@ -242,6 +248,7 @@ for k,v in d.get('data',{}).items(): sys.exit(0) print('no-endpoint') " 2>/dev/null)" || true + fi if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then AddResult "storage-endpoint" "fail" "Secret ${secretName} exists but no endpoint/bucket config found" @@ -263,7 +270,7 @@ print('no-endpoint') function CheckThanosHealth () { : "=== Check 4: Thanos components healthy ===" - if ! oc get namespace "${OBS_NAMESPACE}" &>/dev/null; then + if ! oc get namespace "${OBS_NAMESPACE}" -o name 2>/dev/null; then AddResult "thanos-health" "skip" "Observability namespace ${OBS_NAMESPACE} does not exist" return fi @@ -280,13 +287,17 @@ function CheckThanosHealth () { typeset component="${componentNames[$idx]}" typeset labelSelector="${componentLabels[$idx]}" + typeset podList="" + podList="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \ + --no-headers 2>/dev/null)" || true typeset podCount="" - podCount="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \ - --no-headers 2>/dev/null | wc -l)" || true + podCount="$(printf '%s' "${podList}" | grep -c . || true)" if [[ "${podCount}" -eq 0 ]]; then - podCount="$(oc get pods -n "${OBS_NAMESPACE}" \ - --no-headers 2>/dev/null | grep -c "^${component}")" || true + typeset allPods="" + allPods="$(oc get pods -n "${OBS_NAMESPACE}" \ + --no-headers 2>/dev/null)" || true + podCount="$(printf '%s' "${allPods}" | grep -c "^${component}" || true)" fi if [[ "${podCount}" -eq 0 ]]; then @@ -296,9 +307,11 @@ function CheckThanosHealth () { (( foundCount++ )) || true + typeset labeledPods="" + labeledPods="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \ + --no-headers 2>/dev/null)" || true typeset notReady="" - notReady="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \ - --no-headers 2>/dev/null \ + notReady="$(printf '%s' "${labeledPods}" \ | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')" || true if [[ -n "${notReady}" ]]; then @@ -343,18 +356,23 @@ print(len(d.get('items',[]))) fi if [[ "${obcItemCount:-0}" -eq 0 ]]; then - obcList="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null \ - | python3 -c " + typeset odfObcJson="" + odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || true + if [[ -n "${odfObcJson}" ]]; then + obcList="$(printf '%s' "${odfObcJson}" | python3 -c " import sys,json d=json.load(sys.stdin) obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()] print(json.dumps({'items':obs})) " 2>/dev/null)" || true - obcItemCount="$(echo "${obcList}" | python3 -c " + fi + if [[ -n "${obcList}" ]]; then + obcItemCount="$(echo "${obcList}" | python3 -c " import sys,json d=json.load(sys.stdin) print(len(d.get('items',[]))) " 2>/dev/null)" || true + fi fi if [[ "${obcItemCount:-0}" -eq 0 ]]; then @@ -387,7 +405,7 @@ else: fi typeset unboundObcs="" - unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | grep -v '=Bound$' || true)" + unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | grep -v '=Bound$')" || true if [[ -z "${unboundObcs}" ]]; then : "PASS: All observability OBCs bound: ${obcStatus}" @@ -437,6 +455,7 @@ print('ok') else AddResult "thanos-query" "fail" "Thanos query via ${via} returned ${validation}" fi + true } function CheckThanosQuery () { @@ -458,9 +477,6 @@ else: typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090" : "No external route found; trying internal service: ${svcHost}" - typeset token="" - token="$(oc whoami -t 2>/dev/null)" || true - typeset queryResult="" queryResult="$(oc exec -n "${OBS_NAMESPACE}" \ "$(oc get pods -n "${OBS_NAMESPACE}" -l app.kubernetes.io/name=thanos-query-frontend \ @@ -485,6 +501,7 @@ else: return fi + set +x typeset token="" token="$(oc whoami -t 2>/dev/null)" || true @@ -494,6 +511,7 @@ else: -H "Authorization: Bearer ${token}" \ "https://${queryRoute}/api/v1/query?query=up" \ --max-time 30)" || true + set -x httpCode="$(echo "${responseBody}" | tail -1)" responseBody="$(echo "${responseBody}" | sed '$d')" From 188e81731bff06f0e8fb1a8abed9361d0508ef0c Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 19 Aug 2026 10:56:03 -0500 Subject: [PATCH 05/29] INTEROP-9416: apply Chai Bot approach fixes - Check 3: parse thanos.yaml YAML with metricObjectStorage.key instead of string-matching raw secret data; add mcg to ODF pattern - Check 1: add NooBaa readiness fallback when RGW is absent; rename test case to odf-storage-ready - Check 4: fail on missing S3-critical components (receive/compact/store) instead of passing with a note - Check 6: narrow route discovery to Thanos-specific routes using exact match then targeted fuzzy match --- .../interop-opp-observability-odf-commands.sh | 98 +++++++++++++------ 1 file changed, 70 insertions(+), 28 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh index d7c08762a5177..86f7c07fcd2ee 100755 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -117,12 +117,21 @@ if not items: else: print(items[0].get('status',{}).get('phase','Unknown')) ")"; then - AddResult "rgw-ready" "fail" "Failed to query CephObjectStore" + AddResult "odf-storage-ready" "fail" "Failed to query CephObjectStore" return fi if [[ "${rgwPhase}" == "NotFound" ]]; then - AddResult "rgw-ready" "skip" "No CephObjectStore found; ODF RGW not deployed" + typeset noobaaPhase="" + noobaaPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" \ + -o jsonpath='{.items[0].status.phase}' 2>/dev/null)" || true + if [[ "${noobaaPhase}" == "Ready" ]]; then + AddResult "odf-storage-ready" "pass" "NooBaa Ready (RGW not deployed)" + elif [[ -n "${noobaaPhase}" ]]; then + AddResult "odf-storage-ready" "fail" "NooBaa phase=${noobaaPhase} (expected Ready); RGW not deployed" + else + AddResult "odf-storage-ready" "skip" "Neither CephObjectStore nor NooBaa found in ${ODF_NAMESPACE}" + fi return fi @@ -159,9 +168,9 @@ else: if [[ -z "${failMsg}" ]]; then : "PASS: CephObjectStore Ready, RGW pods Running, StorageClass exists" - AddResult "rgw-ready" "pass" + AddResult "odf-storage-ready" "pass" else - AddResult "rgw-ready" "fail" "${failMsg}" + AddResult "odf-storage-ready" "fail" "${failMsg}" fi true } @@ -208,8 +217,8 @@ else: function CheckStorageEndpoint () { : "=== Check 3: Object storage endpoint ===" - typeset secretName="" - if ! secretName="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ + typeset storageConfig="" + if ! storageConfig="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ --all-namespaces -o json 2>/dev/null | python3 -c " import sys,json d=json.load(sys.stdin) @@ -219,17 +228,22 @@ if not items: else: spec=items[0].get('spec',{}) storage=spec.get('storageConfig',{}).get('metricObjectStorage',{}) - print(storage.get('name','')) + name=storage.get('name','') + key=storage.get('key','thanos.yaml') + print(f'{name}|{key}' if name else '') ")"; then AddResult "storage-endpoint" "fail" "Failed to read MCO storage config" return fi - if [[ -z "${secretName}" ]]; then + if [[ -z "${storageConfig}" ]]; then AddResult "storage-endpoint" "skip" "No metricObjectStorage secret configured in MCO" return fi + typeset secretName="${storageConfig%%|*}" + typeset secretKey="${storageConfig#*|}" + typeset secretJson="" secretJson="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true typeset endpointCheck="" @@ -237,21 +251,30 @@ else: endpointCheck="$(printf '%s' "${secretJson}" | python3 -c " import sys,json,base64,re d=json.load(sys.stdin) -odf_pattern=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs)', re.IGNORECASE) -for k,v in d.get('data',{}).items(): - decoded=base64.b64decode(v).decode('utf-8','replace') - if 'endpoint' in decoded.lower() or 'bucket' in decoded.lower(): - if odf_pattern.search(decoded): - print('odf-backed') - else: - print('external') - sys.exit(0) -print('no-endpoint') -" 2>/dev/null)" || true +target_key=sys.argv[1] if len(sys.argv)>1 else 'thanos.yaml' +raw=d.get('data',{}).get(target_key,'') +if not raw: + print('no-endpoint') + sys.exit(0) +decoded=base64.b64decode(raw).decode('utf-8','replace') +try: + import yaml + cfg=yaml.safe_load(decoded) + endpoint=cfg.get('config',{}).get('endpoint','') if isinstance(cfg,dict) else '' +except Exception: + import re as re2 + m=re2.search(r'endpoint:\s*(.+)',decoded) + endpoint=m.group(1).strip() if m else '' +if not endpoint: + print('no-endpoint') + sys.exit(0) +odf=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE) +print('odf-backed' if odf.search(endpoint) else 'external') +" "${secretKey}" 2>/dev/null)" || true fi if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then - AddResult "storage-endpoint" "fail" "Secret ${secretName} exists but no endpoint/bucket config found" + AddResult "storage-endpoint" "fail" "Secret ${secretName} exists but no endpoint config found in key ${secretKey}" return fi @@ -324,12 +347,26 @@ function CheckThanosHealth () { fi done + typeset -a s3CriticalNames=("thanos-receive" "thanos-compact" "thanos-store") + typeset -a missingCritical=() + typeset mc="" + for mc in "${missingComponents[@]}"; do + typeset cc="" + for cc in "${s3CriticalNames[@]}"; do + if [[ "${mc}" == "${cc}" ]]; then + missingCritical+=("${mc}") + fi + done + done + if (( foundCount == 0 )); then AddResult "thanos-health" "fail" "No Thanos/observability components found in ${OBS_NAMESPACE}" elif [[ -n "${failMsg}" ]]; then AddResult "thanos-health" "fail" "Unhealthy Thanos components: ${failMsg}" + elif (( ${#missingCritical[@]} > 0 )); then + AddResult "thanos-health" "fail" "Missing S3-critical components: ${missingCritical[*]}" elif (( ${#missingComponents[@]} > 0 )); then - AddResult "thanos-health" "pass" "Running (missing: ${missingComponents[*]})" + AddResult "thanos-health" "pass" "Running (optional missing: ${missingComponents[*]})" else AddResult "thanos-health" "pass" fi @@ -461,17 +498,22 @@ print('ok') function CheckThanosQuery () { : "=== Check 6: Thanos query functional ===" + typeset routeJson="" + routeJson="$(oc get routes -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true typeset queryRoute="" - queryRoute="$(oc get routes -n "${OBS_NAMESPACE}" -o json 2>/dev/null \ - | python3 -c " + if [[ -n "${routeJson}" ]]; then + queryRoute="$(printf '%s' "${routeJson}" | python3 -c " import sys,json d=json.load(sys.stdin) -routes=[i for i in d.get('items',[]) if 'query' in i['metadata'].get('name','').lower() or 'observ' in i['metadata'].get('name','').lower()] -if routes: - print(routes[0]['spec']['host']) -else: - print('') +routes=d.get('items',[]) +exact=[r for r in routes if r['metadata']['name']=='observability-thanos-query'] +if exact: + print(exact[0]['spec']['host']) + sys.exit(0) +fuzzy=[r for r in routes if 'thanos' in r['metadata']['name'] and 'query' in r['metadata']['name']] +print(fuzzy[0]['spec']['host'] if fuzzy else '') " 2>/dev/null)" || true + fi if [[ -z "${queryRoute}" ]]; then typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090" From cdd8be64f1a9e08d852234190495995c7fa2539a Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 19 Aug 2026 13:46:40 -0500 Subject: [PATCH 06/29] INTEROP-9416: mpitt self-review R1 convergence fixes - Remove stderr suppression (2>/dev/null) to preserve error output in xtrace for CI debugging - Replace grep -c || true with awk END{print NR} (pipefail-safe) - Use pre-increment (( ++x )) instead of (( x++ )) || true - Replace unsafe jsonpath items[0] with python3 safe-access - Guard oc exec with pod existence check before executing - Consolidate grep|head|awk pipelines into single awk - Replace grep -v with sed for pipefail safety - Add typeset to env var declarations --- .../interop-opp-observability-odf-commands.sh | 115 ++++++++++-------- 1 file changed, 64 insertions(+), 51 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh index 86f7c07fcd2ee..6074184942d43 100755 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -1,6 +1,5 @@ #!/bin/bash -set -euxo pipefail -shopt -s inherit_errexit +set -euxo pipefail; shopt -s inherit_errexit # --------------------------------------------------------------------------- # ACM Observability + ODF Interop Validation (6-point gate) @@ -12,9 +11,9 @@ shopt -s inherit_errexit # Produces JUnit XML consumed by Prow / Sippy / TestGrid. # --------------------------------------------------------------------------- -ACM_NAMESPACE="${ACM_NAMESPACE:-open-cluster-management}" -OBS_NAMESPACE="${OBS_NAMESPACE:-open-cluster-management-observability}" -ODF_NAMESPACE="${ODF_NAMESPACE:-openshift-storage}" +typeset ACM_NAMESPACE="${ACM_NAMESPACE:-open-cluster-management}" +typeset OBS_NAMESPACE="${OBS_NAMESPACE:-open-cluster-management-observability}" +typeset ODF_NAMESPACE="${ODF_NAMESPACE:-openshift-storage}" typeset junitFile="${ARTIFACT_DIR}/junit_observability_odf.xml" @@ -54,9 +53,9 @@ function WriteJunit () { typeset r="" for r in "${tcResultsArr[@]}"; do if [[ "${r}" == "fail" ]]; then - (( failCount++ )) || true + (( ++failCount )) elif [[ "${r}" == "skip" ]]; then - (( skipCount++ )) || true + (( ++skipCount )) fi done @@ -88,13 +87,13 @@ function WriteJunit () { # shellcheck disable=SC2317,SC2329 function CollectExitArtifacts () { : "Collecting observability + ODF diagnostics..." - oc get multiclusterobservabilities.observability.open-cluster-management.io --all-namespaces -o yaml > "${ARTIFACT_DIR}/mco.yaml" 2>/dev/null || true - oc get pods -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-pods.yaml" 2>/dev/null || true - oc get obc -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-obc.yaml" 2>/dev/null || true - oc get secret -n "${OBS_NAMESPACE}" -o name > "${ARTIFACT_DIR}/obs-secrets-list.txt" 2>/dev/null || true - oc get cephobjectstore -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/cephobjectstore.yaml" 2>/dev/null || true - oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw -o yaml > "${ARTIFACT_DIR}/rgw-pods.yaml" 2>/dev/null || true - oc get noobaa -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/noobaa.yaml" 2>/dev/null || true + oc get multiclusterobservabilities.observability.open-cluster-management.io --all-namespaces -o yaml > "${ARTIFACT_DIR}/mco.yaml" || true + oc get pods -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-pods.yaml" || true + oc get obc -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-obc.yaml" || true + oc get secret -n "${OBS_NAMESPACE}" -o name > "${ARTIFACT_DIR}/obs-secrets-list.txt" || true + oc get cephobjectstore -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/cephobjectstore.yaml" || true + oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw -o yaml > "${ARTIFACT_DIR}/rgw-pods.yaml" || true + oc get noobaa -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/noobaa.yaml" || true true } @@ -108,7 +107,7 @@ function CheckRgwReady () { : "=== Check 1: ODF Ceph RGW infrastructure ===" typeset rgwPhase="" - if ! rgwPhase="$(oc get cephobjectstore -n "${ODF_NAMESPACE}" -o json 2>/dev/null | python3 -c " + if ! rgwPhase="$(oc get cephobjectstore -n "${ODF_NAMESPACE}" -o json | python3 -c " import sys,json d=json.load(sys.stdin) items=d.get('items',[]) @@ -123,8 +122,11 @@ else: if [[ "${rgwPhase}" == "NotFound" ]]; then typeset noobaaPhase="" - noobaaPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" \ - -o jsonpath='{.items[0].status.phase}' 2>/dev/null)" || true + noobaaPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | python3 -c " +import sys,json +items=json.load(sys.stdin).get('items',[]) +print(items[0].get('status',{}).get('phase','') if items else '') +")" || true if [[ "${noobaaPhase}" == "Ready" ]]; then AddResult "odf-storage-ready" "pass" "NooBaa Ready (RGW not deployed)" elif [[ -n "${noobaaPhase}" ]]; then @@ -142,9 +144,9 @@ else: typeset rgwPods="" rgwPods="$(oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw \ - --field-selector=status.phase=Running --no-headers 2>/dev/null)" || true + --field-selector=status.phase=Running --no-headers)" || true typeset rgwPodCount="" - rgwPodCount="$(printf '%s' "${rgwPods}" | grep -c . || true)" + rgwPodCount="$(printf '%s' "${rgwPods}" | awk 'END{print NR}')" if [[ "${rgwPodCount}" -eq 0 ]]; then typeset rgwMsg="No rook-ceph-rgw pods Running in ${ODF_NAMESPACE}" @@ -156,7 +158,7 @@ else: fi typeset scExists="" - scExists="$(oc get sc ocs-storagecluster-ceph-rgw -o name 2>/dev/null)" || true + scExists="$(oc get sc ocs-storagecluster-ceph-rgw -o name)" || true if [[ -z "${scExists}" ]]; then typeset scMsg="StorageClass ocs-storagecluster-ceph-rgw not found" if [[ -n "${failMsg}" ]]; then @@ -184,7 +186,7 @@ function CheckMcoReady () { typeset mcoStatus="" if ! mcoStatus="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ - --all-namespaces -o json 2>/dev/null | python3 -c " + --all-namespaces -o json | python3 -c " import sys,json d=json.load(sys.stdin) items=d.get('items',[]) @@ -219,7 +221,7 @@ function CheckStorageEndpoint () { typeset storageConfig="" if ! storageConfig="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ - --all-namespaces -o json 2>/dev/null | python3 -c " + --all-namespaces -o json | python3 -c " import sys,json d=json.load(sys.stdin) items=d.get('items',[]) @@ -245,7 +247,7 @@ else: typeset secretKey="${storageConfig#*|}" typeset secretJson="" - secretJson="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true + secretJson="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json)" || true typeset endpointCheck="" if [[ -n "${secretJson}" ]]; then endpointCheck="$(printf '%s' "${secretJson}" | python3 -c " @@ -270,7 +272,7 @@ if not endpoint: sys.exit(0) odf=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE) print('odf-backed' if odf.search(endpoint) else 'external') -" "${secretKey}" 2>/dev/null)" || true +" "${secretKey}")" || true fi if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then @@ -293,7 +295,7 @@ print('odf-backed' if odf.search(endpoint) else 'external') function CheckThanosHealth () { : "=== Check 4: Thanos components healthy ===" - if ! oc get namespace "${OBS_NAMESPACE}" -o name 2>/dev/null; then + if ! oc get namespace "${OBS_NAMESPACE}" -o name; then AddResult "thanos-health" "skip" "Observability namespace ${OBS_NAMESPACE} does not exist" return fi @@ -312,15 +314,15 @@ function CheckThanosHealth () { typeset podList="" podList="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \ - --no-headers 2>/dev/null)" || true + --no-headers)" || true typeset podCount="" - podCount="$(printf '%s' "${podList}" | grep -c . || true)" + podCount="$(printf '%s' "${podList}" | awk 'END{print NR}')" if [[ "${podCount}" -eq 0 ]]; then typeset allPods="" allPods="$(oc get pods -n "${OBS_NAMESPACE}" \ - --no-headers 2>/dev/null)" || true - podCount="$(printf '%s' "${allPods}" | grep -c "^${component}" || true)" + --no-headers)" || true + podCount="$(printf '%s' "${allPods}" | awk -v pat="^${component}" '$0 ~ pat {c++} END{print c+0}')" fi if [[ "${podCount}" -eq 0 ]]; then @@ -328,11 +330,11 @@ function CheckThanosHealth () { continue fi - (( foundCount++ )) || true + (( ++foundCount )) typeset labeledPods="" labeledPods="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \ - --no-headers 2>/dev/null)" || true + --no-headers)" || true typeset notReady="" notReady="$(printf '%s' "${labeledPods}" \ | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')" || true @@ -381,7 +383,7 @@ function CheckObcBound () { : "=== Check 5: Observability ObjectBucketClaim ===" typeset obcList="" - obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true + obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json)" || true typeset obcItemCount="" if [[ -n "${obcList}" ]]; then @@ -389,26 +391,26 @@ function CheckObcBound () { import sys,json d=json.load(sys.stdin) print(len(d.get('items',[]))) -" 2>/dev/null)" || true +")" || true fi if [[ "${obcItemCount:-0}" -eq 0 ]]; then typeset odfObcJson="" - odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || true + odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json)" || true if [[ -n "${odfObcJson}" ]]; then obcList="$(printf '%s' "${odfObcJson}" | python3 -c " import sys,json d=json.load(sys.stdin) obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()] print(json.dumps({'items':obs})) -" 2>/dev/null)" || true +")" || true fi if [[ -n "${obcList}" ]]; then obcItemCount="$(echo "${obcList}" | python3 -c " import sys,json d=json.load(sys.stdin) print(len(d.get('items',[]))) -" 2>/dev/null)" || true +")" || true fi fi @@ -442,7 +444,7 @@ else: fi typeset unboundObcs="" - unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | grep -v '=Bound$')" || true + unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | sed '/=Bound$/d')" if [[ -z "${unboundObcs}" ]]; then : "PASS: All observability OBCs bound: ${obcStatus}" @@ -481,7 +483,7 @@ if not isinstance(result,list) or len(result)==0: print('empty-result') sys.exit(0) print('ok') -" 2>/dev/null)" || true +")" || true if [[ "${validation}" == "ok" ]]; then AddResult "thanos-query" "pass" @@ -499,7 +501,7 @@ function CheckThanosQuery () { : "=== Check 6: Thanos query functional ===" typeset routeJson="" - routeJson="$(oc get routes -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true + routeJson="$(oc get routes -n "${OBS_NAMESPACE}" -o json)" || true typeset queryRoute="" if [[ -n "${routeJson}" ]]; then queryRoute="$(printf '%s' "${routeJson}" | python3 -c " @@ -512,26 +514,37 @@ if exact: sys.exit(0) fuzzy=[r for r in routes if 'thanos' in r['metadata']['name'] and 'query' in r['metadata']['name']] print(fuzzy[0]['spec']['host'] if fuzzy else '') -" 2>/dev/null)" || true +")" || true fi if [[ -z "${queryRoute}" ]]; then typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090" : "No external route found; trying internal service: ${svcHost}" + typeset queryFrontendPod='' + queryFrontendPod="$(oc get pods -n "${OBS_NAMESPACE}" \ + -l app.kubernetes.io/name=thanos-query-frontend -o json | python3 -c " +import sys,json +items=json.load(sys.stdin).get('items',[]) +print(items[0]['metadata']['name'] if items else '') +")" || true + typeset queryResult="" - queryResult="$(oc exec -n "${OBS_NAMESPACE}" \ - "$(oc get pods -n "${OBS_NAMESPACE}" -l app.kubernetes.io/name=thanos-query-frontend \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo '')" \ - -- wget -qO- --no-check-certificate \ - "http://localhost:9090/api/v1/query?query=up" 2>/dev/null)" || true + if [[ -n "${queryFrontendPod}" ]]; then + queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryFrontendPod}" \ + -- wget -qO- --no-check-certificate \ + "http://localhost:9090/api/v1/query?query=up")" || true + fi if [[ -z "${queryResult}" ]]; then - queryResult="$(oc exec -n "${OBS_NAMESPACE}" \ - "$(oc get pods -n "${OBS_NAMESPACE}" --no-headers 2>/dev/null \ - | grep 'thanos-query' | grep -v 'frontend' | head -1 | awk '{print $1}')" \ - -- wget -qO- --no-check-certificate \ - "http://localhost:9090/api/v1/query?query=up" 2>/dev/null)" || true + typeset queryPod='' + queryPod="$(oc get pods -n "${OBS_NAMESPACE}" --no-headers \ + | awk '/thanos-query/ && !/frontend/ {print $1; exit}')" || true + if [[ -n "${queryPod}" ]]; then + queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryPod}" \ + -- wget -qO- --no-check-certificate \ + "http://localhost:9090/api/v1/query?query=up")" || true + fi fi if [[ -z "${queryResult}" ]]; then @@ -545,7 +558,7 @@ print(fuzzy[0]['spec']['host'] if fuzzy else '') set +x typeset token="" - token="$(oc whoami -t 2>/dev/null)" || true + token="$(oc whoami -t)" || true typeset responseBody="" typeset httpCode="" From 80d939b960ce3fb7c2abe5b25461fcf040b93299 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 19 Aug 2026 13:54:46 -0500 Subject: [PATCH 07/29] INTEROP-9416: mpitt R2 pipeline cleanup - Remove || true from pipelines where both sides always exit 0 - Consolidate duplicate oc get pods call (reuse podList) - Separate oc get | python3 pipelines for noobaa and query pods --- .../interop-opp-observability-odf-commands.sh | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh index 6074184942d43..c7da7b6e9a923 100755 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -121,12 +121,16 @@ else: fi if [[ "${rgwPhase}" == "NotFound" ]]; then + typeset noobaaJson="" + noobaaJson="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json)" || true typeset noobaaPhase="" - noobaaPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | python3 -c " + if [[ -n "${noobaaJson}" ]]; then + noobaaPhase="$(printf '%s' "${noobaaJson}" | python3 -c " import sys,json items=json.load(sys.stdin).get('items',[]) print(items[0].get('status',{}).get('phase','') if items else '') -")" || true +")" + fi if [[ "${noobaaPhase}" == "Ready" ]]; then AddResult "odf-storage-ready" "pass" "NooBaa Ready (RGW not deployed)" elif [[ -n "${noobaaPhase}" ]]; then @@ -272,7 +276,7 @@ if not endpoint: sys.exit(0) odf=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE) print('odf-backed' if odf.search(endpoint) else 'external') -" "${secretKey}")" || true +" "${secretKey}")" fi if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then @@ -332,12 +336,9 @@ function CheckThanosHealth () { (( ++foundCount )) - typeset labeledPods="" - labeledPods="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \ - --no-headers)" || true typeset notReady="" - notReady="$(printf '%s' "${labeledPods}" \ - | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')" || true + notReady="$(printf '%s' "${podList}" \ + | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')" if [[ -n "${notReady}" ]]; then typeset compMsg="${component}: ${notReady//$'\n'/, }" @@ -391,7 +392,7 @@ function CheckObcBound () { import sys,json d=json.load(sys.stdin) print(len(d.get('items',[]))) -")" || true +")" fi if [[ "${obcItemCount:-0}" -eq 0 ]]; then @@ -403,14 +404,14 @@ import sys,json d=json.load(sys.stdin) obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()] print(json.dumps({'items':obs})) -")" || true +")" fi if [[ -n "${obcList}" ]]; then obcItemCount="$(echo "${obcList}" | python3 -c " import sys,json d=json.load(sys.stdin) print(len(d.get('items',[]))) -")" || true +")" fi fi @@ -483,7 +484,7 @@ if not isinstance(result,list) or len(result)==0: print('empty-result') sys.exit(0) print('ok') -")" || true +")" if [[ "${validation}" == "ok" ]]; then AddResult "thanos-query" "pass" @@ -514,20 +515,24 @@ if exact: sys.exit(0) fuzzy=[r for r in routes if 'thanos' in r['metadata']['name'] and 'query' in r['metadata']['name']] print(fuzzy[0]['spec']['host'] if fuzzy else '') -")" || true +")" fi if [[ -z "${queryRoute}" ]]; then typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090" : "No external route found; trying internal service: ${svcHost}" + typeset queryFrontendJson='' + queryFrontendJson="$(oc get pods -n "${OBS_NAMESPACE}" \ + -l app.kubernetes.io/name=thanos-query-frontend -o json)" || true typeset queryFrontendPod='' - queryFrontendPod="$(oc get pods -n "${OBS_NAMESPACE}" \ - -l app.kubernetes.io/name=thanos-query-frontend -o json | python3 -c " + if [[ -n "${queryFrontendJson}" ]]; then + queryFrontendPod="$(printf '%s' "${queryFrontendJson}" | python3 -c " import sys,json items=json.load(sys.stdin).get('items',[]) print(items[0]['metadata']['name'] if items else '') -")" || true +")" + fi typeset queryResult="" if [[ -n "${queryFrontendPod}" ]]; then @@ -537,9 +542,13 @@ print(items[0]['metadata']['name'] if items else '') fi if [[ -z "${queryResult}" ]]; then + typeset allQueryPods='' + allQueryPods="$(oc get pods -n "${OBS_NAMESPACE}" --no-headers)" || true typeset queryPod='' - queryPod="$(oc get pods -n "${OBS_NAMESPACE}" --no-headers \ - | awk '/thanos-query/ && !/frontend/ {print $1; exit}')" || true + if [[ -n "${allQueryPods}" ]]; then + queryPod="$(printf '%s' "${allQueryPods}" \ + | awk '/thanos-query/ && !/frontend/ {print $1; exit}')" + fi if [[ -n "${queryPod}" ]]; then queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryPod}" \ -- wget -qO- --no-check-certificate \ From e5f28f36c01f5486fc4517c8f810fb72a509d15d Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 20 Aug 2026 13:24:28 -0500 Subject: [PATCH 08/29] INTEROP-9416: Fix pod status check after name-prefix fallback When the label selector returned no pods but the name-prefix fallback found matching pods, the notReady check was still operating on the empty label-selector result. Reassign podList in the fallback path so status inspection uses the correct pod listing. Also inline the svcHost variable that was only used in a diagnostic marker. --- .../interop-opp-observability-odf-commands.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh index c7da7b6e9a923..1ec8880af4066 100755 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -319,16 +319,17 @@ function CheckThanosHealth () { typeset podList="" podList="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \ --no-headers)" || true - typeset podCount="" - podCount="$(printf '%s' "${podList}" | awk 'END{print NR}')" - if [[ "${podCount}" -eq 0 ]]; then + if [[ -z "${podList}" ]]; then typeset allPods="" allPods="$(oc get pods -n "${OBS_NAMESPACE}" \ --no-headers)" || true - podCount="$(printf '%s' "${allPods}" | awk -v pat="^${component}" '$0 ~ pat {c++} END{print c+0}')" + podList="$(printf '%s' "${allPods}" | awk -v pat="^${component}" '$0 ~ pat')" fi + typeset podCount="" + podCount="$(printf '%s' "${podList}" | awk 'NF {c++} END{print c+0}')" + if [[ "${podCount}" -eq 0 ]]; then missingComponents+=("${component}") continue @@ -519,8 +520,7 @@ print(fuzzy[0]['spec']['host'] if fuzzy else '') fi if [[ -z "${queryRoute}" ]]; then - typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090" - : "No external route found; trying internal service: ${svcHost}" + : "No external route found; trying internal query service" typeset queryFrontendJson='' queryFrontendJson="$(oc get pods -n "${OBS_NAMESPACE}" \ From 4b2d6b5e479a944f008d924d2c4da96dc2ae5481 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 12 Aug 2026 15:39:25 -0500 Subject: [PATCH 09/29] INTEROP-9417: Add acm-tests-clc-smoke step for reduced CLC scope The existing acm-tests-clc-create step already creates only 1 AWS managed cluster (~50 min runtime) but carries a 28800s (8h) timeout and suppresses failures with || :. This new step provides: - Right-sized timeout: 5400s (90 min) vs 28800s - Strict failure propagation: no || : so downstream steps fail fast if cluster creation does not succeed No CUSTOMER_TAGS or CLOUD_PROVIDERS changes needed; the existing test image already scopes to single-cluster creation via TEST_STAGE=OCPInterop-create internally. Update OPP interop configs (ocp4.22, ocp5.0) to use the new step. The acm-tests-clc-destroy post step remains unchanged. --- ...stron-policy-collection-main__ocp4.22.yaml | 2 +- ...ostron-policy-collection-main__ocp5.0.yaml | 2 +- .../step-registry/acm/tests/clc-smoke/OWNERS | 9 +++ .../acm/tests/clc-smoke/README.md | 38 ++++++++++ .../clc-smoke/acm-tests-clc-smoke-commands.sh | 55 +++++++++++++++ .../acm-tests-clc-smoke-ref.metadata.json | 15 ++++ .../clc-smoke/acm-tests-clc-smoke-ref.yaml | 69 +++++++++++++++++++ 7 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 ci-operator/step-registry/acm/tests/clc-smoke/OWNERS create mode 100644 ci-operator/step-registry/acm/tests/clc-smoke/README.md create mode 100755 ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh create mode 100644 ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json create mode 100644 ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml index c3a92c8259122..a0e8de8d517ff 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml @@ -123,7 +123,7 @@ tests: - chain: cucushift-installer-check-cluster-health - ref: stackrox-opp-readiness - ref: stackrox-opp-smoke - - ref: acm-tests-clc-create + - ref: acm-tests-clc-smoke - ref: acm-fetch-managed-clusters - ref: acm-opp-app - ref: interop-opp-odf-health diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml index e31b7716529a3..decdf2d896ba8 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml @@ -104,7 +104,7 @@ tests: - ref: acm-policies-openshift-plus-setup - ref: acm-policies-openshift-plus - chain: cucushift-installer-check-cluster-health - - ref: acm-tests-clc-create + - ref: acm-tests-clc-smoke - ref: acm-fetch-managed-clusters - ref: acm-opp-app - ref: interop-opp-odf-health diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/OWNERS b/ci-operator/step-registry/acm/tests/clc-smoke/OWNERS new file mode 100644 index 0000000000000..76364ea3076e7 --- /dev/null +++ b/ci-operator/step-registry/acm/tests/clc-smoke/OWNERS @@ -0,0 +1,9 @@ +approvers: +- cspi-qe-ocp-lp +- dtthuynh +- vboulos +options: {} +reviewers: +- cspi-qe-ocp-lp +- dtthuynh +- vboulos diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/README.md b/ci-operator/step-registry/acm/tests/clc-smoke/README.md new file mode 100644 index 0000000000000..8819c7ae40c75 --- /dev/null +++ b/ci-operator/step-registry/acm/tests/clc-smoke/README.md @@ -0,0 +1,38 @@ +# acm-tests-clc-smoke-ref + +## Table of Contents +- [Purpose](#purpose) +- [Process](#process) +- [Requirements](#requirements) + - [Infrastructure](#infrastructure) + - [Environment Variables](#environment-variables) + +## Purpose + +Smoke-scoped variant of [acm-tests-clc-create](../clc-create/README.md) with a right-sized timeout and strict failure handling for OPP interop. + +The full `acm-tests-clc-create` step already creates only 1 AWS managed cluster (~50 min actual runtime) but carries a 28800s (8h) timeout and suppresses failures with `|| :`. This step: +- Reduces the timeout to 5400s (90 min), giving ~80% headroom over the observed average. +- Propagates failures so downstream steps (`acm-fetch-managed-clusters`, `acm-opp-app`) fail fast instead of running against a missing cluster. + +> **IMPORTANT** +> You must use the [acm-tests-clc-destroy-ref](../clc-destroy/README.md) as a post step when using this step. If you do not and succeed in running this step then you will leave clusters running on the ACM QE team's cloud. + +## Process + +- Copies secret options file needed for test execution. +- Injects AWS credentials from the cluster profile into options.yaml. +- Sets dynamic variables based on the provisioned hub cluster. +- Runs `execute_clc_interop_commands.sh` which invokes Cypress with tag filter `@create+aws+-sno+-@clusterpool` (controlled by `TEST_STAGE=OCPInterop-create` inside the image). + +## Requirements + +### Infrastructure + +- An existing OpenShift cluster to act as the target Hub. +- "advanced-cluster-management" operator installed (see [`install-operators`](../../../install-operators/README.md)). +- MCH custom resource installed (see [acm-mch step](../mch/README.md)). + +### Environment Variables + +- Please see [acm-tests-clc-smoke-ref.yaml](acm-tests-clc-smoke-ref.yaml) env section. diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh new file mode 100755 index 0000000000000..4e8545e4dfa1e --- /dev/null +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -euxo pipefail; shopt -s inherit_errexit + +typeset secretsDir="/tmp/secrets" +typeset optionFile="./options.yaml" +typeset awsCredFile="${CLUSTER_PROFILE_DIR}/.awscred" + +if [[ "${SKIP_OCP_DEPLOY:-false}" == "true" ]]; then + cp "${secretsDir}/ci/kubeconfig" "${SHARED_DIR}/kubeconfig" + cp "${secretsDir}/ci/kubeadmin-password" "${SHARED_DIR}/kubeadmin-password" +fi + +cp "${secretsDir}/clc-interop/secret-options-yaml" "${optionFile}" + +if [[ -f "${awsCredFile}" ]]; then + typeset awsAccKeyID= + typeset awsAccKeyToken= + + set +x + awsAccKeyID="$(sed -nE 's/^\s*aws_access_key_id\s*=\s*//p;T;q' "${awsCredFile}")" + awsAccKeyToken="$(sed -nE 's/^\s*aws_secret_access_key\s*=\s*//p;T;q' "${awsCredFile}")" + + [ -n "${awsAccKeyID}" ] && [ -n "${awsAccKeyToken}" ] + + yq -o json eval . "${optionFile}" | + jq -c \ + --arg awsAccKeyID "${awsAccKeyID}" \ + --arg awsAccKeyToken "${awsAccKeyToken}" \ + ' + .options.connections.apiKeys.aws|=( + .awsAccessKeyID=$awsAccKeyID | + .awsSecretAccessKeyID=$awsAccKeyToken + ) + ' | + yq -p json -o yaml eval . > "${optionFile}.tmp" + mv -f "${optionFile}.tmp" "${optionFile}" + set -x + + unset awsAccKeyID awsAccKeyToken +fi + +set +x +export CYPRESS_OPTIONS_HUB_PASSWORD= +CYPRESS_OPTIONS_HUB_PASSWORD="$(cat "${SHARED_DIR}/kubeadmin-password")" +set -x + +CYPRESS_BASE_URL="$(oc whoami --show-console)" \ +CYPRESS_HUB_API_URL="$(oc whoami --show-server)" \ +CYPRESS_CLC_OCP_IMAGE_VERSION="$(cat "${secretsDir}/clc/ocp_image_version")" \ +CLOUD_PROVIDERS="$(cat "${secretsDir}/clc/ocp_cloud_providers")" \ +bash +x ./execute_clc_interop_commands.sh + +unset CYPRESS_OPTIONS_HUB_PASSWORD + +cp -r reports "${ARTIFACT_DIR}/" diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json new file mode 100644 index 0000000000000..bfcc856a27d44 --- /dev/null +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp", + "dtthuynh", + "vboulos" + ], + "reviewers": [ + "cspi-qe-ocp-lp", + "dtthuynh", + "vboulos" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml new file mode 100644 index 0000000000000..d4a67209bffb3 --- /dev/null +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml @@ -0,0 +1,69 @@ +ref: + as: acm-tests-clc-smoke + from: clc-ui-e2e + commands: acm-tests-clc-smoke-commands.sh + timeout: 5400s + resources: + requests: + cpu: '2' + memory: 6Gi + credentials: + - namespace: test-credentials + name: opp-acm-clc-credentials + mount_path: /tmp/secrets/clc-interop + - namespace: test-credentials + name: acm-clc-credentials + mount_path: /tmp/secrets/clc + - namespace: test-credentials + name: acm-ci-credentials + mount_path: /tmp/secrets/ci + env: + - name: CYPRESS_OC_IDP + default: "kube:admin" + documentation: |- + Identity + - name: CYPRESS_OPTIONS_HUB_USER + default: "kubeadmin" + documentation: |- + Hub cluster username + - name: CYPRESS_SPOKE_CLUSTER + default: "" + documentation: |- + Identify spoke clusters + - name: BROWSER + default: "chrome" + documentation: |- + Set browser for cypress + - name: CUSTOMER_TAGS + default: "" + documentation: |- + Cypress grep tag filter (passed through to test execution) + - name: CYPRESS_CLC_OC_IDP + default: "clc-e2e-htpasswd" + documentation: |- + Serves tests RBAC settings + - name: CYPRESS_CLC_RBAC_PASS + default: "test-RBAC-4-e2e" + documentation: |- + Serves tests RBAC settings + - name: CYPRESS_CLC_OCP_IMAGE_REGISTRY + default: "quay.io/openshift-release-dev/ocp-release" + documentation: |- + Image registry + - name: CYPRESS_ACM_NAMESPACE + default: "ocm" + documentation: |- + Acm namespace + - name: CYPRESS_MCE_NAMESPACE + default: "multicluster-engine" + documentation: |- + Mce namespace + - name: IMPORT_KUBERNETES_CLUSTERS + default: "" + documentation: |- + Comma separated list of imports + documentation: |- + Smoke-scoped ACM cluster lifecycle step that creates a single managed + cluster on AWS (~50 min). Differs from acm-tests-clc-create only in + timeout (5400s vs 28800s) and failure propagation (no || :) so that + downstream steps fail fast if cluster creation does not succeed. From c1438c27b10675eb93284720981208a8b4026455 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 13 Aug 2026 11:30:45 -0500 Subject: [PATCH 10/29] fixup: harden xtrace and report collection - Keep tracing disabled through cluster endpoint assignments to prevent logging CYPRESS_BASE_URL and CYPRESS_HUB_API_URL in CI output - Capture test exit status so reports are always copied to ARTIFACT_DIR before propagating the failure - Add trailing newline to metadata.json --- .../acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh | 7 +++++-- .../tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh index 4e8545e4dfa1e..032694d56cb21 100755 --- a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh @@ -42,14 +42,17 @@ fi set +x export CYPRESS_OPTIONS_HUB_PASSWORD= CYPRESS_OPTIONS_HUB_PASSWORD="$(cat "${SHARED_DIR}/kubeadmin-password")" -set -x + +typeset clcStatus=0 CYPRESS_BASE_URL="$(oc whoami --show-console)" \ CYPRESS_HUB_API_URL="$(oc whoami --show-server)" \ CYPRESS_CLC_OCP_IMAGE_VERSION="$(cat "${secretsDir}/clc/ocp_image_version")" \ CLOUD_PROVIDERS="$(cat "${secretsDir}/clc/ocp_cloud_providers")" \ -bash +x ./execute_clc_interop_commands.sh +bash +x ./execute_clc_interop_commands.sh || clcStatus=$? +set -x unset CYPRESS_OPTIONS_HUB_PASSWORD cp -r reports "${ARTIFACT_DIR}/" +exit "${clcStatus}" diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json index bfcc856a27d44..aff5d0a60d630 100644 --- a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json @@ -12,4 +12,4 @@ "vboulos" ] } -} \ No newline at end of file +} From 5e29d385024f28289e410b262b3495a03e1a7ec1 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 13 Aug 2026 11:32:48 -0500 Subject: [PATCH 11/29] fixup: mpitt hardening (credential validation + secret handling) - Replace silent [ -n ] && [ -n ] with explicit error message on credential extraction failure for faster CI triage - Use jq --rawfile for AWS secret key to keep it off the process command line (awsAccKeyID kept as --arg since semi-public) --- .../acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh index 032694d56cb21..83661193bf2ac 100755 --- a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-commands.sh @@ -20,16 +20,19 @@ if [[ -f "${awsCredFile}" ]]; then awsAccKeyID="$(sed -nE 's/^\s*aws_access_key_id\s*=\s*//p;T;q' "${awsCredFile}")" awsAccKeyToken="$(sed -nE 's/^\s*aws_secret_access_key\s*=\s*//p;T;q' "${awsCredFile}")" - [ -n "${awsAccKeyID}" ] && [ -n "${awsAccKeyToken}" ] + if [[ -z "${awsAccKeyID}" ]] || [[ -z "${awsAccKeyToken}" ]]; then + echo "ERROR: Failed to extract AWS credentials from ${awsCredFile}" 1>&2 + exit 1 + fi yq -o json eval . "${optionFile}" | jq -c \ --arg awsAccKeyID "${awsAccKeyID}" \ - --arg awsAccKeyToken "${awsAccKeyToken}" \ + --rawfile awsAccKeyToken <(printf '%s' "${awsAccKeyToken}") \ ' .options.connections.apiKeys.aws|=( .awsAccessKeyID=$awsAccKeyID | - .awsSecretAccessKeyID=$awsAccKeyToken + .awsSecretAccessKeyID=($awsAccKeyToken | rtrimstr("\n")) ) ' | yq -p json -o yaml eval . > "${optionFile}.tmp" From a0b4e41ebb3dd05dfd30280d5e74537abcd67f8e Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 13 Aug 2026 11:40:59 -0500 Subject: [PATCH 12/29] fixup: regenerate metadata and add SKIP_OCP_DEPLOY env var - Revert trailing newline in metadata.json (auto-generated file must match generator output exactly) - Declare SKIP_OCP_DEPLOY in YAML env section for discoverability - Regenerate metadata via make registry-metadata --- .../acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json | 2 +- .../acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json index aff5d0a60d630..bfcc856a27d44 100644 --- a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.metadata.json @@ -12,4 +12,4 @@ "vboulos" ] } -} +} \ No newline at end of file diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml index d4a67209bffb3..d5f301b1be8c2 100644 --- a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml @@ -62,6 +62,10 @@ ref: default: "" documentation: |- Comma separated list of imports + - name: SKIP_OCP_DEPLOY + default: "false" + documentation: |- + When true, copies kubeconfig from CI secrets instead of using cluster profile documentation: |- Smoke-scoped ACM cluster lifecycle step that creates a single managed cluster on AWS (~50 min). Differs from acm-tests-clc-create only in From 41de22362699582d80d49a711341a0e5a4a8b8c3 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 20 Aug 2026 14:29:42 -0500 Subject: [PATCH 13/29] INTEROP-9431: graceful skip when ODF absent + fix policy race condition ODF health check: add CheckOdfInstalled pre-check that marks all 8 checks as "skip" (not "fail") when no ODF/OCS CSV exists, so the job stops failing on clusters where ODF is not yet available (OCP 5.0). ACM policies: wait for at least 4 policies before running oc wait, preventing premature exit when only the first policy has appeared. --- .../acm-policies-openshift-plus-commands.sh | 5 ++-- .../interop-opp-odf-health-commands.sh | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh index 0f847d68d0cec..e30e88e2da3dd 100644 --- a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh +++ b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh @@ -18,9 +18,10 @@ if [[ -n "${QUAY_OPERATOR_CHANNEL}" ]]; then fi echo 'y' | ./deploy.sh -p policygenerator/policy-sets/stable/openshift-plus -n policies -u https://github.com/stolostron/policy-collection.git -a openshift-plus +typeset -i expectedMinPolicies=4 typeset -i pollDeadline=$((SECONDS + 600)) -until (($(oc get policies -n policies -o name 2>/dev/null | wc -l))); do - ((SECONDS > pollDeadline)) && { : "Error: no policies appeared after 10 minutes"; exit 1; } +until (( $(oc get policies -n policies -o name 2>/dev/null | wc -l) >= expectedMinPolicies )); do + ((SECONDS > pollDeadline)) && { : "Error: fewer than ${expectedMinPolicies} policies after 10 minutes"; exit 1; } sleep 5 done diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh index 156a7f9b28caa..e1adff607a79a 100755 --- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh +++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh @@ -485,6 +485,16 @@ print(d['items'][0].get('status',{}).get('ceph',{}).get('health','unknown') if d # Main # --------------------------------------------------------------------------- +function CheckOdfInstalled () { + if ! oc get namespace "${ODF_NAMESPACE}" &>/dev/null; then + return 1 + fi + typeset csvCount + csvCount="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null | \ + python3 -c "import sys,json; print(len([i for i in json.load(sys.stdin).get('items',[]) if 'odf' in i['metadata']['name'].lower() or 'ocs' in i['metadata']['name'].lower()]))" 2>/dev/null)" || csvCount="0" + [[ "${csvCount}" -gt 0 ]] +} + function Main () { if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then export KUBECONFIG="${SHARED_DIR}/kubeconfig" @@ -494,6 +504,20 @@ function Main () { : "Namespace: ${ODF_NAMESPACE}" : "Artifacts dir: ${ARTIFACT_DIR}" + if ! CheckOdfInstalled; then + typeset skipMsg="ODF is not installed (no ODF/OCS CSV in ${ODF_NAMESPACE})" + typeset -a checkNames=("odf-csv-phase" "storagecluster-ready" "cephcluster-health" + "storageclasses-available" "pvc-provision-rbd" "pvc-provision-cephfs" + "noobaa-s3-functional" "ceph-health-detail") + typeset name="" + for name in "${checkNames[@]}"; do + AddResult "${name}" "skip" "${skipMsg}" + done + WriteJunit + : "ODF Health Check: ALL SKIPPED (ODF not installed)" + exit 0 + fi + CheckOdfCsv || true CheckStorageCluster || true CheckCephCluster || true From 83e68d33b409674f5544f36e053c5345f334110e Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 20 Aug 2026 18:42:12 -0500 Subject: [PATCH 14/29] INTEROP-9431: expand secondary policies + enable flag for OCP 5.0 ODF 5.0 is not in the catalog for OCP 5.0 (OCPSTRAT-3483), causing cascading NonCompliant across 16 of 20 policies (ODF core + observability + Quay chains). ACS is also NonCompliant due to package-level deprecation. Changes: - Expand secondaryPoliciesArr to include all ODF, observability, Quay, ACS, and compliance policies that cascade from the ODF gap - Set IGNORE_SECONDARY_POLICIES=true for both OCP 5.0 jobs (aws, vsphere) - Fix policy poll race condition (wait for >= 4 policies) Only policy-configure-subscription-admin-hub remains as a critical policy on 5.0. When ODF ships (~4 weeks post GA), remove the flag and trim the secondary list back to the original 4 entries. Verified with Chai Bot: cascading dependency analysis confirmed ODF absence blocks observability (via policy-odf-noobaa) and Quay (via policy-odf-status) chains. --- ...stolostron-policy-collection-main__ocp5.0.yaml | 2 ++ .../acm-policies-openshift-plus-commands.sh | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml index decdf2d896ba8..3e57e1a486ce5 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml @@ -77,6 +77,7 @@ tests: FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + IGNORE_SECONDARY_POLICIES: "true" OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"} @@ -135,6 +136,7 @@ tests: FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + IGNORE_SECONDARY_POLICIES: "true" OPENSHIFT_REQUIRED_CORES: "72" OPENSHIFT_REQUIRED_MEMORY: "288" OPERATORS: | diff --git a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh index e30e88e2da3dd..d9454a931add6 100644 --- a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh +++ b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh @@ -42,8 +42,23 @@ done typeset -a secondaryPoliciesArr=( policy-acs + policy-acs-monitor-certs + policy-acs-operator-central + policy-acs-sync-resources + policy-advanced-managed-cluster-security policy-advanced-managed-cluster-status + policy-compliance-operator-install + policy-config-quay policy-hub-quay-bridge + policy-install-quay + policy-observability-operator + policy-observability-storage + policy-observability-storage-status + policy-odf + policy-odf-cluster + policy-odf-noobaa + policy-odf-status + policy-quay-bridge policy-quay-status ) From 7958bf7f3a425565f6b144f628473143ac396340 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 07:31:08 -0500 Subject: [PATCH 15/29] INTEROP-9431: address review findings (xtrace, error handling, CSV predicate) - Enable xtrace (-x) in ODF health script for CI log debuggability - Distinguish ODF-absent (return 1) from probe-error (return 2) in CheckOdfInstalled; Main exits nonzero on probe errors - Use anchored ^(odf-|ocs-)operator regex matching CheckOdfCsv - Print timeout error to stderr instead of no-op : in policy poll --- .../acm-policies-openshift-plus-commands.sh | 5 +++- .../interop-opp-odf-health-commands.sh | 27 ++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh index d9454a931add6..3bdf87550e472 100644 --- a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh +++ b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh @@ -21,7 +21,10 @@ echo 'y' | ./deploy.sh -p policygenerator/policy-sets/stable/openshift-plus -n p typeset -i expectedMinPolicies=4 typeset -i pollDeadline=$((SECONDS + 600)) until (( $(oc get policies -n policies -o name 2>/dev/null | wc -l) >= expectedMinPolicies )); do - ((SECONDS > pollDeadline)) && { : "Error: fewer than ${expectedMinPolicies} policies after 10 minutes"; exit 1; } + ((SECONDS > pollDeadline)) && { + printf '%s\n' "Error: fewer than ${expectedMinPolicies} policies after 10 minutes" >&2 + exit 1 + } sleep 5 done diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh index e1adff607a79a..0d2c431223db1 100755 --- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh +++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh @@ -1,6 +1,5 @@ #!/bin/bash -set -euo pipefail -shopt -s inherit_errexit +set -euxo pipefail; shopt -s inherit_errexit # --------------------------------------------------------------------------- # ODF Health Check (7-point gate) @@ -489,9 +488,19 @@ function CheckOdfInstalled () { if ! oc get namespace "${ODF_NAMESPACE}" &>/dev/null; then return 1 fi - typeset csvCount - csvCount="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null | \ - python3 -c "import sys,json; print(len([i for i in json.load(sys.stdin).get('items',[]) if 'odf' in i['metadata']['name'].lower() or 'ocs' in i['metadata']['name'].lower()]))" 2>/dev/null)" || csvCount="0" + typeset csvJson="" + if ! csvJson="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null)"; then + printf '%s\n' "Error: failed to list CSVs in ${ODF_NAMESPACE}" >&2 + return 2 + fi + typeset csvCount="" + if ! csvCount="$(printf '%s' "${csvJson}" | python3 -c " +import sys,json,re; d=json.load(sys.stdin) +print(len([i for i in d.get('items',[]) if re.match(r'^(odf-|ocs-)operator',i['metadata']['name'])])) +")"; then + printf '%s\n' "Error: failed to parse CSV JSON from ${ODF_NAMESPACE}" >&2 + return 2 + fi [[ "${csvCount}" -gt 0 ]] } @@ -504,7 +513,13 @@ function Main () { : "Namespace: ${ODF_NAMESPACE}" : "Artifacts dir: ${ARTIFACT_DIR}" - if ! CheckOdfInstalled; then + typeset -i odfProbeResult=0 + CheckOdfInstalled || odfProbeResult=$? + if (( odfProbeResult == 2 )); then + : "ODF Health Check: PROBE ERROR (cannot determine ODF state)" + exit 1 + fi + if (( odfProbeResult == 1 )); then typeset skipMsg="ODF is not installed (no ODF/OCS CSV in ${ODF_NAMESPACE})" typeset -a checkNames=("odf-csv-phase" "storagecluster-ready" "cephcluster-health" "storageclasses-available" "pvc-provision-rbd" "pvc-provision-cephfs" From 949918ae39b4fc528121c48f6c54e74f4fa312d9 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 07:54:11 -0500 Subject: [PATCH 16/29] INTEROP-9431: document expectedMinPolicies threshold rationale --- .../openshift-plus/acm-policies-openshift-plus-commands.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh index 3bdf87550e472..193b13929c24b 100644 --- a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh +++ b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh @@ -18,6 +18,8 @@ if [[ -n "${QUAY_OPERATOR_CHANNEL}" ]]; then fi echo 'y' | ./deploy.sh -p policygenerator/policy-sets/stable/openshift-plus -n policies -u https://github.com/stolostron/policy-collection.git -a openshift-plus +# openshift-plus generates ~25 policies; require 4+ before oc wait to avoid +# racing the GitOps Subscription propagation (stolostron/policy-collection#174) typeset -i expectedMinPolicies=4 typeset -i pollDeadline=$((SECONDS + 600)) until (( $(oc get policies -n policies -o name 2>/dev/null | wc -l) >= expectedMinPolicies )); do From cfffdf870c081fb11fcc7826792173e1e4321a19 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 08:01:54 -0500 Subject: [PATCH 17/29] fix: mpitt R2 residual fixes --- .../interop-tests-opp-quay-smoke-commands.sh | 66 +++++++++++-------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index f726090036915..83a1740aec1dd 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -1,11 +1,12 @@ #!/bin/bash -set -euo pipefail +set -eux -o pipefail shopt -s inherit_errexit ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}" mkdir -p "${ARTIFACT_DIR}" typeset junitFile="${ARTIFACT_DIR}/junit_quay_interop.xml" -typeset imageTag="${BUILD_ID:-$(date +%s)}" +typeset imageTag='' +imageTag="${BUILD_ID:-$(date +%s)}" typeset -A testStatus typeset -A testDuration @@ -33,13 +34,15 @@ function RecordResult () { testStatus["${name}"]="${status}" testDuration["${name}"]="${dur}" testFailureMsg["${name}"]="${msg}" + true } # shellcheck disable=SC2329 function GenerateJunit () { typeset -i total=${#allTests[@]} typeset -i failures=0 skipped=0 - typeset -i elapsed=$(( $(date +%s) - suiteStart )) + typeset -i elapsed=0 + elapsed=$(( $(date +%s) - suiteStart )) for t in "${allTests[@]}"; do [[ "${testStatus[${t}]}" == "failed" ]] && failures=$((failures + 1)) @@ -53,28 +56,29 @@ function GenerateJunit () { EOF for t in "${allTests[@]}"; do - typeset escaped_name - escaped_name=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') - typeset escaped_msg - escaped_msg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escapedName + escapedName=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escapedMsg + escapedMsg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') if [[ "${testStatus[${t}]}" == "failed" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" elif [[ "${testStatus[${t}]}" == "skipped" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" else - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" fi done - cat >> "${junitFile}" <> "${junitFile}" <<'EOF' EOF cat "${junitFile}" + true } -trap GenerateJunit EXIT +trap '{ ( GenerateJunit; true ); }' EXIT function DiscoverQuay () { QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}') @@ -82,6 +86,7 @@ function DiscoverQuay () { QUAY_HOST=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.status.registryEndpoint}') QUAY_HOST="${QUAY_HOST#https://}" export QUAY_NS QUAY_REGISTRY QUAY_HOST + true } function GetQuayAuth () { @@ -91,25 +96,26 @@ function GetQuayAuth () { configSecret="${QUAY_REGISTRY}-config-bundle" fi - QUAY_USER=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_EMAIL}' 2>/dev/null | base64 -d || echo "") + QUAY_USER=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_EMAIL}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_USER="" if [[ -z "${QUAY_USER}" ]]; then QUAY_USER="quayadmin" fi - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_PASSWORD}' 2>/dev/null | base64 -d || echo "") + QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_PASSWORD}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" if [[ -z "${QUAY_PASSWORD}" ]]; then typeset initSecret="${QUAY_REGISTRY}-init-config-bundle-secret" - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${initSecret}" -o jsonpath='{.data.superuser-password}' 2>/dev/null | base64 -d || echo "") + QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${initSecret}" -o jsonpath='{.data.superuser-password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" fi if [[ -z "${QUAY_PASSWORD}" ]]; then for secret in $(oc get secrets -n "${QUAY_NS}" -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep -i "quay.*config"); do - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${secret}" -o go-template='{{index .data "config.yaml"}}' 2>/dev/null | base64 -d | grep -oP "(?<=SUPER_USER_PASSWORD: ).*" || echo "") + QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${secret}" -o go-template='{{index .data "config.yaml"}}' 2>/dev/null | base64 -d | grep -oP "(?<=SUPER_USER_PASSWORD: ).*") || QUAY_PASSWORD="" [[ -n "${QUAY_PASSWORD}" ]] && break done fi export QUAY_USER QUAY_PASSWORD + true } function PreflightCheck () { @@ -117,30 +123,34 @@ function PreflightCheck () { echo "ERROR: Quay route not reachable at ${QUAY_HOST}" >&2 return 1 fi + true } function CreateTestOrg () { typeset signinPayload + set +x signinPayload=$(python3 -c "import json,sys; print(json.dumps({'user':sys.argv[1],'pass':sys.argv[2]}))" "${QUAY_USER}" "${QUAY_PASSWORD}") typeset token token=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \ -H "Content-Type: application/json" \ -d "${signinPayload}" | \ - python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || echo "") + python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null) || token="" if [[ -z "${token}" ]]; then token=$(curl -sk -H "Authorization: Basic $(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64)" \ "https://${QUAY_HOST}/api/v1/user/" | \ - python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null || echo "") + python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null) || token="" fi QUAY_TOKEN="${token}" export QUAY_TOKEN + set -x curl -sk -X POST "https://${QUAY_HOST}/api/v1/organization/" \ -H "Authorization: Bearer ${QUAY_TOKEN}" \ -H "Content-Type: application/json" \ -d '{"name":"interop-smoke-test","email":"interop-test@example.com"}' || true + true } ################################################################################ @@ -154,9 +164,11 @@ function RunPushPull () { typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" typeset authFile="/tmp/quay-auth.json" + set +x cat > "${authFile}" </dev/null 2>&1; then + "docker://${pushTarget}"; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "Image not pullable from Quay after push" "${elapsed}" return 1 @@ -194,7 +206,7 @@ import sys, json data = json.load(sys.stdin) items = data.get('items', []) print(len(items)) -" 2>/dev/null || echo "0") +" 2>/dev/null) || pvcCount="0" if [[ "${pvcCount}" == "0" ]]; then pvcCount=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " @@ -202,7 +214,7 @@ import sys, json data = json.load(sys.stdin) items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] print(len(items)) -" 2>/dev/null || echo "0") +" 2>/dev/null) || pvcCount="0" fi if [[ "${pvcCount}" == "0" ]]; then @@ -218,7 +230,7 @@ data = json.load(sys.stdin) items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound'] print(' '.join(unbound)) -" 2>/dev/null || echo "") +" 2>/dev/null) || unboundPvcs="" if [[ -n "${unboundPvcs}" ]]; then elapsed=$(( $(date +%s) - start )) @@ -234,7 +246,7 @@ items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name', sc_names = set(i['spec'].get('storageClassName','') for i in items) odf = any('ocs' in s or 'ceph' in s or 'odf' in s for s in sc_names) print('true' if odf else 'false') -" 2>/dev/null || echo "false") +" 2>/dev/null) || odfBacked="false" if [[ "${odfBacked}" != "true" ]]; then elapsed=$(( $(date +%s) - start )) @@ -256,14 +268,14 @@ function RunAcsScan () { start=$(date +%s) typeset acsHost acsPassword - acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null || echo "") + acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null) || acsHost="" if [[ -z "${acsHost}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS Central route not found" "${elapsed}" return 1 fi - acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "") + acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || acsPassword="" if [[ -z "${acsPassword}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS admin password not found" "${elapsed}" @@ -276,7 +288,7 @@ function RunAcsScan () { while (( attempts < maxAttempts )); do typeset scanResult scanResult=$(curl -sk -u "admin:${acsPassword}" \ - "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null || echo "") + "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null) || scanResult="" if echo "${scanResult}" | python3 -c " import sys, json @@ -322,7 +334,7 @@ function Main () { "${_fURL[@]}" \ https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/ci-operator/interop/common/ExitTrap--PostProcessPrep.sh )" || true - if type -t ExitTrap--PostProcessPrep 1>/dev/null; then + if type -t ExitTrap--PostProcessPrep; then LP_IO__ET_PPP__NEW_TS_NAME="${DR__RP__CR_COMP_NAME}--%s" \ ExitTrap--PostProcessPrep || true fi From c983bacd3341f50e2fb47cd346143bae9090464e Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 20 Aug 2026 11:22:13 -0500 Subject: [PATCH 18/29] INTEROP-9430: fix interop-tests-opp-quay-smoke step Three root causes fixed: 1. Missing skopeo: add cli-with-skopeo image via dockerfile_literal in all 4 ci-operator configs (ocp4.22, ocp4.22-fips, ocp5.0, ocp5.1) and update ref.yaml from: cli -> from: cli-with-skopeo. 2. Broken Quay auth: rewrite GetQuayAuth to read credentials from the quayadmin secret (created by ACM openshift-plus PolicySet admin-user job) instead of the non-existent SUPER_USER_PASSWORD config bundle field. Falls back to quaydevel secret, then to /api/v1/user/initialize. Rewrite CreateTestOrg to use Bearer token directly or CSRF signin flow. 3. Wrong ODF validation: rewrite RunOdfPvcCheck -> RunOdfStorageCheck to validate OBCs and NooBaa health instead of checking PVC storage classes (NooBaa using default gp3-csi is by design). --- ...-policy-collection-main__ocp4.22-fips.yaml | 6 + ...stron-policy-collection-main__ocp4.22.yaml | 6 + ...ostron-policy-collection-main__ocp5.0.yaml | 6 + ...ostron-policy-collection-main__ocp5.1.yaml | 6 + .../interop-tests-opp-quay-smoke-commands.sh | 206 +++++++++--------- .../interop-tests-opp-quay-smoke-ref.yaml | 7 +- 6 files changed, 134 insertions(+), 103 deletions(-) diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml index 1221f3a80be7a..f5050d7facf2d 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml @@ -40,6 +40,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo - dockerfile_literal: | FROM registry.access.redhat.com/ubi9/openjdk-17:1.21 USER root diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml index a0e8de8d517ff..1864430307295 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml @@ -40,6 +40,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo - dockerfile_literal: | FROM registry.access.redhat.com/ubi9/openjdk-17:1.21 USER root diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml index 3e57e1a486ce5..a821e99b0950e 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml @@ -36,6 +36,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo releases: latest: candidate: diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml index 8598841060b45..8fb2da67b580c 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml @@ -36,6 +36,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo releases: latest: candidate: diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index 83a1740aec1dd..e91e3d02fce38 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -1,19 +1,18 @@ #!/bin/bash -set -eux -o pipefail +set -euo pipefail shopt -s inherit_errexit ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}" mkdir -p "${ARTIFACT_DIR}" typeset junitFile="${ARTIFACT_DIR}/junit_quay_interop.xml" -typeset imageTag='' -imageTag="${BUILD_ID:-$(date +%s)}" +typeset imageTag="${BUILD_ID:-$(date +%s)}" typeset -A testStatus typeset -A testDuration typeset -A testFailureMsg typeset -a allTests=( "[sig-interop][Jira:INTEROP][Feature:Quay] Push and pull image via Quay route" - "[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF PVC backing Quay storage" + "[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF object storage integration" "[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" ) @@ -34,15 +33,13 @@ function RecordResult () { testStatus["${name}"]="${status}" testDuration["${name}"]="${dur}" testFailureMsg["${name}"]="${msg}" - true } # shellcheck disable=SC2329 function GenerateJunit () { typeset -i total=${#allTests[@]} typeset -i failures=0 skipped=0 - typeset -i elapsed=0 - elapsed=$(( $(date +%s) - suiteStart )) + typeset -i elapsed=$(( $(date +%s) - suiteStart )) for t in "${allTests[@]}"; do [[ "${testStatus[${t}]}" == "failed" ]] && failures=$((failures + 1)) @@ -56,29 +53,28 @@ function GenerateJunit () { EOF for t in "${allTests[@]}"; do - typeset escapedName - escapedName=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') - typeset escapedMsg - escapedMsg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escaped_name + escaped_name=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escaped_msg + escaped_msg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') if [[ "${testStatus[${t}]}" == "failed" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" elif [[ "${testStatus[${t}]}" == "skipped" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" else - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" fi done - cat >> "${junitFile}" <<'EOF' + cat >> "${junitFile}" < EOF cat "${junitFile}" - true } -trap '{ ( GenerateJunit; true ); }' EXIT +trap GenerateJunit EXIT function DiscoverQuay () { QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}') @@ -86,36 +82,53 @@ function DiscoverQuay () { QUAY_HOST=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.status.registryEndpoint}') QUAY_HOST="${QUAY_HOST#https://}" export QUAY_NS QUAY_REGISTRY QUAY_HOST - true } function GetQuayAuth () { - typeset configSecret - configSecret=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.spec.configBundleSecret}') - if [[ -z "${configSecret}" ]]; then - configSecret="${QUAY_REGISTRY}-config-bundle" - fi + QUAY_USER="" + QUAY_PASSWORD="" + QUAY_TOKEN="" - QUAY_USER=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_EMAIL}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_USER="" - if [[ -z "${QUAY_USER}" ]]; then + if oc get secret quayadmin -n "${QUAY_NS}" &>/dev/null; then + QUAY_TOKEN=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null || echo "") + QUAY_PASSWORD=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || echo "") QUAY_USER="quayadmin" + if [[ -n "${QUAY_TOKEN}" || -n "${QUAY_PASSWORD}" ]]; then + echo "INFO: Quay credentials obtained from quayadmin secret" + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 0 + fi fi - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_PASSWORD}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" - if [[ -z "${QUAY_PASSWORD}" ]]; then - typeset initSecret="${QUAY_REGISTRY}-init-config-bundle-secret" - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${initSecret}" -o jsonpath='{.data.superuser-password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" + if oc get secret quaydevel -n "${QUAY_NS}" &>/dev/null; then + QUAY_PASSWORD=$(oc get secret quaydevel -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || echo "") + QUAY_USER="quaydevel" + if [[ -n "${QUAY_PASSWORD}" ]]; then + echo "INFO: Quay credentials obtained from quaydevel secret" + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 0 + fi fi - if [[ -z "${QUAY_PASSWORD}" ]]; then - for secret in $(oc get secrets -n "${QUAY_NS}" -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep -i "quay.*config"); do - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${secret}" -o go-template='{{index .data "config.yaml"}}' 2>/dev/null | base64 -d | grep -oP "(?<=SUPER_USER_PASSWORD: ).*") || QUAY_PASSWORD="" - [[ -n "${QUAY_PASSWORD}" ]] && break - done + typeset initPassword + initPassword=$(python3 -c "import secrets,string; print(''.join(secrets.choice(string.ascii_letters+string.digits) for _ in range(20)))") + typeset initResult + initResult=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/user/initialize" \ + -H "Content-Type: application/json" \ + -d "{\"username\":\"quayadmin\",\"password\":\"${initPassword}\",\"email\":\"quayadmin@example.com\",\"access_token\":true}" 2>/dev/null || echo "") + + QUAY_TOKEN=$(echo "${initResult}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('access_token',''))" 2>/dev/null || echo "") + if [[ -n "${QUAY_TOKEN}" ]]; then + QUAY_USER="quayadmin" + QUAY_PASSWORD="${initPassword}" + echo "INFO: Quay admin user initialized via /api/v1/user/initialize" + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 0 fi - export QUAY_USER QUAY_PASSWORD - true + echo "ERROR: Could not obtain Quay credentials from any source" >&2 + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 1 } function PreflightCheck () { @@ -123,34 +136,37 @@ function PreflightCheck () { echo "ERROR: Quay route not reachable at ${QUAY_HOST}" >&2 return 1 fi - true } function CreateTestOrg () { - typeset signinPayload - set +x - signinPayload=$(python3 -c "import json,sys; print(json.dumps({'user':sys.argv[1],'pass':sys.argv[2]}))" "${QUAY_USER}" "${QUAY_PASSWORD}") - typeset token - token=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \ - -H "Content-Type: application/json" \ - -d "${signinPayload}" | \ - python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null) || token="" - - if [[ -z "${token}" ]]; then - token=$(curl -sk -H "Authorization: Basic $(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64)" \ - "https://${QUAY_HOST}/api/v1/user/" | \ - python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null) || token="" + if [[ -z "${QUAY_TOKEN}" && -n "${QUAY_PASSWORD}" ]]; then + typeset cookieFile="/tmp/quay-cookies.txt" + typeset csrf + csrf=$(curl -sk "https://${QUAY_HOST}/csrf_token" -c "${cookieFile}" | \ + python3 -c "import sys,json; print(json.load(sys.stdin).get('csrf_token',''))" 2>/dev/null || echo "") + + if [[ -n "${csrf}" ]]; then + typeset signinResult + signinResult=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \ + -H "Content-Type: application/json" \ + -H "X-CSRF-Token: ${csrf}" \ + -b "${cookieFile}" -c "${cookieFile}" \ + -d "{\"username\":\"${QUAY_USER}\",\"password\":\"${QUAY_PASSWORD}\"}" 2>/dev/null || echo "") + QUAY_TOKEN=$(echo "${signinResult}" | \ + python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || echo "") + fi + rm -f "${cookieFile}" + export QUAY_TOKEN fi - QUAY_TOKEN="${token}" - export QUAY_TOKEN - set -x + if [[ -z "${QUAY_TOKEN}" ]]; then + echo "WARNING: No Quay token available; org creation may fail" >&2 + fi curl -sk -X POST "https://${QUAY_HOST}/api/v1/organization/" \ -H "Authorization: Bearer ${QUAY_TOKEN}" \ -H "Content-Type: application/json" \ -d '{"name":"interop-smoke-test","email":"interop-test@example.com"}' || true - true } ################################################################################ @@ -164,11 +180,9 @@ function RunPushPull () { typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" typeset authFile="/tmp/quay-auth.json" - set +x cat > "${authFile}" </dev/null 2>&1; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "Image not pullable from Quay after push" "${elapsed}" return 1 @@ -193,64 +207,56 @@ EOF } ################################################################################ -# Test Case 2: Verify ODF PVC backing Quay storage +# Test Case 2: Verify ODF object storage integration ################################################################################ -function RunOdfPvcCheck () { - typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF PVC backing Quay storage" +function RunOdfStorageCheck () { + typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF object storage integration" typeset -i start elapsed start=$(date +%s) - typeset pvcCount - pvcCount=$(oc get pvc -n "${QUAY_NS}" -l app=quay -o json 2>/dev/null | python3 -c " -import sys, json -data = json.load(sys.stdin) -items = data.get('items', []) -print(len(items)) -" 2>/dev/null) || pvcCount="0" + typeset noobaaPhase + noobaaPhase=$(oc get noobaa -n openshift-storage -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "") + if [[ "${noobaaPhase}" != "Ready" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "NooBaa not Ready (phase: ${noobaaPhase:-not found})" "${elapsed}" + return 1 + fi - if [[ "${pvcCount}" == "0" ]]; then - pvcCount=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " -import sys, json -data = json.load(sys.stdin) -items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] -print(len(items)) -" 2>/dev/null) || pvcCount="0" + typeset obcCount + obcCount=$(oc get objectbucketclaim -n openshift-storage -o json 2>/dev/null | \ + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null || echo "0") + if [[ "${obcCount}" == "0" ]]; then + obcCount=$(oc get objectbucketclaim --all-namespaces -o json 2>/dev/null | \ + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null || echo "0") fi - if [[ "${pvcCount}" == "0" ]]; then + if [[ "${obcCount}" == "0" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "No Quay-related PVCs found in ${QUAY_NS}" "${elapsed}" + RecordResult "${testName}" "failed" "No ObjectBucketClaims found" "${elapsed}" return 1 fi - typeset unboundPvcs - unboundPvcs=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " -import sys, json -data = json.load(sys.stdin) -items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] -unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound'] -print(' '.join(unbound)) -" 2>/dev/null) || unboundPvcs="" - - if [[ -n "${unboundPvcs}" ]]; then + typeset obCount + obCount=$(oc get objectbucket -o json 2>/dev/null | \ + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null || echo "0") + if [[ "${obCount}" == "0" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "Unbound PVCs: ${unboundPvcs}" "${elapsed}" + RecordResult "${testName}" "failed" "No ObjectBucket resources found for OBCs" "${elapsed}" return 1 fi - typeset odfBacked - odfBacked=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " + typeset unboundPvcs + unboundPvcs=$(oc get pvc -n "${QUAY_NS}" -o json 2>/dev/null | python3 -c " import sys, json data = json.load(sys.stdin) items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] -sc_names = set(i['spec'].get('storageClassName','') for i in items) -odf = any('ocs' in s or 'ceph' in s or 'odf' in s for s in sc_names) -print('true' if odf else 'false') -" 2>/dev/null) || odfBacked="false" +unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound'] +print(' '.join(unbound)) +" 2>/dev/null || echo "") - if [[ "${odfBacked}" != "true" ]]; then + if [[ -n "${unboundPvcs}" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "Quay PVCs not using ODF/Ceph storage class" "${elapsed}" + RecordResult "${testName}" "failed" "Unbound Quay PVCs: ${unboundPvcs}" "${elapsed}" return 1 fi @@ -268,14 +274,14 @@ function RunAcsScan () { start=$(date +%s) typeset acsHost acsPassword - acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null) || acsHost="" + acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null || echo "") if [[ -z "${acsHost}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS Central route not found" "${elapsed}" return 1 fi - acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || acsPassword="" + acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "") if [[ -z "${acsPassword}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS admin password not found" "${elapsed}" @@ -288,7 +294,7 @@ function RunAcsScan () { while (( attempts < maxAttempts )); do typeset scanResult scanResult=$(curl -sk -u "admin:${acsPassword}" \ - "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null) || scanResult="" + "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null || echo "") if echo "${scanResult}" | python3 -c " import sys, json @@ -322,7 +328,7 @@ function Main () { typeset -i status=0 RunPushPull || status=1 - RunOdfPvcCheck || status=1 + RunOdfStorageCheck || status=1 RunAcsScan || status=1 rm -f /tmp/quay-auth.json @@ -334,7 +340,7 @@ function Main () { "${_fURL[@]}" \ https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/ci-operator/interop/common/ExitTrap--PostProcessPrep.sh )" || true - if type -t ExitTrap--PostProcessPrep; then + if type -t ExitTrap--PostProcessPrep 1>/dev/null; then LP_IO__ET_PPP__NEW_TS_NAME="${DR__RP__CR_COMP_NAME}--%s" \ ExitTrap--PostProcessPrep || true fi diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml index eb2e58149f440..5b0caae06170a 100644 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml @@ -1,6 +1,6 @@ ref: as: interop-tests-opp-quay-smoke - from: cli + from: cli-with-skopeo cli: latest commands: interop-tests-opp-quay-smoke-commands.sh timeout: 30m0s @@ -11,8 +11,9 @@ ref: memory: 256Mi documentation: |- Validates Quay as a cross-product registry within the OPP bundle. - Tests image push/pull via the Quay route, verifies ODF-backed PVC storage, - and confirms ACS detects and scans the pushed image. + Tests image push/pull via the Quay route, verifies ODF object storage + integration (OBCs and NooBaa health), and confirms ACS detects and scans + the pushed image. env: - name: DR__RP__CR_COMP_NAME default: "lp-interop--Quay" From aa0f9f240bd3c579452c78137ff30912ad860560 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 20 Aug 2026 14:30:01 -0500 Subject: [PATCH 19/29] fix: use $oauthtoken for token-only registry auth When quayadmin secret has token but no password, use Quay's OAuth registry auth convention ($oauthtoken:) instead of encoding an empty password. --- .../interop-tests-opp-quay-smoke-commands.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index e91e3d02fce38..552bc121552f2 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -180,8 +180,15 @@ function RunPushPull () { typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" typeset authFile="/tmp/quay-auth.json" + typeset registryAuth + if [[ -n "${QUAY_TOKEN}" ]]; then + registryAuth=$(echo -n "\$oauthtoken:${QUAY_TOKEN}" | base64) + else + registryAuth=$(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64) + fi + cat > "${authFile}" < Date: Fri, 21 Aug 2026 07:35:00 -0500 Subject: [PATCH 20/29] fix ACS scan: register Quay integration and trigger explicit scan ACS doesn't passively discover images in internal Quay registries. Register Quay as an image integration in ACS (with insecure TLS), then explicitly request a scan via /v1/images/scan. Retry the scan request every minute during the 10-minute poll window. --- .../interop-tests-opp-quay-smoke-commands.sh | 83 ++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index 552bc121552f2..99147077e8dda 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -275,6 +275,77 @@ print(' '.join(unbound)) ################################################################################ # Test Case 3: ACS scan of pushed Quay image ################################################################################ +function RegisterQuayInAcs () { + typeset acsHost="${1}" acsPassword="${2}" + + typeset existing + existing=$(curl -sk -u "admin:${acsPassword}" \ + "https://${acsHost}/v1/imageintegrations" 2>/dev/null | \ + python3 -c " +import sys, json, os +host = os.environ['QUAY_HOST'] +data = json.load(sys.stdin) +for i in data.get('integrations', []): + if host in i.get('docker', {}).get('endpoint', ''): + print(i['id']) + sys.exit(0) +sys.exit(1) +" 2>/dev/null || echo "") + + if [[ -n "${existing}" ]]; then + echo "INFO: Quay integration already registered in ACS" + return 0 + fi + + typeset regUser regPass + if [[ -n "${QUAY_TOKEN}" ]]; then + regUser="\$oauthtoken" + regPass="${QUAY_TOKEN}" + else + regUser="${QUAY_USER}" + regPass="${QUAY_PASSWORD}" + fi + + python3 -c " +import json, sys, os +payload = { + 'name': 'interop-quay-smoke', + 'type': 'docker', + 'categories': ['REGISTRY'], + 'docker': { + 'endpoint': os.environ['QUAY_HOST'], + 'username': sys.argv[1], + 'password': sys.argv[2], + 'insecure': True + }, + 'skipTestIntegration': True +} +print(json.dumps(payload)) +" "${regUser}" "${regPass}" | \ + curl -sk -X POST "https://${acsHost}/v1/imageintegrations" \ + -u "admin:${acsPassword}" \ + -H "Content-Type: application/json" \ + -d @- >/dev/null 2>&1 || true + + echo "INFO: Registered Quay at ${QUAY_HOST} as ACS image integration" +} + +function RequestAcsScan () { + typeset acsHost="${1}" acsPassword="${2}" imageName="${3}" + + python3 -c " +import json, sys +payload = {'imageName': sys.argv[1], 'force': True} +print(json.dumps(payload)) +" "${imageName}" | \ + curl -sk -X POST "https://${acsHost}/v1/images/scan" \ + -u "admin:${acsPassword}" \ + -H "Content-Type: application/json" \ + -d @- >/dev/null 2>&1 || true + + echo "INFO: Requested ACS scan of ${imageName}" +} + function RunAcsScan () { typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" typeset -i start elapsed @@ -296,7 +367,11 @@ function RunAcsScan () { fi typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" - typeset -i attempts=0 maxAttempts=20 + + RegisterQuayInAcs "${acsHost}" "${acsPassword}" + RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" + + typeset -i attempts=0 maxAttempts=40 while (( attempts < maxAttempts )); do typeset scanResult @@ -314,12 +389,16 @@ sys.exit(0 if len(images) > 0 else 1) return 0 fi + if (( attempts % 4 == 3 )); then + RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" + fi + attempts=$((attempts + 1)) sleep 15 done elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "ACS did not detect pushed image within 5 minutes" "${elapsed}" + RecordResult "${testName}" "failed" "ACS did not detect pushed image within 10 minutes" "${elapsed}" return 1 } From 850c7f723cb90bd6a692a4f873181ae1cbdd64dc Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 07:52:17 -0500 Subject: [PATCH 21/29] fix: mpitt best practices R1 (35 fixes) set -eux -o pipefail, SC2155 splits, camelCase locals, heredoc quoting, subshell trap handler, removed stdout suppression, broke pipelines with || true, replaced || echo anti-patterns, added trailing true to functions. --- .../interop-tests-opp-quay-smoke-commands.sh | 87 +++++++++++-------- 1 file changed, 50 insertions(+), 37 deletions(-) diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index 99147077e8dda..5b1d0a0fffbd7 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -1,11 +1,12 @@ #!/bin/bash -set -euo pipefail +set -eux -o pipefail shopt -s inherit_errexit ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}" mkdir -p "${ARTIFACT_DIR}" typeset junitFile="${ARTIFACT_DIR}/junit_quay_interop.xml" -typeset imageTag="${BUILD_ID:-$(date +%s)}" +typeset imageTag='' +imageTag="${BUILD_ID:-$(date +%s)}" typeset -A testStatus typeset -A testDuration @@ -33,13 +34,15 @@ function RecordResult () { testStatus["${name}"]="${status}" testDuration["${name}"]="${dur}" testFailureMsg["${name}"]="${msg}" + true } # shellcheck disable=SC2329 function GenerateJunit () { typeset -i total=${#allTests[@]} typeset -i failures=0 skipped=0 - typeset -i elapsed=$(( $(date +%s) - suiteStart )) + typeset -i elapsed=0 + elapsed=$(( $(date +%s) - suiteStart )) for t in "${allTests[@]}"; do [[ "${testStatus[${t}]}" == "failed" ]] && failures=$((failures + 1)) @@ -53,28 +56,29 @@ function GenerateJunit () { EOF for t in "${allTests[@]}"; do - typeset escaped_name - escaped_name=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') - typeset escaped_msg - escaped_msg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escapedName + escapedName=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escapedMsg + escapedMsg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') if [[ "${testStatus[${t}]}" == "failed" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" elif [[ "${testStatus[${t}]}" == "skipped" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" else - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" fi done - cat >> "${junitFile}" <> "${junitFile}" <<'EOF' EOF cat "${junitFile}" + true } -trap GenerateJunit EXIT +trap '{ ( GenerateJunit ); }' EXIT function DiscoverQuay () { QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}') @@ -82,6 +86,7 @@ function DiscoverQuay () { QUAY_HOST=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.status.registryEndpoint}') QUAY_HOST="${QUAY_HOST#https://}" export QUAY_NS QUAY_REGISTRY QUAY_HOST + true } function GetQuayAuth () { @@ -90,8 +95,8 @@ function GetQuayAuth () { QUAY_TOKEN="" if oc get secret quayadmin -n "${QUAY_NS}" &>/dev/null; then - QUAY_TOKEN=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null || echo "") - QUAY_PASSWORD=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || echo "") + QUAY_TOKEN=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_TOKEN="" + QUAY_PASSWORD=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" QUAY_USER="quayadmin" if [[ -n "${QUAY_TOKEN}" || -n "${QUAY_PASSWORD}" ]]; then echo "INFO: Quay credentials obtained from quayadmin secret" @@ -101,7 +106,7 @@ function GetQuayAuth () { fi if oc get secret quaydevel -n "${QUAY_NS}" &>/dev/null; then - QUAY_PASSWORD=$(oc get secret quaydevel -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || echo "") + QUAY_PASSWORD=$(oc get secret quaydevel -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" QUAY_USER="quaydevel" if [[ -n "${QUAY_PASSWORD}" ]]; then echo "INFO: Quay credentials obtained from quaydevel secret" @@ -115,9 +120,9 @@ function GetQuayAuth () { typeset initResult initResult=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/user/initialize" \ -H "Content-Type: application/json" \ - -d "{\"username\":\"quayadmin\",\"password\":\"${initPassword}\",\"email\":\"quayadmin@example.com\",\"access_token\":true}" 2>/dev/null || echo "") + -d "{\"username\":\"quayadmin\",\"password\":\"${initPassword}\",\"email\":\"quayadmin@example.com\",\"access_token\":true}" 2>/dev/null) || initResult="" - QUAY_TOKEN=$(echo "${initResult}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('access_token',''))" 2>/dev/null || echo "") + QUAY_TOKEN=$(echo "${initResult}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('access_token',''))" 2>/dev/null) || QUAY_TOKEN="" if [[ -n "${QUAY_TOKEN}" ]]; then QUAY_USER="quayadmin" QUAY_PASSWORD="${initPassword}" @@ -136,6 +141,7 @@ function PreflightCheck () { echo "ERROR: Quay route not reachable at ${QUAY_HOST}" >&2 return 1 fi + true } function CreateTestOrg () { @@ -143,7 +149,7 @@ function CreateTestOrg () { typeset cookieFile="/tmp/quay-cookies.txt" typeset csrf csrf=$(curl -sk "https://${QUAY_HOST}/csrf_token" -c "${cookieFile}" | \ - python3 -c "import sys,json; print(json.load(sys.stdin).get('csrf_token',''))" 2>/dev/null || echo "") + python3 -c "import sys,json; print(json.load(sys.stdin).get('csrf_token',''))" 2>/dev/null) || csrf="" if [[ -n "${csrf}" ]]; then typeset signinResult @@ -151,9 +157,9 @@ function CreateTestOrg () { -H "Content-Type: application/json" \ -H "X-CSRF-Token: ${csrf}" \ -b "${cookieFile}" -c "${cookieFile}" \ - -d "{\"username\":\"${QUAY_USER}\",\"password\":\"${QUAY_PASSWORD}\"}" 2>/dev/null || echo "") + -d "{\"username\":\"${QUAY_USER}\",\"password\":\"${QUAY_PASSWORD}\"}" 2>/dev/null) || signinResult="" QUAY_TOKEN=$(echo "${signinResult}" | \ - python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || echo "") + python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null) || QUAY_TOKEN="" fi rm -f "${cookieFile}" export QUAY_TOKEN @@ -167,6 +173,7 @@ function CreateTestOrg () { -H "Authorization: Bearer ${QUAY_TOKEN}" \ -H "Content-Type: application/json" \ -d '{"name":"interop-smoke-test","email":"interop-test@example.com"}' || true + true } ################################################################################ @@ -202,7 +209,7 @@ EOF if ! skopeo inspect --tls-verify=false \ --authfile="${authFile}" \ - "docker://${pushTarget}" >/dev/null 2>&1; then + "docker://${pushTarget}"; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "Image not pullable from Quay after push" "${elapsed}" return 1 @@ -222,7 +229,7 @@ function RunOdfStorageCheck () { start=$(date +%s) typeset noobaaPhase - noobaaPhase=$(oc get noobaa -n openshift-storage -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "") + noobaaPhase=$(oc get noobaa -n openshift-storage -o jsonpath='{.items[0].status.phase}' 2>/dev/null) || noobaaPhase="" if [[ "${noobaaPhase}" != "Ready" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "NooBaa not Ready (phase: ${noobaaPhase:-not found})" "${elapsed}" @@ -231,10 +238,10 @@ function RunOdfStorageCheck () { typeset obcCount obcCount=$(oc get objectbucketclaim -n openshift-storage -o json 2>/dev/null | \ - python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null || echo "0") + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obcCount="0" if [[ "${obcCount}" == "0" ]]; then obcCount=$(oc get objectbucketclaim --all-namespaces -o json 2>/dev/null | \ - python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null || echo "0") + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obcCount="0" fi if [[ "${obcCount}" == "0" ]]; then @@ -245,7 +252,7 @@ function RunOdfStorageCheck () { typeset obCount obCount=$(oc get objectbucket -o json 2>/dev/null | \ - python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null || echo "0") + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obCount="0" if [[ "${obCount}" == "0" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "No ObjectBucket resources found for OBCs" "${elapsed}" @@ -259,7 +266,7 @@ data = json.load(sys.stdin) items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound'] print(' '.join(unbound)) -" 2>/dev/null || echo "") +" 2>/dev/null) || unboundPvcs="" if [[ -n "${unboundPvcs}" ]]; then elapsed=$(( $(date +%s) - start )) @@ -290,7 +297,7 @@ for i in data.get('integrations', []): print(i['id']) sys.exit(0) sys.exit(1) -" 2>/dev/null || echo "") +" 2>/dev/null) || existing="" if [[ -n "${existing}" ]]; then echo "INFO: Quay integration already registered in ACS" @@ -306,7 +313,8 @@ sys.exit(1) regPass="${QUAY_PASSWORD}" fi - python3 -c " + typeset regPayload='' + regPayload=$(python3 -c " import json, sys, os payload = { 'name': 'interop-quay-smoke', @@ -321,29 +329,34 @@ payload = { 'skipTestIntegration': True } print(json.dumps(payload)) -" "${regUser}" "${regPass}" | \ +" "${regUser}" "${regPass}") + curl -sk -X POST "https://${acsHost}/v1/imageintegrations" \ -u "admin:${acsPassword}" \ -H "Content-Type: application/json" \ - -d @- >/dev/null 2>&1 || true + -d "${regPayload}" || true echo "INFO: Registered Quay at ${QUAY_HOST} as ACS image integration" + true } function RequestAcsScan () { typeset acsHost="${1}" acsPassword="${2}" imageName="${3}" - python3 -c " + typeset scanPayload='' + scanPayload=$(python3 -c " import json, sys payload = {'imageName': sys.argv[1], 'force': True} print(json.dumps(payload)) -" "${imageName}" | \ +" "${imageName}") + curl -sk -X POST "https://${acsHost}/v1/images/scan" \ -u "admin:${acsPassword}" \ -H "Content-Type: application/json" \ - -d @- >/dev/null 2>&1 || true + -d "${scanPayload}" || true echo "INFO: Requested ACS scan of ${imageName}" + true } function RunAcsScan () { @@ -352,14 +365,14 @@ function RunAcsScan () { start=$(date +%s) typeset acsHost acsPassword - acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null || echo "") + acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null) || acsHost="" if [[ -z "${acsHost}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS Central route not found" "${elapsed}" return 1 fi - acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "") + acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d) || acsPassword="" if [[ -z "${acsPassword}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS admin password not found" "${elapsed}" @@ -376,7 +389,7 @@ function RunAcsScan () { while (( attempts < maxAttempts )); do typeset scanResult scanResult=$(curl -sk -u "admin:${acsPassword}" \ - "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null || echo "") + "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null) || scanResult="" if echo "${scanResult}" | python3 -c " import sys, json @@ -426,7 +439,7 @@ function Main () { "${_fURL[@]}" \ https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/ci-operator/interop/common/ExitTrap--PostProcessPrep.sh )" || true - if type -t ExitTrap--PostProcessPrep 1>/dev/null; then + if type -t ExitTrap--PostProcessPrep; then LP_IO__ET_PPP__NEW_TS_NAME="${DR__RP__CR_COMP_NAME}--%s" \ ExitTrap--PostProcessPrep || true fi From 5fdc5f835d1f9abd02a96c572545af1886fae0b0 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 08:09:03 -0500 Subject: [PATCH 22/29] fix: print timeout error before exit in acm-policies step --- .../acm-policies-openshift-plus-commands.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh index 193b13929c24b..f04cd58e50fb6 100644 --- a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh +++ b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh @@ -35,7 +35,7 @@ typeset -a quayNamespacesArr=(quay openshift-quay quay-enterprise) typeset quayFound=false for ns in "${quayNamespacesArr[@]}"; do if (($(oc get quayregistry -n "${ns}" -o name 2>/dev/null | wc -l))); then - : "Found Quay Operator deployment in namespace ${ns}, waiting for ready condition" + echo "Found Quay Operator deployment in namespace ${ns}, waiting for ready condition" oc wait quayregistry --all -n "${ns}" \ --for condition=Available=True \ --timeout=10m || true @@ -43,7 +43,7 @@ for ns in "${quayNamespacesArr[@]}"; do break fi done -[[ "${quayFound}" == "false" ]] && : "Warning: no QuayRegistry found in namespaces: ${quayNamespacesArr[*]}" +[[ "${quayFound}" == "false" ]] && echo "Warning: no QuayRegistry found in namespaces: ${quayNamespacesArr[*]}" >&2 typeset -a secondaryPoliciesArr=( policy-acs @@ -78,12 +78,12 @@ if [[ "${IGNORE_SECONDARY_POLICIES}" == "true" ]]; then --for jsonpath='{.status.compliant}'=Compliant \ --timeout=40m } || { - : "Critical policies failed to become compliant:" + echo "ERROR: Critical policies failed to become compliant:" >&2 oc get policies -n policies | grep -Ev "$(IFS='|'; echo "${secondaryPoliciesArr[*]}")" || true exit 1 } else - : "All policies are secondary (ignored), no critical policies to wait for" + echo "All policies are secondary (ignored), no critical policies to wait for" fi else { @@ -91,7 +91,7 @@ else --for jsonpath='{.status.compliant}'=Compliant \ --timeout=40m } || { - : "Policies failed to become compliant:" + echo "ERROR: Policies failed to become compliant:" >&2 oc get policies -n policies exit 1 } From 1b56bba95fd82837671c8cc3c6be682e636dcdf6 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 08:09:10 -0500 Subject: [PATCH 23/29] fix: address review findings in quay-smoke step --- .../interop-tests-opp-quay-smoke-commands.sh | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index 5b1d0a0fffbd7..c0172a1cf3fb6 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -78,13 +78,17 @@ EOF true } -trap '{ ( GenerateJunit ); }' EXIT +trap '{ typeset -i rc=$?; ( GenerateJunit ); exit ${rc}; }' EXIT function DiscoverQuay () { QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}') QUAY_REGISTRY=$(oc get quayregistry -n "${QUAY_NS}" -o jsonpath='{.items[0].metadata.name}') QUAY_HOST=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.status.registryEndpoint}') QUAY_HOST="${QUAY_HOST#https://}" + if [[ -z "${QUAY_HOST}" ]]; then + echo "ERROR: Quay registry route not ready (empty host)" >&2 + return 1 + fi export QUAY_NS QUAY_REGISTRY QUAY_HOST true } @@ -138,7 +142,7 @@ function GetQuayAuth () { function PreflightCheck () { if ! curl -sk --connect-timeout 15 "https://${QUAY_HOST}/api/v1/discovery" | grep -qi "quay"; then - echo "ERROR: Quay route not reachable at ${QUAY_HOST}" >&2 + echo "ERROR: Quay registry endpoint not reachable" >&2 return 1 fi true @@ -187,6 +191,12 @@ function RunPushPull () { typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" typeset authFile="/tmp/quay-auth.json" + if [[ -z "${QUAY_TOKEN}" && -z "${QUAY_PASSWORD}" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "No valid Quay authentication token or password available" "${elapsed}" + return 1 + fi + typeset registryAuth if [[ -n "${QUAY_TOKEN}" ]]; then registryAuth=$(echo -n "\$oauthtoken:${QUAY_TOKEN}" | base64) @@ -336,7 +346,7 @@ print(json.dumps(payload)) -H "Content-Type: application/json" \ -d "${regPayload}" || true - echo "INFO: Registered Quay at ${QUAY_HOST} as ACS image integration" + echo "INFO: Registered Quay registry endpoint as ACS image integration" true } @@ -425,10 +435,14 @@ function Main () { PreflightCheck || { echo "FATAL: Quay not reachable; skipping all tests" >&2; exit 1; } CreateTestOrg - typeset -i status=0 - RunPushPull || status=1 + typeset -i status=0 pushPassed=0 + RunPushPull && pushPassed=1 || status=1 RunOdfStorageCheck || status=1 - RunAcsScan || status=1 + if (( pushPassed )); then + RunAcsScan || status=1 + else + RecordResult "[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" "skipped" "Skipped: push-pull test failed; no image available to scan" + fi rm -f /tmp/quay-auth.json From 27b9e0101b59e2a4d8f5e85f29259813f309bb44 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 08:09:10 -0500 Subject: [PATCH 24/29] fix: address review findings in odf-health and stackrox steps --- .../interop-opp-odf-health-commands.sh | 16 ++++++--- .../stackrox-opp-readiness-commands.sh | 35 ++++++++++--------- .../opp-smoke/stackrox-opp-smoke-commands.sh | 15 +------- 3 files changed, 30 insertions(+), 36 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh index 0d2c431223db1..36441844fcd92 100755 --- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh +++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit # --------------------------------------------------------------------------- # ODF Health Check (7-point gate) @@ -104,7 +104,7 @@ function CheckOdfCsv () { typeset csvPhase="" if ! csvPhase="$(oc get csv -n "${ODF_NAMESPACE}" -o json | python3 -c " import sys,json,re; d=json.load(sys.stdin) -m=[i for i in d.get('items',[]) if re.match(r'^(odf-|ocs-)operator',i['metadata']['name'])] +m=[i for i in d.get('items',[]) if re.match(r'^(odf-operator|ocs-operator)',i['metadata']['name'])] print((m[0].get('status',{}).get('phase','NotFound')) if m else 'NotFound') ")"; then AddResult "odf-csv-phase" "fail" "Failed to query ODF CSVs in ${ODF_NAMESPACE}" @@ -489,14 +489,20 @@ function CheckOdfInstalled () { return 1 fi typeset csvJson="" - if ! csvJson="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null)"; then - printf '%s\n' "Error: failed to list CSVs in ${ODF_NAMESPACE}" >&2 + typeset -i ocExit=0 + csvJson="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || ocExit=$? + if (( ocExit != 0 )); then + printf '%s\n' "Error: oc get csv failed (exit ${ocExit}) in ${ODF_NAMESPACE}" >&2 + return 2 + fi + if [[ -z "${csvJson}" ]]; then + printf '%s\n' "Error: oc get csv returned empty output in ${ODF_NAMESPACE}" >&2 return 2 fi typeset csvCount="" if ! csvCount="$(printf '%s' "${csvJson}" | python3 -c " import sys,json,re; d=json.load(sys.stdin) -print(len([i for i in d.get('items',[]) if re.match(r'^(odf-|ocs-)operator',i['metadata']['name'])])) +print(len([i for i in d.get('items',[]) if re.match(r'^(odf-operator|ocs-operator)',i['metadata']['name'])])) ")"; then printf '%s\n' "Error: failed to parse CSV JSON from ${ODF_NAMESPACE}" >&2 return 2 diff --git a/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh b/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh index 5a78660adc402..45dcea495679c 100755 --- a/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh +++ b/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eux -o pipefail +set -euo pipefail shopt -s inherit_errexit # --------------------------------------------------------------------------- @@ -84,9 +84,20 @@ function CheckCentralRoute () { WaitFor "Central route" CheckCentralRoute -set +x -centralUrl="$(oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}')" -set -x +typeset -i routeElapsed=0 +centralUrl="" +while [[ -z "${centralUrl}" ]]; do + centralUrl="$(oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}' 2>/dev/null)" || true + if [[ -n "${centralUrl}" ]]; then + break + fi + if (( routeElapsed >= 30 )); then + echo "[readiness] FATAL: Central route host empty after 30s" + exit 1 + fi + sleep 5 + (( routeElapsed += 5 )) || true +done echo "[readiness] Central route discovered" # --------------------------------------------------------------------------- @@ -94,10 +105,8 @@ echo "[readiness] Central route discovered" # --------------------------------------------------------------------------- typeset roxAdminPassword="" echo "[readiness] Extracting roxAdminPassword..." -set +x roxAdminPassword="$(oc get secret -n "${centralNs}" central-htpasswd \ -o jsonpath='{.data.password}' | base64 -d)" -set -x if [[ -z "${roxAdminPassword}" ]]; then echo "[readiness] FATAL: could not extract roxAdminPassword" @@ -109,12 +118,10 @@ echo "[readiness] roxAdminPassword extracted successfully" # Check 2: Central API health (authenticated v1/metadata) # --------------------------------------------------------------------------- function CheckCentralApi () { - set +x typeset httpCode="" httpCode="$(curl -sk -o /dev/null -w '%{http_code}' \ -u "admin:${roxAdminPassword}" \ - "https://${centralUrl}/v1/metadata" --max-time 10)" || { set -x; return 1; } - set -x + "https://${centralUrl}/v1/metadata" --max-time 10)" || return 1 [[ "${httpCode}" == "200" ]] } @@ -124,12 +131,10 @@ WaitFor "Central API health (v1/metadata)" CheckCentralApi # Check 3: At least 1 secured cluster connected # --------------------------------------------------------------------------- function CheckClustersConnected () { - set +x typeset clusterCount="" clusterCount="$(curl -sk -u "admin:${roxAdminPassword}" \ "https://${centralUrl}/v1/clusters" --max-time 10 \ - | JsonLength clusters)" || { set -x; return 1; } - set -x + | JsonLength clusters)" || return 1 [[ "${clusterCount}" -ge 1 ]] } @@ -184,12 +189,10 @@ WaitFor "sensor pods Running in ${scNs}" CheckSensorPods # Check 5: Default policies loaded (count > 80) # --------------------------------------------------------------------------- function CheckPoliciesLoaded () { - set +x typeset policyCount="" policyCount="$(curl -sk -u "admin:${roxAdminPassword}" \ "https://${centralUrl}/v1/policies?query=" --max-time 10 \ - | JsonLength policies)" || { set -x; return 1; } - set -x + | JsonLength policies)" || return 1 echo "[readiness] policy count: ${policyCount}" [[ "${policyCount}" -gt 80 ]] } @@ -198,10 +201,8 @@ WaitFor "default policies loaded (>80)" CheckPoliciesLoaded echo "[readiness] Writing connection details to SHARED_DIR..." -set +x echo "${roxAdminPassword}" > "${SHARED_DIR}/ROX_ADMIN_PASSWORD" echo "${centralUrl}" > "${SHARED_DIR}/CENTRAL_URL" -set -x echo "${centralNs}" > "${SHARED_DIR}/CENTRAL_NS" echo "${scNs}" > "${SHARED_DIR}/SC_NS" diff --git a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh index 3907a7d723dab..972ab22b7f8a0 100755 --- a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh +++ b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eux -o pipefail +set -euo pipefail shopt -s inherit_errexit if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then @@ -8,10 +8,8 @@ fi echo "[smoke] Reading connection details from SHARED_DIR..." -set +x CENTRAL_URL="$(cat "${SHARED_DIR}/CENTRAL_URL")" ROX_ADMIN_PASSWORD="$(cat "${SHARED_DIR}/ROX_ADMIN_PASSWORD")" -set -x echo "[smoke] Connection details loaded from SHARED_DIR" @@ -51,7 +49,6 @@ grep -q 'DEFAULT_CLUSTER_NAME = "local-cluster"' \ /tmp/stackrox/qa-tests-backend/src/main/groovy/services/ClusterService.groovy \ || { echo "[smoke] FATAL: DEFAULT_CLUSTER_NAME patch failed"; exit 1; } -set +x export API_HOSTNAME="${CENTRAL_URL}" export API_PORT="443" export ROX_USERNAME="admin" @@ -72,7 +69,6 @@ if [[ -f /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2)" export GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2 fi -set -x cd /tmp/stackrox/qa-tests-backend @@ -103,13 +99,4 @@ if [[ -d build/reports/tests/testSMOKE ]]; then fi echo "[smoke] Test run finished with exit code: ${testExit}" -if [[ "${testExit}" -ne 0 ]] && [[ -d build/test-results/testSMOKE ]]; then - typeset total="" - total="$(find build/test-results/testSMOKE -name '*.xml' -exec grep -l 'testcase' {} \; | wc -l)" - if [[ "${total}" -gt 0 ]]; then - echo "[smoke] Tests executed and results captured; treating as informational (exit 0)." - echo "[smoke] Review JUnit XML in ARTIFACT_DIR for individual test failures." - exit 0 - fi -fi exit "${testExit}" From 678a2743da9ca719501ede6110211ad4b1bc05f8 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 08:13:38 -0500 Subject: [PATCH 25/29] fix: address review findings in observability-odf step --- .../interop-opp-observability-odf-commands.sh | 83 +++++++++---------- 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh index 1ec8880af4066..936d2730dce00 100755 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit # --------------------------------------------------------------------------- # ACM Observability + ODF Interop Validation (6-point gate) @@ -256,26 +256,32 @@ else: if [[ -n "${secretJson}" ]]; then endpointCheck="$(printf '%s' "${secretJson}" | python3 -c " import sys,json,base64,re +sys.tracebacklimit=0 d=json.load(sys.stdin) target_key=sys.argv[1] if len(sys.argv)>1 else 'thanos.yaml' raw=d.get('data',{}).get(target_key,'') if not raw: print('no-endpoint') sys.exit(0) -decoded=base64.b64decode(raw).decode('utf-8','replace') +try: + content=base64.b64decode(raw).decode('utf-8','replace') +except Exception: + print('no-endpoint') + sys.exit(0) +endpoint='' try: import yaml - cfg=yaml.safe_load(decoded) + cfg=yaml.safe_load(content) endpoint=cfg.get('config',{}).get('endpoint','') if isinstance(cfg,dict) else '' except Exception: - import re as re2 - m=re2.search(r'endpoint:\s*(.+)',decoded) + m=re.search(r'endpoint:\s*(.+)',content) endpoint=m.group(1).strip() if m else '' +del content if not endpoint: print('no-endpoint') sys.exit(0) -odf=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE) -print('odf-backed' if odf.search(endpoint) else 'external') +odf_pat=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE) +print('odf-backed' if odf_pat.search(endpoint) else 'external') " "${secretKey}")" fi @@ -351,26 +357,12 @@ function CheckThanosHealth () { fi done - typeset -a s3CriticalNames=("thanos-receive" "thanos-compact" "thanos-store") - typeset -a missingCritical=() - typeset mc="" - for mc in "${missingComponents[@]}"; do - typeset cc="" - for cc in "${s3CriticalNames[@]}"; do - if [[ "${mc}" == "${cc}" ]]; then - missingCritical+=("${mc}") - fi - done - done - if (( foundCount == 0 )); then AddResult "thanos-health" "fail" "No Thanos/observability components found in ${OBS_NAMESPACE}" elif [[ -n "${failMsg}" ]]; then AddResult "thanos-health" "fail" "Unhealthy Thanos components: ${failMsg}" - elif (( ${#missingCritical[@]} > 0 )); then - AddResult "thanos-health" "fail" "Missing S3-critical components: ${missingCritical[*]}" elif (( ${#missingComponents[@]} > 0 )); then - AddResult "thanos-health" "pass" "Running (optional missing: ${missingComponents[*]})" + AddResult "thanos-health" "fail" "Missing components: ${missingComponents[*]}" else AddResult "thanos-health" "pass" fi @@ -385,38 +377,41 @@ function CheckObcBound () { : "=== Check 5: Observability ObjectBucketClaim ===" typeset obcList="" - obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json)" || true + obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true - typeset obcItemCount="" - if [[ -n "${obcList}" ]]; then - obcItemCount="$(echo "${obcList}" | python3 -c " + typeset obcItemCount=0 + obcItemCount="$(printf '%s' "${obcList}" | python3 -c " import sys,json -d=json.load(sys.stdin) -print(len(d.get('items',[]))) +try: + d=json.load(sys.stdin) + print(len(d.get('items',[]))) +except Exception: + print(0) ")" - fi - if [[ "${obcItemCount:-0}" -eq 0 ]]; then + if [[ "${obcItemCount}" -eq 0 ]]; then typeset odfObcJson="" - odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json)" || true - if [[ -n "${odfObcJson}" ]]; then - obcList="$(printf '%s' "${odfObcJson}" | python3 -c " + odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || true + obcList="$(printf '%s' "${odfObcJson}" | python3 -c " import sys,json -d=json.load(sys.stdin) -obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()] -print(json.dumps({'items':obs})) +try: + d=json.load(sys.stdin) + obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()] + print(json.dumps({'items':obs})) +except Exception: + print(json.dumps({'items':[]})) ")" - fi - if [[ -n "${obcList}" ]]; then - obcItemCount="$(echo "${obcList}" | python3 -c " + obcItemCount="$(printf '%s' "${obcList}" | python3 -c " import sys,json -d=json.load(sys.stdin) -print(len(d.get('items',[]))) +try: + d=json.load(sys.stdin) + print(len(d.get('items',[]))) +except Exception: + print(0) ")" - fi fi - if [[ "${obcItemCount:-0}" -eq 0 ]]; then + if [[ "${obcItemCount}" -eq 0 ]]; then AddResult "obc-bound" "skip" "No ObjectBucketClaim found for observability" return fi @@ -565,7 +560,6 @@ print(items[0]['metadata']['name'] if items else '') return fi - set +x typeset token="" token="$(oc whoami -t)" || true @@ -575,7 +569,6 @@ print(items[0]['metadata']['name'] if items else '') -H "Authorization: Bearer ${token}" \ "https://${queryRoute}/api/v1/query?query=up" \ --max-time 30)" || true - set -x httpCode="$(echo "${responseBody}" | tail -1)" responseBody="$(echo "${responseBody}" | sed '$d')" From 4b69011644b964e069b906c4722528af8d7bd5ce Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 08:13:54 -0500 Subject: [PATCH 26/29] fix: mpitt R3 secret handling + trap form --- .../interop-tests-opp-quay-smoke-commands.sh | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index c0172a1cf3fb6..eaf164b557e59 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -78,7 +78,7 @@ EOF true } -trap '{ typeset -i rc=$?; ( GenerateJunit ); exit ${rc}; }' EXIT +trap '{ ( GenerateJunit; true ); }' EXIT function DiscoverQuay () { QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}') @@ -98,21 +98,24 @@ function GetQuayAuth () { QUAY_PASSWORD="" QUAY_TOKEN="" - if oc get secret quayadmin -n "${QUAY_NS}" &>/dev/null; then + set +x + if oc get secret quayadmin -n "${QUAY_NS}" 2>/dev/null; then QUAY_TOKEN=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_TOKEN="" QUAY_PASSWORD=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" QUAY_USER="quayadmin" if [[ -n "${QUAY_TOKEN}" || -n "${QUAY_PASSWORD}" ]]; then + set -x echo "INFO: Quay credentials obtained from quayadmin secret" export QUAY_USER QUAY_PASSWORD QUAY_TOKEN return 0 fi fi - if oc get secret quaydevel -n "${QUAY_NS}" &>/dev/null; then + if oc get secret quaydevel -n "${QUAY_NS}" 2>/dev/null; then QUAY_PASSWORD=$(oc get secret quaydevel -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" QUAY_USER="quaydevel" if [[ -n "${QUAY_PASSWORD}" ]]; then + set -x echo "INFO: Quay credentials obtained from quaydevel secret" export QUAY_USER QUAY_PASSWORD QUAY_TOKEN return 0 @@ -130,10 +133,12 @@ function GetQuayAuth () { if [[ -n "${QUAY_TOKEN}" ]]; then QUAY_USER="quayadmin" QUAY_PASSWORD="${initPassword}" + set -x echo "INFO: Quay admin user initialized via /api/v1/user/initialize" export QUAY_USER QUAY_PASSWORD QUAY_TOKEN return 0 fi + set -x echo "ERROR: Could not obtain Quay credentials from any source" >&2 export QUAY_USER QUAY_PASSWORD QUAY_TOKEN @@ -157,6 +162,7 @@ function CreateTestOrg () { if [[ -n "${csrf}" ]]; then typeset signinResult + set +x signinResult=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \ -H "Content-Type: application/json" \ -H "X-CSRF-Token: ${csrf}" \ @@ -164,6 +170,7 @@ function CreateTestOrg () { -d "{\"username\":\"${QUAY_USER}\",\"password\":\"${QUAY_PASSWORD}\"}" 2>/dev/null) || signinResult="" QUAY_TOKEN=$(echo "${signinResult}" | \ python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null) || QUAY_TOKEN="" + set -x fi rm -f "${cookieFile}" export QUAY_TOKEN @@ -198,6 +205,7 @@ function RunPushPull () { fi typeset registryAuth + set +x if [[ -n "${QUAY_TOKEN}" ]]; then registryAuth=$(echo -n "\$oauthtoken:${QUAY_TOKEN}" | base64) else @@ -207,6 +215,7 @@ function RunPushPull () { cat > "${authFile}" </dev/null | base64 -d) || acsPassword="" + set +x + acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || acsPassword="" + set -x if [[ -z "${acsPassword}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS admin password not found" "${elapsed}" @@ -391,15 +406,19 @@ function RunAcsScan () { typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" + set +x RegisterQuayInAcs "${acsHost}" "${acsPassword}" RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" + set -x typeset -i attempts=0 maxAttempts=40 while (( attempts < maxAttempts )); do typeset scanResult + set +x scanResult=$(curl -sk -u "admin:${acsPassword}" \ "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null) || scanResult="" + set -x if echo "${scanResult}" | python3 -c " import sys, json @@ -413,7 +432,9 @@ sys.exit(0 if len(images) > 0 else 1) fi if (( attempts % 4 == 3 )); then + set +x RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" + set -x fi attempts=$((attempts + 1)) From 1af6687d39763f6a52b1d575466377b7a669c4a5 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 10:44:36 -0500 Subject: [PATCH 27/29] fix: CLC best_effort + wire observability-odf step - Add best_effort: true to acm-tests-clc-smoke so CLC failures don't block independent downstream checks (odf-health, quay-smoke) - Wire interop-opp-observability-odf into 4.22, 4.22-fips, and 5.0 test sequences (after odf-health, before quay-smoke) Addresses Chai Bot cross-cutting concerns on batch PR #83813. --- .../stolostron-policy-collection-main__ocp4.22-fips.yaml | 1 + .../stolostron-policy-collection-main__ocp4.22.yaml | 2 ++ .../stolostron-policy-collection-main__ocp5.0.yaml | 2 ++ .../acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml | 7 ++++--- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml index f5050d7facf2d..fe455b70e415c 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml @@ -135,6 +135,7 @@ tests: - ref: acm-fetch-managed-clusters - ref: acm-opp-app - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: interop-tests-opp-quay-smoke - ref: acm-tests-observability zz_generated_metadata: diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml index 1864430307295..9df9af594e67c 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml @@ -133,6 +133,7 @@ tests: - ref: acm-fetch-managed-clusters - ref: acm-opp-app - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: interop-tests-opp-quay-smoke - ref: acm-tests-observability - as: interop-opp-vsphere @@ -183,6 +184,7 @@ tests: - ref: acm-policies-openshift-plus - chain: cucushift-installer-check-cluster-health - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: acm-tests-observability - ref: acm-opp-app workflow: acm-ipi-vsphere diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml index a821e99b0950e..df9364f22d947 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml @@ -115,6 +115,7 @@ tests: - ref: acm-fetch-managed-clusters - ref: acm-opp-app - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: interop-tests-opp-quay-smoke - ref: acm-tests-observability - as: interop-opp-vsphere @@ -164,6 +165,7 @@ tests: - ref: acm-policies-openshift-plus - chain: cucushift-installer-check-cluster-health - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: acm-tests-observability - ref: acm-opp-app workflow: acm-ipi-vsphere diff --git a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml index d5f301b1be8c2..775b3cda884ae 100644 --- a/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml +++ b/ci-operator/step-registry/acm/tests/clc-smoke/acm-tests-clc-smoke-ref.yaml @@ -3,6 +3,7 @@ ref: from: clc-ui-e2e commands: acm-tests-clc-smoke-commands.sh timeout: 5400s + best_effort: true resources: requests: cpu: '2' @@ -68,6 +69,6 @@ ref: When true, copies kubeconfig from CI secrets instead of using cluster profile documentation: |- Smoke-scoped ACM cluster lifecycle step that creates a single managed - cluster on AWS (~50 min). Differs from acm-tests-clc-create only in - timeout (5400s vs 28800s) and failure propagation (no || :) so that - downstream steps fail fast if cluster creation does not succeed. + cluster on AWS (~50 min). Runs with best_effort so that CLC failures + do not block independent downstream validations (ODF health, Quay smoke, + observability). JUnit results are still reported for failure visibility. From 6cb76fd435adc2e838e4162983758a0bfc1cf82f Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 13:50:26 -0500 Subject: [PATCH 28/29] fix: add best_effort to stackrox-opp-smoke ACS test suite flakes should not block independent downstream validations (ODF health, Quay smoke, observability, CLC). --- .../step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml index 74d11f0815aa9..329a34c9d7a71 100644 --- a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml +++ b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml @@ -11,6 +11,7 @@ ref: memory: 4Gi from: acs-smoke-runner timeout: 1h0m0s + best_effort: true documentation: |- Run the ACS qa-tests-backend SMOKE suite against a live ACS instance. Reads connection credentials from SHARED_DIR From fe72c79b83ed59f052515dc70eb3dd32806a350f Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 18:30:40 -0500 Subject: [PATCH 29/29] fix: observability-odf use curl instead of wget, graceful skip - Replace wget with curl (cli image doesn't have wget) - Skip instead of fail when no Thanos components are deployed (namespace exists but is empty) --- .../interop-opp-observability-odf-commands.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh index 936d2730dce00..0d7fcf9504446 100755 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -358,7 +358,7 @@ function CheckThanosHealth () { done if (( foundCount == 0 )); then - AddResult "thanos-health" "fail" "No Thanos/observability components found in ${OBS_NAMESPACE}" + AddResult "thanos-health" "skip" "No Thanos/observability components found in ${OBS_NAMESPACE}; observability not deployed" elif [[ -n "${failMsg}" ]]; then AddResult "thanos-health" "fail" "Unhealthy Thanos components: ${failMsg}" elif (( ${#missingComponents[@]} > 0 )); then @@ -532,8 +532,7 @@ print(items[0]['metadata']['name'] if items else '') typeset queryResult="" if [[ -n "${queryFrontendPod}" ]]; then queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryFrontendPod}" \ - -- wget -qO- --no-check-certificate \ - "http://localhost:9090/api/v1/query?query=up")" || true + -- curl -sk "http://localhost:9090/api/v1/query?query=up")" || true fi if [[ -z "${queryResult}" ]]; then @@ -546,8 +545,7 @@ print(items[0]['metadata']['name'] if items else '') fi if [[ -n "${queryPod}" ]]; then queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryPod}" \ - -- wget -qO- --no-check-certificate \ - "http://localhost:9090/api/v1/query?query=up")" || true + -- curl -sk "http://localhost:9090/api/v1/query?query=up")" || true fi fi