✨ report ClusterObjectSet rollout as Ready=False/ObjectsNotReady

api/v1/clusterobjectset_types.go

@@ -30,7 +30,7 @@ const (
 
 	// Condition Reasons
 	ClusterObjectSetReasonArchived          = "Archived"
-	ClusterObjectSetReasonProbeFailure      = "ProbeFailure"
+	ClusterObjectSetReasonObjectsNotReady   = "ObjectsNotReady"
 	ClusterObjectSetReasonObjectCollision   = "ObjectCollision"
 	ClusterObjectSetReasonRetryableError    = "RetryableError"
 	ClusterObjectSetReasonValidationFailure = "ValidationFailure"
@@ -499,8 +499,7 @@ type ClusterObjectSetStatus struct {
 	// objects are not at the desired state; Unknown is the initial state, before
 	// the first reconciliation has evaluated the revision.
 	//   - True with reason AllObjectsReady: the revision has rolled out and all objects are at the desired state.
-	//   - False with reason ProbeFailure: one or more objects are failing their readiness probes during rollout.
-	//   - False with reason RollingOut: the revision is actively rolling out and has not yet become ready.
+	//   - False with reason ObjectsNotReady: one or more objects are still being applied or have not yet passed their readiness probes during rollout.
 	//   - False with reason ObjectCollision: one or more objects are controlled by another owner, or are unowned while collision protection prevents adoption, so this revision cannot take ownership of them.
 	//   - False with reason ValidationFailure: the revision failed a pre-apply check (an invalid revision/phase object or probe, preflight validation, a mutable referenced Secret, or changed referenced content) and requires manual intervention.
 	//   - False with reason ProgressDeadlineExceeded: the revision did not roll out within spec.progressDeadlineMinutes.

docs/draft/concepts/clusterobjectsets.md

@@ -217,7 +217,7 @@ Indicates whether all objects in the revision have been successfully rolled out
 | Status | Reason | Meaning |
 | --- | --- | --- |
 | True | `AllObjectsReady` | All objects are at the desired state |
-| False | `ProbeFailure` | One or more probes failing |
+| False | `ObjectsNotReady` | One or more objects are still being applied or have not yet passed their readiness probes during rollout |
 | False | `ObjectCollision` | One or more objects are controlled by another owner, or are unowned and collision protection prevents adoption |
 | False | `ValidationFailure` | A pre-apply check failed (an invalid revision/phase object or probe, preflight validation, a mutable referenced Secret, or changed referenced content); requires manual intervention |
 | Unknown | `RetryableError` | Error prevented probe observation |

helm/olmv1/base/object-controller/crd/experimental/olm.operatorframework.io_clusterobjectsets.yaml

@@ -559,8 +559,7 @@ spec:
                   objects are not at the desired state; Unknown is the initial state, before
                   the first reconciliation has evaluated the revision.
                     - True with reason AllObjectsReady: the revision has rolled out and all objects are at the desired state.
-                    - False with reason ProbeFailure: one or more objects are failing their readiness probes during rollout.
-                    - False with reason RollingOut: the revision is actively rolling out and has not yet become ready.
+                    - False with reason ObjectsNotReady: one or more objects are still being applied or have not yet passed their readiness probes during rollout.
                     - False with reason ObjectCollision: one or more objects are controlled by another owner, or are unowned while collision protection prevents adoption, so this revision cannot take ownership of them.
                     - False with reason ValidationFailure: the revision failed a pre-apply check (an invalid revision/phase object or probe, preflight validation, a mutable referenced Secret, or changed referenced content) and requires manual intervention.
                     - False with reason ProgressDeadlineExceeded: the revision did not roll out within spec.progressDeadlineMinutes.

internal/object-controller/controllers/clusterobjectset_controller.go

@@ -294,11 +294,6 @@ func (c *ClusterObjectSetReconciler) reconcile(ctx context.Context, cos *ocv1.Cl
 		}
 	}
 
-	revisionNumber := cos.Spec.Revision
-	if rres.InTransition() {
-		setReadyWithDeadline(cos, metav1.ConditionFalse, ocv1.ReasonRollingOut, fmt.Sprintf("Revision %d is rolling out.", revisionNumber), isDeadlineExceeded)
-	}
-
 	//nolint:nestif
 	if rres.IsComplete() {
 		// Archive previous revisions
@@ -326,7 +321,7 @@ func (c *ClusterObjectSetReconciler) reconcile(ctx context.Context, cos *ocv1.Cl
 			cos.Status.CompletedAt = metav1.NewTime(c.Clock.Now())
 		}
 	} else {
-		var probeFailureMsgs []string
+		var notReadyMsgs []string
 		for _, pres := range rres.GetPhases() {
 			if pres.IsComplete() {
 				continue
@@ -344,29 +339,32 @@ func (c *ClusterObjectSetReconciler) reconcile(ctx context.Context, cos *ocv1.Cl
 				}
 				obj := ores.Object()
 				gvk := obj.GetObjectKind().GroupVersionKind()
-				// I think these can be pretty large and verbose. We may want to
-				// work a little on the formatting...?
-				probeFailureMsgs = append(probeFailureMsgs, fmt.Sprintf(
+				// Count every not-ready object (the condition reports the count); the
+				// per-object detail here is for the V(1) debug log, not the message.
+				notReadyMsgs = append(notReadyMsgs, fmt.Sprintf(
 					"Object %s.%s %s/%s: %v",
 					gvk.Kind, gvk.GroupVersion().String(),
 					obj.GetNamespace(), obj.GetName(), strings.Join(pr.Messages, " and "),
 				))
-				break
 			}
 		}
 
-		rollingOutMsg := fmt.Sprintf("Revision %d is rolling out.", revisionNumber)
-		switch {
-		case isDeadlineExceeded:
-			// Deadline wins over ProbeFailure. Use the generic rolling-out summary as
-			// the "last status", not the per-object probe detail, so the reconstructed
-			// ClusterExtension Progressing/ProgressDeadlineExceeded message matches the
-			// message the removed COS Progressing condition historically carried.
-			setReadyWithDeadline(cos, metav1.ConditionFalse, ocv1.ReasonRollingOut, rollingOutMsg, true)
-		case len(probeFailureMsgs) > 0:
-			setReadyWithDeadline(cos, metav1.ConditionFalse, ocv1.ClusterObjectSetReasonProbeFailure, strings.Join(probeFailureMsgs, "\n"), false)
-		default:
-			setReadyWithDeadline(cos, metav1.ConditionFalse, ocv1.ReasonRollingOut, rollingOutMsg, false)
+		// Generic summary used as the ProgressDeadlineExceeded "last status", kept
+		// stable (not the volatile per-object detail) so the deadline message does not
+		// churn. The non-deadline path reports the count. Per-object detail is kept out
+		// of the condition message and logged at V(1) for debugging.
+		const genericNotReadyMsg = "Objects are not yet ready."
+		if len(notReadyMsgs) > 0 {
+			l.V(1).Info("objects not yet ready", "count", len(notReadyMsgs), "details", notReadyMsgs)
+		}
+		if isDeadlineExceeded {
+			setReadyWithDeadline(cos, metav1.ConditionFalse, ocv1.ClusterObjectSetReasonObjectsNotReady, genericNotReadyMsg, true)
+		} else {
+			notReadyMsg := genericNotReadyMsg
+			if len(notReadyMsgs) > 0 {
+				notReadyMsg = fmt.Sprintf("%d object(s) are not yet ready.", len(notReadyMsgs))
+			}
+			setReadyWithDeadline(cos, metav1.ConditionFalse, ocv1.ClusterObjectSetReasonObjectsNotReady, notReadyMsg, false)
 		}
 		if hasDeadline && !isDeadlineExceeded {
 			return ctrl.Result{RequeueAfter: remaining}, nil

internal/object-controller/controllers/clusterobjectset_controller_test.go

@@ -148,7 +148,7 @@ func Test_ClusterObjectSetReconciler_Reconcile_RevisionReconciliation(t *testing
 			},
 		},
 		{
-			name:                    "set Ready:False:RollingOut status condition during rollout when no probe failures are detected",
+			name:                    "set Ready:False:ObjectsNotReady status condition during rollout when no probe failures are detected",
 			reconcilingRevisionName: clusterObjectSetName,
 			revisionResult:          newMockRevisionResult(mockCtrl, revisionResultConfig{}),
 			existingObjs: func() []client.Object {
@@ -165,8 +165,8 @@ func Test_ClusterObjectSetReconciler_Reconcile_RevisionReconciliation(t *testing
 				cond := meta.FindStatusCondition(rev.Status.Conditions, ocv1.ClusterObjectSetTypeReady)
 				require.NotNil(t, cond)
 				require.Equal(t, metav1.ConditionFalse, cond.Status)
-				require.Equal(t, ocv1.ReasonRollingOut, cond.Reason)
-				require.Equal(t, "Revision 1 is rolling out.", cond.Message)
+				require.Equal(t, ocv1.ClusterObjectSetReasonObjectsNotReady, cond.Reason)
+				require.Equal(t, "Objects are not yet ready.", cond.Message)
 				require.Equal(t, int64(1), cond.ObservedGeneration)
 			},
 		},
@@ -210,11 +210,11 @@ func Test_ClusterObjectSetReconciler_Reconcile_RevisionReconciliation(t *testing
 				require.Equal(t, metav1.ConditionFalse, cond.Status)
 				require.NotEqual(t, ocv1.ClusterObjectSetReasonValidationFailure, cond.Reason,
 					"a paused downstream phase's validation error must not be reported as a terminal validation failure")
-				require.Equal(t, ocv1.ReasonRollingOut, cond.Reason)
+				require.Equal(t, ocv1.ClusterObjectSetReasonObjectsNotReady, cond.Reason)
 			},
 		},
 		{
-			name:                    "set Ready:False:ProbeFailure condition when probe failures are detected and revision is in transition",
+			name:                    "set Ready:False:ObjectsNotReady condition when probe failures are detected and revision is in transition",
 			reconcilingRevisionName: clusterObjectSetName,
 			revisionResult: newMockRevisionResult(mockCtrl, revisionResultConfig{
 				inTransition: true,
@@ -296,13 +296,63 @@ func Test_ClusterObjectSetReconciler_Reconcile_RevisionReconciliation(t *testing
 				cond := meta.FindStatusCondition(rev.Status.Conditions, ocv1.ClusterObjectSetTypeReady)
 				require.NotNil(t, cond)
 				require.Equal(t, metav1.ConditionFalse, cond.Status)
-				require.Equal(t, ocv1.ClusterObjectSetReasonProbeFailure, cond.Reason)
-				require.Equal(t, "Object Service.v1 my-namespace/my-service: something bad happened and something worse happened\nObject ConfigMap.v1 my-namespace/my-configmap: we have a problem", cond.Message)
+				require.Equal(t, ocv1.ClusterObjectSetReasonObjectsNotReady, cond.Reason)
+				require.Equal(t, "2 object(s) are not yet ready.", cond.Message)
 				require.Equal(t, int64(1), cond.ObservedGeneration)
 			},
 		},
 		{
-			name:                    "set Ready:False:ProbeFailure condition when probe failures are detected and revision is not in transition",
+			// Multiple objects failing within the same phase must all be counted, so the
+			// reported count reflects objects, not phases (no early break in the loop).
+			name:                    "ObjectsNotReady counts all not-ready objects within a phase",
+			reconcilingRevisionName: clusterObjectSetName,
+			revisionResult: newMockRevisionResult(mockCtrl, revisionResultConfig{
+				inTransition: true,
+				isComplete:   false,
+				phases: []machinery.PhaseResult{
+					newMockPhaseResult(mockCtrl, phaseResultConfig{
+						name:       "somephase",
+						isComplete: false,
+						objects: []machinery.ObjectResult{
+							newMockObjectResult(mockCtrl, objectResultConfig{
+								object: func() client.Object {
+									obj := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "svc-a", Namespace: "ns"}}
+									obj.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service"))
+									return obj
+								}(),
+								probes: machinerytypes.ProbeResultContainer{
+									boxcutter.ProgressProbeType: {Status: machinerytypes.ProbeStatusFalse, Messages: []string{"not ready"}},
+								},
+							}),
+							newMockObjectResult(mockCtrl, objectResultConfig{
+								object: func() client.Object {
+									obj := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-b", Namespace: "ns"}}
+									obj.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap"))
+									return obj
+								}(),
+								probes: machinerytypes.ProbeResultContainer{
+									boxcutter.ProgressProbeType: {Status: machinerytypes.ProbeStatusFalse, Messages: []string{"not ready"}},
+								},
+							}),
+						},
+					}),
+				},
+			}),
+			existingObjs: func() []client.Object {
+				ext := newTestClusterExtension()
+				return []client.Object{ext, newTestClusterObjectSet(t, clusterObjectSetName, ext, testScheme)}
+			},
+			validate: func(t *testing.T, c client.Client) {
+				rev := &ocv1.ClusterObjectSet{}
+				require.NoError(t, c.Get(t.Context(), client.ObjectKey{Name: clusterObjectSetName}, rev))
+				cond := meta.FindStatusCondition(rev.Status.Conditions, ocv1.ClusterObjectSetTypeReady)
+				require.NotNil(t, cond)
+				require.Equal(t, ocv1.ClusterObjectSetReasonObjectsNotReady, cond.Reason)
+				require.Equal(t, "2 object(s) are not yet ready.", cond.Message)
+			},
+		},
+		{
+			name:                    "set Ready:False:ObjectsNotReady condition when probe failures are detected and revision is not in transition",
 			reconcilingRevisionName: clusterObjectSetName,
 			revisionResult: newMockRevisionResult(mockCtrl, revisionResultConfig{
 				inTransition: false,
@@ -384,13 +434,13 @@ func Test_ClusterObjectSetReconciler_Reconcile_RevisionReconciliation(t *testing
 				cond := meta.FindStatusCondition(rev.Status.Conditions, ocv1.ClusterObjectSetTypeReady)
 				require.NotNil(t, cond)
 				require.Equal(t, metav1.ConditionFalse, cond.Status)
-				require.Equal(t, ocv1.ClusterObjectSetReasonProbeFailure, cond.Reason)
-				require.Equal(t, "Object Service.v1 my-namespace/my-service: something bad happened and something worse happened\nObject ConfigMap.v1 my-namespace/my-configmap: we have a problem", cond.Message)
+				require.Equal(t, ocv1.ClusterObjectSetReasonObjectsNotReady, cond.Reason)
+				require.Equal(t, "2 object(s) are not yet ready.", cond.Message)
 				require.Equal(t, int64(1), cond.ObservedGeneration)
 			},
 		},
 		{
-			name: "set Ready:False:RollingOut condition while revision is transitioning",
+			name: "set Ready:False:ObjectsNotReady condition while revision is transitioning",
 			revisionResult: newMockRevisionResult(mockCtrl, revisionResultConfig{
 				inTransition: true,
 			}),
@@ -409,8 +459,8 @@ func Test_ClusterObjectSetReconciler_Reconcile_RevisionReconciliation(t *testing
 				cond := meta.FindStatusCondition(rev.Status.Conditions, ocv1.ClusterObjectSetTypeReady)
 				require.NotNil(t, cond)
 				require.Equal(t, metav1.ConditionFalse, cond.Status)
-				require.Equal(t, ocv1.ReasonRollingOut, cond.Reason)
-				require.Equal(t, "Revision 1 is rolling out.", cond.Message)
+				require.Equal(t, ocv1.ClusterObjectSetReasonObjectsNotReady, cond.Reason)
+				require.Equal(t, "Objects are not yet ready.", cond.Message)
 				require.Equal(t, int64(1), cond.ObservedGeneration)
 			},
 		},
@@ -1332,8 +1382,8 @@ func Test_ClusterObjectSetReconciler_Reconcile_ProgressDeadline(t *testing.T) {
 		},
 		{
 			// A rolling revision with failing probes AND an exceeded progress deadline must
-			// set Ready=False/ProgressDeadlineExceeded (deadline wins over ProbeFailure).
-			name: "Ready set to false/ProgressDeadlineExceeded when deadline is exceeded during probe failure (deadline wins over ProbeFailure)",
+			// set Ready=False/ProgressDeadlineExceeded (deadline wins over ObjectsNotReady).
+			name: "Ready set to false/ProgressDeadlineExceeded when deadline is exceeded during probe failure (deadline wins over ObjectsNotReady)",
 			existingObjs: func() []client.Object {
 				ext := newTestClusterExtension()
 				rev1 := newTestClusterObjectSet(t, clusterObjectSetName, ext, testScheme)
@@ -1383,10 +1433,9 @@ func Test_ClusterObjectSetReconciler_Reconcile_ProgressDeadline(t *testing.T) {
 				require.NotNil(t, cnd)
 				require.Equal(t, metav1.ConditionFalse, cnd.Status)
 				require.Equal(t, ocv1.ReasonProgressDeadlineExceeded, cnd.Reason)
-				// The deadline "last status" must be the generic rolling-out summary, not the
-				// per-object probe detail, so the reconstructed ClusterExtension Progressing
-				// message matches what the removed COS Progressing condition historically carried.
-				require.Equal(t, "Revision has not rolled out for 1 minute(s). Last status: Revision 1 is rolling out.", cnd.Message)
+				// The deadline "last status" must be the generic not-ready summary, not the
+				// per-object probe detail, so the deadline message stays stable.
+				require.Equal(t, "Revision has not rolled out for 1 minute(s). Last status: Objects are not yet ready.", cnd.Message)
 			},
 		},
 		{
@@ -1411,7 +1460,7 @@ func Test_ClusterObjectSetReconciler_Reconcile_ProgressDeadline(t *testing.T) {
 				require.NoError(t, err)
 				cnd := meta.FindStatusCondition(rev.Status.Conditions, ocv1.ClusterObjectSetTypeReady)
 				require.Equal(t, metav1.ConditionFalse, cnd.Status)
-				require.Equal(t, ocv1.ReasonRollingOut, cnd.Reason)
+				require.Equal(t, ocv1.ClusterObjectSetReasonObjectsNotReady, cnd.Reason)
 			},
 		},
 		{
@@ -1425,7 +1474,7 @@ func Test_ClusterObjectSetReconciler_Reconcile_ProgressDeadline(t *testing.T) {
 					Type:               ocv1.ClusterObjectSetTypeReady,
 					Status:             metav1.ConditionFalse,
 					Reason:             ocv1.ReasonProgressDeadlineExceeded,
-					Message:            "Revision has not rolled out for 1 minute(s). Last status: Revision 1 is rolling out.",
+					Message:            "Revision has not rolled out for 1 minute(s). Last status: Objects are not yet ready.",
 					ObservedGeneration: rev1.Generation,
 				})
 				return []client.Object{rev1, ext}
@@ -1474,7 +1523,7 @@ func Test_ClusterObjectSetReconciler_Reconcile_ProgressDeadline(t *testing.T) {
 				require.NoError(t, err)
 				cnd := meta.FindStatusCondition(rev.Status.Conditions, ocv1.ClusterObjectSetTypeReady)
 				require.Equal(t, metav1.ConditionFalse, cnd.Status)
-				require.Equal(t, ocv1.ReasonRollingOut, cnd.Reason)
+				require.Equal(t, ocv1.ClusterObjectSetReasonObjectsNotReady, cnd.Reason)
 			},
 		},
 	} {

internal/operator-controller/controllers/common_controller.go

@@ -276,7 +276,7 @@ func progressingFromReady(ready *metav1.Condition, completed bool) metav1.Condit
 	case ocv1.ClusterObjectSetReasonRetryableError, ocv1.ClusterObjectSetReasonObjectCollision:
 		cond.Reason = ocv1.ReasonRetrying
 	default:
-		// ProbeFailure, RollingOut, or AllObjectsReady-but-not-yet-complete.
+		// Any other (non-error) reason on a not-yet-complete revision is rolling out.
 		cond.Reason = ocv1.ReasonRollingOut
 	}
 	return cond