From 972ddda4782b5e0c69ed1c60e8b01d79b1b67de2 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Thu, 25 Jun 2026 18:55:41 -0400 Subject: [PATCH 01/15] Netobserv Day0 test case --- .../networking/network_observability.go | 293 ++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 test/extended/networking/network_observability.go diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go new file mode 100644 index 000000000000..2a9e92465ac9 --- /dev/null +++ b/test/extended/networking/network_observability.go @@ -0,0 +1,293 @@ +package networking + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + netobservOperatorNamespace = "openshift-netobserv-operator" + netobservNamespace = "openshift-network-observability" + netobservPrivilegedNS = "openshift-network-observability-privileged" + flowCollectorName = "cluster" + flpMetricsPort = "9401" +) + +type flowCollectorCondition struct { + Type string `json:"type"` + Status string `json:"status"` +} + +var _ = g.Describe("[sig-network][Feature:NetObserv]", func() { + oc := exutil.NewCLIWithoutNamespace("netobserv-e2e") + + g.It("should not be installed on single node clusters", func(ctx context.Context) { + isSingleNode, err := exutil.IsSingleNode(ctx, oc.AdminConfigClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if !isSingleNode { + g.Skip("test only applies to single node clusters") + } + + g.By("checking that the operator namespace does not exist") + _, err = oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, netobservOperatorNamespace, metav1.GetOptions{}) + o.Expect(err).To(o.HaveOccurred(), + "Network observability operator namespace %q should not exist on single node clusters", netobservOperatorNamespace) + + g.By("checking that the workload namespace does not exist") + _, err = oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, netobservNamespace, metav1.GetOptions{}) + o.Expect(err).To(o.HaveOccurred(), + "Network observability namespace %q should not exist on single node clusters", netobservNamespace) + + g.By("checking that the FlowCollector CRD is not installed") + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("crd", "flowcollectors.flows.netobserv.io").Output() + if err == nil { + framework.Failf("FlowCollector CRD should not be installed on single node clusters, but found: %s", output) + } + }) + + g.It("should have all components healthy and producing flow data", func(ctx context.Context) { + g.By("verifying operator namespace exists") + _, err := oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, netobservOperatorNamespace, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), + "Network observability operator namespace %q must exist", netobservOperatorNamespace) + + g.By("checking FlowCollector CR has Ready status") + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "flowcollector", flowCollectorName, + "-o=jsonpath={.status.conditions[*]}", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred(), "FlowCollector CR %q should exist", flowCollectorName) + + var conditions []flowCollectorCondition + condJSON := "[" + strings.ReplaceAll(strings.TrimSpace(output), "} {", "},{") + "]" + err = json.Unmarshal([]byte(condJSON), &conditions) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to parse FlowCollector conditions") + + ready := false + for _, c := range conditions { + if c.Type == "Ready" && c.Status == "True" { + ready = true + break + } + } + o.Expect(ready).To(o.BeTrue(), "FlowCollector should have Ready=True condition") + + g.By("checking operator pod is running") + pods, err := oc.AdminKubeClient().CoreV1().Pods(netobservOperatorNamespace).List(ctx, metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(pods.Items).NotTo(o.BeEmpty(), "expected at least one pod in %s", netobservOperatorNamespace) + + found := false + for _, pod := range pods.Items { + if strings.Contains(pod.Name, "netobserv-controller-manager") { + o.Expect(string(pod.Status.Phase)).To(o.Equal("Running"), + "netobserv-controller-manager pod should be Running, got %s", pod.Status.Phase) + found = true + break + } + } + o.Expect(found).To(o.BeTrue(), "netobserv-controller-manager pod not found in %s", netobservOperatorNamespace) + + g.By("checking FLP pods are running") + o.Eventually(func() bool { + flpPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=flowlogs-pipeline", + }) + if err != nil { + framework.Logf("Error listing FLP pods: %v", err) + return false + } + if len(flpPods.Items) == 0 { + framework.Logf("No FLP pods found in %s", netobservNamespace) + return false + } + for _, pod := range flpPods.Items { + if pod.Status.Phase != "Running" { + framework.Logf("FLP pod %s is %s, not Running", pod.Name, pod.Status.Phase) + return false + } + } + return true + }, 3*time.Minute, 5*time.Second).Should(o.BeTrue(), "FLP pods should be Running") + + g.By("checking eBPF agent DaemonSet readiness") + o.Eventually(func() bool { + ds, err := oc.AdminKubeClient().AppsV1().DaemonSets(netobservPrivilegedNS).List(ctx, metav1.ListOptions{}) + if err != nil { + framework.Logf("Error listing DaemonSets in %s: %v", netobservPrivilegedNS, err) + return false + } + for _, d := range ds.Items { + if strings.Contains(d.Name, "netobserv-ebpf-agent") { + desired := d.Status.DesiredNumberScheduled + readyCount := d.Status.NumberReady + if desired == 0 { + framework.Logf("eBPF DaemonSet desired=0") + return false + } + if desired != readyCount { + framework.Logf("eBPF DaemonSet desired=%d ready=%d", desired, readyCount) + return false + } + return true + } + } + framework.Logf("No eBPF agent DaemonSet found in %s", netobservPrivilegedNS) + return false + }, 3*time.Minute, 5*time.Second).Should(o.BeTrue(), "eBPF agent DaemonSet should have desired=ready") + + g.By("verifying all eBPF agent pods are Running") + ebpfPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservPrivilegedNS).List(ctx, metav1.ListOptions{ + LabelSelector: "app=netobserv-ebpf-agent", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(ebpfPods.Items).NotTo(o.BeEmpty(), "expected eBPF agent pods in %s", netobservPrivilegedNS) + for _, pod := range ebpfPods.Items { + o.Expect(string(pod.Status.Phase)).To(o.Equal("Running"), + "eBPF agent pod %s should be Running", pod.Name) + } + + g.By("checking console plugin if deployed") + pluginPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=netobserv-plugin", + }) + if err == nil && len(pluginPods.Items) > 0 { + for _, pod := range pluginPods.Items { + o.Expect(string(pod.Status.Phase)).To(o.Equal("Running"), + "console plugin pod %s should be Running", pod.Name) + } + + pluginOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "consoleplugin", "netobserv-plugin", + "-o=jsonpath={.metadata.name}", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(pluginOutput).To(o.Equal("netobserv-plugin")) + } else { + framework.Logf("Console plugin not deployed, skipping console plugin checks") + } + + g.By("checking operator logs for excessive errors") + logOutput, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args( + "-n", netobservOperatorNamespace, + "deployment/netobserv-controller-manager", + "--tail=50", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + errorLines := []string{} + for _, line := range strings.Split(logOutput, "\n") { + lower := strings.ToLower(line) + if strings.Contains(lower, "error") && !strings.Contains(lower, "loglevel") { + errorLines = append(errorLines, line) + } + } + if len(errorLines) > 5 { + framework.Logf("WARNING: found %d error lines in operator logs:\n%s", + len(errorLines), strings.Join(errorLines[:5], "\n")) + } + + g.By("checking ServiceMonitors exist") + smOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "servicemonitor", "-n", netobservNamespace, + "-o=jsonpath={.items[*].metadata.name}", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.TrimSpace(smOutput)).NotTo(o.BeEmpty(), + "expected at least one ServiceMonitor in %s", netobservNamespace) + framework.Logf("ServiceMonitors in %s: %s", netobservNamespace, smOutput) + + smPrivOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "servicemonitor", "-n", netobservPrivilegedNS, + "-o=jsonpath={.items[*].metadata.name}", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.TrimSpace(smPrivOutput)).NotTo(o.BeEmpty(), + "expected at least one ServiceMonitor in %s", netobservPrivilegedNS) + framework.Logf("ServiceMonitors in %s: %s", netobservPrivilegedNS, smPrivOutput) + + g.By("checking alert rules are deployed") + rulesOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "prometheusrules", "-n", netobservNamespace, + "-o=jsonpath={.items[*].metadata.name}", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.TrimSpace(rulesOutput)).NotTo(o.BeEmpty(), + "expected PrometheusRules in %s", netobservNamespace) + framework.Logf("PrometheusRules in %s: %s", netobservNamespace, rulesOutput) + + g.By("verifying FLP is producing and processing flow data") + flpPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=flowlogs-pipeline", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(flpPods.Items).NotTo(o.BeEmpty(), "no FLP pods found") + + flpPod := flpPods.Items[0].Name + o.Eventually(func() bool { + metricsOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args( + "-n", netobservNamespace, flpPod, "--", + "curl", "-s", fmt.Sprintf("http://localhost:%s/metrics", flpMetricsPort), + ).Output() + if err != nil { + framework.Logf("Error querying FLP metrics: %v", err) + return false + } + + for _, line := range strings.Split(metricsOutput, "\n") { + if strings.HasPrefix(line, "netobserv_ingest_flows_processed") && !strings.HasPrefix(line, "#") { + parts := strings.Fields(line) + if len(parts) >= 2 { + val, err := strconv.ParseFloat(parts[len(parts)-1], 64) + if err == nil && val > 0 { + framework.Logf("FLP processed flows metric: %v", val) + return true + } + } + } + } + framework.Logf("netobserv_ingest_flows_processed metric is zero or not found") + return false + }, 3*time.Minute, 10*time.Second).Should(o.BeTrue(), + "FLP should show non-zero netobserv_ingest_flows_processed metric") + + g.By("verifying Prometheus is scraping NetObserv metrics") + o.Eventually(func() bool { + promOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args( + "-n", "openshift-monitoring", + "prometheus-k8s-0", "-c", "prometheus", "--", + "curl", "-s", + "http://localhost:9090/api/v1/query?query=netobserv_ingest_flows_processed", + ).Output() + if err != nil { + framework.Logf("Error querying Prometheus: %v", err) + return false + } + + type promResult struct { + Data struct { + Result []interface{} `json:"result"` + } `json:"data"` + } + var result promResult + if err := json.Unmarshal([]byte(promOutput), &result); err != nil { + framework.Logf("Error parsing Prometheus response: %v", err) + return false + } + count := len(result.Data.Result) + framework.Logf("Prometheus netobserv_ingest_flows_processed result count: %d", count) + return count > 0 + }, 5*time.Minute, 15*time.Second).Should(o.BeTrue(), + "Prometheus should have netobserv_ingest_flows_processed results") + }) +}) \ No newline at end of file From a4d945e3d988d0284103b3da64d6dc9d4219e17d Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Thu, 25 Jun 2026 19:12:49 -0400 Subject: [PATCH 02/15] Coderabbiati suggestions --- .../networking/network_observability.go | 72 ++++++++++++++----- 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index 2a9e92465ac9..6742df910afd 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -11,6 +11,7 @@ import ( g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" exutil "github.com/openshift/origin/test/extended/util" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" ) @@ -40,37 +41,43 @@ var _ = g.Describe("[sig-network][Feature:NetObserv]", func() { g.By("checking that the operator namespace does not exist") _, err = oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, netobservOperatorNamespace, metav1.GetOptions{}) - o.Expect(err).To(o.HaveOccurred(), - "Network observability operator namespace %q should not exist on single node clusters", netobservOperatorNamespace) + o.Expect(apierrors.IsNotFound(err)).To(o.BeTrue(), + "Network observability operator namespace %q should not exist on single node clusters (err: %v)", netobservOperatorNamespace, err) g.By("checking that the workload namespace does not exist") _, err = oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, netobservNamespace, metav1.GetOptions{}) - o.Expect(err).To(o.HaveOccurred(), - "Network observability namespace %q should not exist on single node clusters", netobservNamespace) + o.Expect(apierrors.IsNotFound(err)).To(o.BeTrue(), + "Network observability namespace %q should not exist on single node clusters (err: %v)", netobservNamespace, err) g.By("checking that the FlowCollector CRD is not installed") - output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("crd", "flowcollectors.flows.netobserv.io").Output() - if err == nil { - framework.Failf("FlowCollector CRD should not be installed on single node clusters, but found: %s", output) - } + crdOutput, crdErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("crd", "flowcollectors.flows.netobserv.io").Output() + o.Expect(crdErr).To(o.HaveOccurred(), + "FlowCollector CRD should not be installed on single node clusters, but found: %s", crdOutput) + o.Expect(strings.Contains(crdOutput, "NotFound") || strings.Contains(crdOutput, "not found")).To(o.BeTrue(), + "expected not-found error for FlowCollector CRD, got: %s", crdOutput) }) g.It("should have all components healthy and producing flow data", func(ctx context.Context) { + isSingleNode, err := exutil.IsSingleNode(ctx, oc.AdminConfigClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isSingleNode { + g.Skip("NetObserv is not expected on single node clusters") + } + g.By("verifying operator namespace exists") - _, err := oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, netobservOperatorNamespace, metav1.GetOptions{}) + _, err = oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, netobservOperatorNamespace, metav1.GetOptions{}) o.Expect(err).NotTo(o.HaveOccurred(), "Network observability operator namespace %q must exist", netobservOperatorNamespace) g.By("checking FlowCollector CR has Ready status") output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( "flowcollector", flowCollectorName, - "-o=jsonpath={.status.conditions[*]}", + "-o=jsonpath={.status.conditions}", ).Output() o.Expect(err).NotTo(o.HaveOccurred(), "FlowCollector CR %q should exist", flowCollectorName) var conditions []flowCollectorCondition - condJSON := "[" + strings.ReplaceAll(strings.TrimSpace(output), "} {", "},{") + "]" - err = json.Unmarshal([]byte(condJSON), &conditions) + err = json.Unmarshal([]byte(output), &conditions) o.Expect(err).NotTo(o.HaveOccurred(), "failed to parse FlowCollector conditions") ready := false @@ -187,8 +194,7 @@ var _ = g.Describe("[sig-network][Feature:NetObserv]", func() { errorLines := []string{} for _, line := range strings.Split(logOutput, "\n") { - lower := strings.ToLower(line) - if strings.Contains(lower, "error") && !strings.Contains(lower, "loglevel") { + if strings.Contains(line, "\"level\":\"error\"") || strings.Contains(line, "level=error") { errorLines = append(errorLines, line) } } @@ -262,10 +268,17 @@ var _ = g.Describe("[sig-network][Feature:NetObserv]", func() { "FLP should show non-zero netobserv_ingest_flows_processed metric") g.By("verifying Prometheus is scraping NetObserv metrics") + promPods, err := oc.AdminKubeClient().CoreV1().Pods("openshift-monitoring").List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=prometheus", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(promPods.Items).NotTo(o.BeEmpty(), "expected at least one Prometheus pod in openshift-monitoring") + promPodName := promPods.Items[0].Name + o.Eventually(func() bool { promOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args( "-n", "openshift-monitoring", - "prometheus-k8s-0", "-c", "prometheus", "--", + promPodName, "-c", "prometheus", "--", "curl", "-s", "http://localhost:9090/api/v1/query?query=netobserv_ingest_flows_processed", ).Output() @@ -276,7 +289,9 @@ var _ = g.Describe("[sig-network][Feature:NetObserv]", func() { type promResult struct { Data struct { - Result []interface{} `json:"result"` + Result []struct { + Value []json.RawMessage `json:"value"` + } `json:"result"` } `json:"data"` } var result promResult @@ -284,9 +299,28 @@ var _ = g.Describe("[sig-network][Feature:NetObserv]", func() { framework.Logf("Error parsing Prometheus response: %v", err) return false } - count := len(result.Data.Result) - framework.Logf("Prometheus netobserv_ingest_flows_processed result count: %d", count) - return count > 0 + if len(result.Data.Result) == 0 { + framework.Logf("Prometheus netobserv_ingest_flows_processed: no results yet") + return false + } + for _, r := range result.Data.Result { + if len(r.Value) >= 2 { + var valStr string + if err := json.Unmarshal(r.Value[1], &valStr); err != nil { + continue + } + val, err := strconv.ParseFloat(valStr, 64) + if err != nil { + continue + } + if val > 0 { + framework.Logf("Prometheus netobserv_ingest_flows_processed sample value: %v", val) + return true + } + } + } + framework.Logf("Prometheus netobserv_ingest_flows_processed: all sample values are zero") + return false }, 5*time.Minute, 15*time.Second).Should(o.BeTrue(), "Prometheus should have netobserv_ingest_flows_processed results") }) From c698dc0a885eb6257633a7e434571c28994e1311 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Thu, 25 Jun 2026 19:22:45 -0400 Subject: [PATCH 03/15] test Warnings --- .../networking/network_observability.go | 483 +++++++++--------- 1 file changed, 254 insertions(+), 229 deletions(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index 6742df910afd..d5e4e30d1cb9 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -57,271 +57,296 @@ var _ = g.Describe("[sig-network][Feature:NetObserv]", func() { "expected not-found error for FlowCollector CRD, got: %s", crdOutput) }) - g.It("should have all components healthy and producing flow data", func(ctx context.Context) { - isSingleNode, err := exutil.IsSingleNode(ctx, oc.AdminConfigClient()) - o.Expect(err).NotTo(o.HaveOccurred()) - if isSingleNode { - g.Skip("NetObserv is not expected on single node clusters") - } + g.Context("health checks", func() { + g.BeforeEach(func(ctx context.Context) { + isSingleNode, err := exutil.IsSingleNode(ctx, oc.AdminConfigClient()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to determine cluster topology") + if isSingleNode { + g.Skip("NetObserv is not expected on single node clusters") + } - g.By("verifying operator namespace exists") - _, err = oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, netobservOperatorNamespace, metav1.GetOptions{}) - o.Expect(err).NotTo(o.HaveOccurred(), - "Network observability operator namespace %q must exist", netobservOperatorNamespace) - - g.By("checking FlowCollector CR has Ready status") - output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( - "flowcollector", flowCollectorName, - "-o=jsonpath={.status.conditions}", - ).Output() - o.Expect(err).NotTo(o.HaveOccurred(), "FlowCollector CR %q should exist", flowCollectorName) - - var conditions []flowCollectorCondition - err = json.Unmarshal([]byte(output), &conditions) - o.Expect(err).NotTo(o.HaveOccurred(), "failed to parse FlowCollector conditions") - - ready := false - for _, c := range conditions { - if c.Type == "Ready" && c.Status == "True" { - ready = true - break + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to determine if cluster is MicroShift") + if isMicroShift { + g.Skip("NetObserv is not supported on MicroShift") } - } - o.Expect(ready).To(o.BeTrue(), "FlowCollector should have Ready=True condition") + }) - g.By("checking operator pod is running") - pods, err := oc.AdminKubeClient().CoreV1().Pods(netobservOperatorNamespace).List(ctx, metav1.ListOptions{}) - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(pods.Items).NotTo(o.BeEmpty(), "expected at least one pod in %s", netobservOperatorNamespace) + g.It("should have FlowCollector CR in Ready state", func(ctx context.Context) { + g.By("verifying operator namespace exists") + _, err := oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, netobservOperatorNamespace, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), + "Network observability operator namespace %q must exist", netobservOperatorNamespace) - found := false - for _, pod := range pods.Items { - if strings.Contains(pod.Name, "netobserv-controller-manager") { - o.Expect(string(pod.Status.Phase)).To(o.Equal("Running"), - "netobserv-controller-manager pod should be Running, got %s", pod.Status.Phase) - found = true - break - } - } - o.Expect(found).To(o.BeTrue(), "netobserv-controller-manager pod not found in %s", netobservOperatorNamespace) + g.By("checking FlowCollector CR has Ready status") + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "flowcollector", flowCollectorName, + "-o=jsonpath={.status.conditions}", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred(), "FlowCollector CR %q should exist", flowCollectorName) - g.By("checking FLP pods are running") - o.Eventually(func() bool { - flpPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: "app=flowlogs-pipeline", - }) - if err != nil { - framework.Logf("Error listing FLP pods: %v", err) - return false + var conditions []flowCollectorCondition + err = json.Unmarshal([]byte(output), &conditions) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to parse FlowCollector conditions") + + ready := false + for _, c := range conditions { + if c.Type == "Ready" && c.Status == "True" { + ready = true + break + } } - if len(flpPods.Items) == 0 { - framework.Logf("No FLP pods found in %s", netobservNamespace) - return false + o.Expect(ready).To(o.BeTrue(), "FlowCollector should have Ready=True condition") + }) + + g.It("should have operator pod running", func(ctx context.Context) { + pods, err := oc.AdminKubeClient().CoreV1().Pods(netobservOperatorNamespace).List(ctx, metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to list pods in %s", netobservOperatorNamespace) + o.Expect(pods.Items).NotTo(o.BeEmpty(), "expected at least one pod in %s", netobservOperatorNamespace) + + found := false + for _, pod := range pods.Items { + if strings.Contains(pod.Name, "netobserv-controller-manager") { + o.Expect(string(pod.Status.Phase)).To(o.Equal("Running"), + "netobserv-controller-manager pod should be Running, got %s", pod.Status.Phase) + found = true + break + } } - for _, pod := range flpPods.Items { - if pod.Status.Phase != "Running" { - framework.Logf("FLP pod %s is %s, not Running", pod.Name, pod.Status.Phase) + o.Expect(found).To(o.BeTrue(), "netobserv-controller-manager pod not found in %s", netobservOperatorNamespace) + }) + + g.It("should have FLP pods running", func(ctx context.Context) { + o.Eventually(func() bool { + flpPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=flowlogs-pipeline", + }) + if err != nil { + framework.Logf("failed to list FLP pods") return false } - } - return true - }, 3*time.Minute, 5*time.Second).Should(o.BeTrue(), "FLP pods should be Running") - - g.By("checking eBPF agent DaemonSet readiness") - o.Eventually(func() bool { - ds, err := oc.AdminKubeClient().AppsV1().DaemonSets(netobservPrivilegedNS).List(ctx, metav1.ListOptions{}) - if err != nil { - framework.Logf("Error listing DaemonSets in %s: %v", netobservPrivilegedNS, err) - return false - } - for _, d := range ds.Items { - if strings.Contains(d.Name, "netobserv-ebpf-agent") { - desired := d.Status.DesiredNumberScheduled - readyCount := d.Status.NumberReady - if desired == 0 { - framework.Logf("eBPF DaemonSet desired=0") + if len(flpPods.Items) == 0 { + framework.Logf("no FLP pods found in %s", netobservNamespace) + return false + } + for _, pod := range flpPods.Items { + if pod.Status.Phase != "Running" { + framework.Logf("FLP pod %s phase is %s", pod.Name, pod.Status.Phase) return false } - if desired != readyCount { - framework.Logf("eBPF DaemonSet desired=%d ready=%d", desired, readyCount) - return false + } + return true + }, 3*time.Minute, 5*time.Second).Should(o.BeTrue(), "FLP pods should be Running in %s", netobservNamespace) + }) + + g.It("should have eBPF agent DaemonSet fully ready", func(ctx context.Context) { + g.By("checking eBPF agent DaemonSet readiness") + o.Eventually(func() bool { + ds, err := oc.AdminKubeClient().AppsV1().DaemonSets(netobservPrivilegedNS).List(ctx, metav1.ListOptions{}) + if err != nil { + framework.Logf("failed to list DaemonSets in %s", netobservPrivilegedNS) + return false + } + for _, d := range ds.Items { + if strings.Contains(d.Name, "netobserv-ebpf-agent") { + desired := d.Status.DesiredNumberScheduled + readyCount := d.Status.NumberReady + if desired == 0 { + framework.Logf("eBPF DaemonSet desired=0") + return false + } + if desired != readyCount { + framework.Logf("eBPF DaemonSet desired=%d ready=%d", desired, readyCount) + return false + } + return true } - return true } - } - framework.Logf("No eBPF agent DaemonSet found in %s", netobservPrivilegedNS) - return false - }, 3*time.Minute, 5*time.Second).Should(o.BeTrue(), "eBPF agent DaemonSet should have desired=ready") + framework.Logf("no eBPF agent DaemonSet found in %s", netobservPrivilegedNS) + return false + }, 3*time.Minute, 5*time.Second).Should(o.BeTrue(), "eBPF agent DaemonSet should have desired=ready") - g.By("verifying all eBPF agent pods are Running") - ebpfPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservPrivilegedNS).List(ctx, metav1.ListOptions{ - LabelSelector: "app=netobserv-ebpf-agent", + g.By("verifying all eBPF agent pods are Running") + ebpfPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservPrivilegedNS).List(ctx, metav1.ListOptions{ + LabelSelector: "app=netobserv-ebpf-agent", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to list eBPF agent pods in %s", netobservPrivilegedNS) + o.Expect(ebpfPods.Items).NotTo(o.BeEmpty(), "expected eBPF agent pods in %s", netobservPrivilegedNS) + for _, pod := range ebpfPods.Items { + o.Expect(string(pod.Status.Phase)).To(o.Equal("Running"), + "eBPF agent pod %s should be Running", pod.Name) + } }) - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(ebpfPods.Items).NotTo(o.BeEmpty(), "expected eBPF agent pods in %s", netobservPrivilegedNS) - for _, pod := range ebpfPods.Items { - o.Expect(string(pod.Status.Phase)).To(o.Equal("Running"), - "eBPF agent pod %s should be Running", pod.Name) - } - g.By("checking console plugin if deployed") - pluginPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: "app=netobserv-plugin", - }) - if err == nil && len(pluginPods.Items) > 0 { + g.It("should have console plugin healthy if deployed [apigroup:console.openshift.io]", func(ctx context.Context) { + pluginPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=netobserv-plugin", + }) + if err != nil || len(pluginPods.Items) == 0 { + g.Skip("console plugin not deployed") + } + for _, pod := range pluginPods.Items { o.Expect(string(pod.Status.Phase)).To(o.Equal("Running"), - "console plugin pod %s should be Running", pod.Name) + "console plugin pod %s should be Running, got %s", pod.Name, pod.Status.Phase) } pluginOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( "consoleplugin", "netobserv-plugin", "-o=jsonpath={.metadata.name}", ).Output() - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(pluginOutput).To(o.Equal("netobserv-plugin")) - } else { - framework.Logf("Console plugin not deployed, skipping console plugin checks") - } + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get consoleplugin resource") + o.Expect(pluginOutput).To(o.Equal("netobserv-plugin"), + "consoleplugin resource name mismatch") + }) - g.By("checking operator logs for excessive errors") - logOutput, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args( - "-n", netobservOperatorNamespace, - "deployment/netobserv-controller-manager", - "--tail=50", - ).Output() - o.Expect(err).NotTo(o.HaveOccurred()) + g.It("should not have excessive errors in operator logs", func(ctx context.Context) { + logOutput, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args( + "-n", netobservOperatorNamespace, + "deployment/netobserv-controller-manager", + "--tail=50", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to fetch operator logs") - errorLines := []string{} - for _, line := range strings.Split(logOutput, "\n") { - if strings.Contains(line, "\"level\":\"error\"") || strings.Contains(line, "level=error") { - errorLines = append(errorLines, line) + errorCount := 0 + for _, line := range strings.Split(logOutput, "\n") { + if strings.Contains(line, "\"level\":\"error\"") || strings.Contains(line, "level=error") { + errorCount++ + } } - } - if len(errorLines) > 5 { - framework.Logf("WARNING: found %d error lines in operator logs:\n%s", - len(errorLines), strings.Join(errorLines[:5], "\n")) - } - - g.By("checking ServiceMonitors exist") - smOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( - "servicemonitor", "-n", netobservNamespace, - "-o=jsonpath={.items[*].metadata.name}", - ).Output() - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(strings.TrimSpace(smOutput)).NotTo(o.BeEmpty(), - "expected at least one ServiceMonitor in %s", netobservNamespace) - framework.Logf("ServiceMonitors in %s: %s", netobservNamespace, smOutput) - - smPrivOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( - "servicemonitor", "-n", netobservPrivilegedNS, - "-o=jsonpath={.items[*].metadata.name}", - ).Output() - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(strings.TrimSpace(smPrivOutput)).NotTo(o.BeEmpty(), - "expected at least one ServiceMonitor in %s", netobservPrivilegedNS) - framework.Logf("ServiceMonitors in %s: %s", netobservPrivilegedNS, smPrivOutput) - - g.By("checking alert rules are deployed") - rulesOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( - "prometheusrules", "-n", netobservNamespace, - "-o=jsonpath={.items[*].metadata.name}", - ).Output() - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(strings.TrimSpace(rulesOutput)).NotTo(o.BeEmpty(), - "expected PrometheusRules in %s", netobservNamespace) - framework.Logf("PrometheusRules in %s: %s", netobservNamespace, rulesOutput) + o.Expect(errorCount).To(o.BeNumerically("<=", 5), + "found %d error-level log entries in the last 50 operator log lines (threshold: 5)", errorCount) + }) - g.By("verifying FLP is producing and processing flow data") - flpPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: "app=flowlogs-pipeline", + g.It("should have monitoring resources deployed", func(ctx context.Context) { + g.By("checking ServiceMonitors exist") + smCount, err := countResources(oc, "servicemonitor", netobservNamespace) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to list ServiceMonitors in %s", netobservNamespace) + o.Expect(smCount).To(o.BeNumerically(">", 0), + "expected at least one ServiceMonitor in %s", netobservNamespace) + framework.Logf("Found %d ServiceMonitor(s) in %s", smCount, netobservNamespace) + + smPrivCount, err := countResources(oc, "servicemonitor", netobservPrivilegedNS) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to list ServiceMonitors in %s", netobservPrivilegedNS) + o.Expect(smPrivCount).To(o.BeNumerically(">", 0), + "expected at least one ServiceMonitor in %s", netobservPrivilegedNS) + framework.Logf("Found %d ServiceMonitor(s) in %s", smPrivCount, netobservPrivilegedNS) + + g.By("checking alert rules are deployed") + rulesCount, err := countResources(oc, "prometheusrules", netobservNamespace) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to list PrometheusRules in %s", netobservNamespace) + o.Expect(rulesCount).To(o.BeNumerically(">", 0), + "expected at least one PrometheusRule in %s", netobservNamespace) + framework.Logf("Found %d PrometheusRule(s) in %s", rulesCount, netobservNamespace) }) - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(flpPods.Items).NotTo(o.BeEmpty(), "no FLP pods found") - flpPod := flpPods.Items[0].Name - o.Eventually(func() bool { - metricsOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args( - "-n", netobservNamespace, flpPod, "--", - "curl", "-s", fmt.Sprintf("http://localhost:%s/metrics", flpMetricsPort), - ).Output() - if err != nil { - framework.Logf("Error querying FLP metrics: %v", err) - return false - } + g.It("should have FLP producing non-zero flow data", func(ctx context.Context) { + flpPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=flowlogs-pipeline", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to list FLP pods in %s", netobservNamespace) + o.Expect(flpPods.Items).NotTo(o.BeEmpty(), "no FLP pods found in %s", netobservNamespace) + + flpPod := flpPods.Items[0].Name + o.Eventually(func() bool { + metricsOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args( + "-n", netobservNamespace, flpPod, "--", + "curl", "-s", fmt.Sprintf("http://localhost:%s/metrics", flpMetricsPort), + ).Output() + if err != nil { + framework.Logf("failed to query FLP metrics endpoint") + return false + } - for _, line := range strings.Split(metricsOutput, "\n") { - if strings.HasPrefix(line, "netobserv_ingest_flows_processed") && !strings.HasPrefix(line, "#") { - parts := strings.Fields(line) - if len(parts) >= 2 { - val, err := strconv.ParseFloat(parts[len(parts)-1], 64) - if err == nil && val > 0 { - framework.Logf("FLP processed flows metric: %v", val) - return true + for _, line := range strings.Split(metricsOutput, "\n") { + if strings.HasPrefix(line, "netobserv_ingest_flows_processed") && !strings.HasPrefix(line, "#") { + parts := strings.Fields(line) + if len(parts) >= 2 { + val, err := strconv.ParseFloat(parts[len(parts)-1], 64) + if err == nil && val > 0 { + framework.Logf("FLP processed flows metric: %v", val) + return true + } } } } - } - framework.Logf("netobserv_ingest_flows_processed metric is zero or not found") - return false - }, 3*time.Minute, 10*time.Second).Should(o.BeTrue(), - "FLP should show non-zero netobserv_ingest_flows_processed metric") - - g.By("verifying Prometheus is scraping NetObserv metrics") - promPods, err := oc.AdminKubeClient().CoreV1().Pods("openshift-monitoring").List(ctx, metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=prometheus", - }) - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(promPods.Items).NotTo(o.BeEmpty(), "expected at least one Prometheus pod in openshift-monitoring") - promPodName := promPods.Items[0].Name - - o.Eventually(func() bool { - promOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args( - "-n", "openshift-monitoring", - promPodName, "-c", "prometheus", "--", - "curl", "-s", - "http://localhost:9090/api/v1/query?query=netobserv_ingest_flows_processed", - ).Output() - if err != nil { - framework.Logf("Error querying Prometheus: %v", err) + framework.Logf("netobserv_ingest_flows_processed metric is zero or not found") return false - } + }, 3*time.Minute, 10*time.Second).Should(o.BeTrue(), + "FLP should show non-zero netobserv_ingest_flows_processed metric") + }) - type promResult struct { - Data struct { - Result []struct { - Value []json.RawMessage `json:"value"` - } `json:"result"` - } `json:"data"` - } - var result promResult - if err := json.Unmarshal([]byte(promOutput), &result); err != nil { - framework.Logf("Error parsing Prometheus response: %v", err) - return false - } - if len(result.Data.Result) == 0 { - framework.Logf("Prometheus netobserv_ingest_flows_processed: no results yet") - return false - } - for _, r := range result.Data.Result { - if len(r.Value) >= 2 { - var valStr string - if err := json.Unmarshal(r.Value[1], &valStr); err != nil { - continue - } - val, err := strconv.ParseFloat(valStr, 64) - if err != nil { - continue - } - if val > 0 { - framework.Logf("Prometheus netobserv_ingest_flows_processed sample value: %v", val) - return true + g.It("should have Prometheus scraping non-zero NetObserv metrics", func(ctx context.Context) { + promPods, err := oc.AdminKubeClient().CoreV1().Pods("openshift-monitoring").List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=prometheus", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to list Prometheus pods in openshift-monitoring") + o.Expect(promPods.Items).NotTo(o.BeEmpty(), "expected at least one Prometheus pod in openshift-monitoring") + promPodName := promPods.Items[0].Name + + o.Eventually(func() bool { + promOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args( + "-n", "openshift-monitoring", + promPodName, "-c", "prometheus", "--", + "curl", "-s", + "http://localhost:9090/api/v1/query?query=netobserv_ingest_flows_processed", + ).Output() + if err != nil { + framework.Logf("failed to query Prometheus API") + return false + } + + type promResult struct { + Data struct { + Result []struct { + Value []json.RawMessage `json:"value"` + } `json:"result"` + } `json:"data"` + } + var result promResult + if err := json.Unmarshal([]byte(promOutput), &result); err != nil { + framework.Logf("failed to parse Prometheus response") + return false + } + if len(result.Data.Result) == 0 { + framework.Logf("Prometheus netobserv_ingest_flows_processed: no results yet") + return false + } + for _, r := range result.Data.Result { + if len(r.Value) >= 2 { + var valStr string + if err := json.Unmarshal(r.Value[1], &valStr); err != nil { + continue + } + val, err := strconv.ParseFloat(valStr, 64) + if err != nil { + continue + } + if val > 0 { + framework.Logf("Prometheus netobserv_ingest_flows_processed sample value: %v", val) + return true + } } } - } - framework.Logf("Prometheus netobserv_ingest_flows_processed: all sample values are zero") - return false - }, 5*time.Minute, 15*time.Second).Should(o.BeTrue(), - "Prometheus should have netobserv_ingest_flows_processed results") + framework.Logf("Prometheus netobserv_ingest_flows_processed: all sample values are zero") + return false + }, 5*time.Minute, 15*time.Second).Should(o.BeTrue(), + "Prometheus should have non-zero netobserv_ingest_flows_processed results") + }) }) -}) \ No newline at end of file +}) + +func countResources(oc *exutil.CLI, resource, namespace string) (int, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + resource, "-n", namespace, + "-o=jsonpath={.items[*].metadata.name}", + ).Output() + if err != nil { + return 0, err + } + trimmed := strings.TrimSpace(output) + if trimmed == "" { + return 0, nil + } + return len(strings.Fields(trimmed)), nil +} \ No newline at end of file From 72d32c7eefad9abaf0b1dd560ea912ddbecd1fe0 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Fri, 26 Jun 2026 07:45:42 -0400 Subject: [PATCH 04/15] gofmt --- test/extended/networking/network_observability.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index d5e4e30d1cb9..d69d76290d56 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -349,4 +349,4 @@ func countResources(oc *exutil.CLI, resource, namespace string) (int, error) { return 0, nil } return len(strings.Fields(trimmed)), nil -} \ No newline at end of file +} From 70db7d96bf73f70b7ce165d2796918aeb762e3bf Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Fri, 26 Jun 2026 11:51:39 -0400 Subject: [PATCH 05/15] OCPFeatureGate:NetworkObservabilityInstall --- test/extended/networking/network_observability.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index d69d76290d56..ffe5d436edf3 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -29,7 +29,7 @@ type flowCollectorCondition struct { Status string `json:"status"` } -var _ = g.Describe("[sig-network][Feature:NetObserv]", func() { +var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Feature:NetObserv]", func() { oc := exutil.NewCLIWithoutNamespace("netobserv-e2e") g.It("should not be installed on single node clusters", func(ctx context.Context) { From b892f77becb5451c802da3959ca8333b56ab6072 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Thu, 2 Jul 2026 09:07:45 -0400 Subject: [PATCH 06/15] namespace change --- test/extended/networking/network_observability.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index ffe5d436edf3..d580d8500e8b 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -18,8 +18,8 @@ import ( const ( netobservOperatorNamespace = "openshift-netobserv-operator" - netobservNamespace = "openshift-network-observability" - netobservPrivilegedNS = "openshift-network-observability-privileged" + netobservNamespace = "network-observability" + netobservPrivilegedNS = "network-observability-privileged" flowCollectorName = "cluster" flpMetricsPort = "9401" ) From 2e295c3da9b17bfda284af52db94e7212fe6aa67 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Mon, 6 Jul 2026 14:06:22 -0400 Subject: [PATCH 07/15] namespace update --- test/extended/networking/network_observability.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index d580d8500e8b..3a2e5a34d42a 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -17,7 +17,7 @@ import ( ) const ( - netobservOperatorNamespace = "openshift-netobserv-operator" + netobservOperatorNamespace = "netobserv-operator" netobservNamespace = "network-observability" netobservPrivilegedNS = "network-observability-privileged" flowCollectorName = "cluster" From 65a7bae7b2efe3d772dc659382d2aa21d51ba6cd Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Tue, 7 Jul 2026 12:19:19 -0400 Subject: [PATCH 08/15] netobserv NS --- test/extended/networking/network_observability.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index 3a2e5a34d42a..7ed85a6dd6a7 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -18,8 +18,8 @@ import ( const ( netobservOperatorNamespace = "netobserv-operator" - netobservNamespace = "network-observability" - netobservPrivilegedNS = "network-observability-privileged" + netobservNamespace = "netobserv" + netobservPrivilegedNS = "netobserv-privileged" flowCollectorName = "cluster" flpMetricsPort = "9401" ) From dd36c9fb925b148098ec29608ba7e357ad5358a7 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Tue, 14 Jul 2026 20:27:25 -0400 Subject: [PATCH 09/15] flag tests --- .../networking/network_observability.go | 209 ++++++++++++++++++ 1 file changed, 209 insertions(+) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index 7ed85a6dd6a7..885ae35c0a28 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -10,9 +10,13 @@ import ( g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" + configv1 "github.com/openshift/api/config/v1" + applyconfigv1 "github.com/openshift/client-go/config/applyconfigurations/config/v1" exutil "github.com/openshift/origin/test/extended/util" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" ) @@ -22,6 +26,7 @@ const ( netobservPrivilegedNS = "netobserv-privileged" flowCollectorName = "cluster" flpMetricsPort = "9401" + netobservFieldManager = "netobserv-policy-e2e" ) type flowCollectorCondition struct { @@ -350,3 +355,207 @@ func countResources(oc *exutil.CLI, resource, namespace string) (int, error) { } return len(strings.Fields(trimmed)), nil } + +func uninstallNetObserv(ctx context.Context, oc *exutil.CLI, client kubernetes.Interface) { + g.By("removing FlowCollector CR") + _, _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args( + "flowcollector", flowCollectorName, "--ignore-not-found", + ).Output() + + g.By("removing NetObserv operator subscription") + _, _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args( + "subscription", "--all", "-n", netobservOperatorNamespace, "--ignore-not-found", + ).Output() + + g.By("removing NetObserv CSVs") + _, _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args( + "csv", "--all", "-n", netobservOperatorNamespace, "--ignore-not-found", + ).Output() + + g.By("waiting for operator pods to terminate") + o.Eventually(func() bool { + pods, err := client.CoreV1().Pods(netobservOperatorNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return false + } + return len(pods.Items) == 0 + }, 5*time.Minute, 10*time.Second).Should(o.BeTrue(), + "operator pods should terminate after uninstall") + + g.By("waiting for FLP pods to terminate") + o.Eventually(func() bool { + pods, err := client.CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=flowlogs-pipeline", + }) + if err != nil { + return false + } + return len(pods.Items) == 0 + }, 3*time.Minute, 10*time.Second).Should(o.BeTrue(), + "FLP pods should terminate after uninstall") + + g.By("waiting for eBPF agent pods to terminate") + o.Eventually(func() bool { + pods, err := client.CoreV1().Pods(netobservPrivilegedNS).List(ctx, metav1.ListOptions{ + LabelSelector: "app=netobserv-ebpf-agent", + }) + if err != nil { + return false + } + return len(pods.Items) == 0 + }, 3*time.Minute, 10*time.Second).Should(o.BeTrue(), + "eBPF agent pods should terminate after uninstall") +} + +func verifyNetObservNotInstalled(ctx context.Context, client kubernetes.Interface) { + g.By("confirming operator stays absent") + o.Consistently(func() bool { + pods, err := client.CoreV1().Pods(netobservOperatorNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return true + } + for _, pod := range pods.Items { + if strings.Contains(pod.Name, "netobserv-controller-manager") && pod.Status.Phase == "Running" { + framework.Logf("unexpected running controller-manager pod: %s", pod.Name) + return false + } + } + flpPods, err := client.CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=flowlogs-pipeline", + }) + if err != nil { + return true + } + if len(flpPods.Items) > 0 { + framework.Logf("unexpected FLP pods found: %d", len(flpPods.Items)) + return false + } + return true + }, 3*time.Minute, 30*time.Second).Should(o.BeTrue(), + "NetObserv should remain uninstalled with NoAction policy") +} + +func verifyNetObservHealthy(ctx context.Context, client kubernetes.Interface) { + g.By("checking operator namespace exists") + o.Eventually(func() error { + _, err := client.CoreV1().Namespaces().Get(ctx, netobservOperatorNamespace, metav1.GetOptions{}) + return err + }, 5*time.Minute, 15*time.Second).Should(o.Succeed(), + "operator namespace %s should exist", netobservOperatorNamespace) + + g.By("checking operator pod is running") + o.Eventually(func() bool { + pods, err := client.CoreV1().Pods(netobservOperatorNamespace).List(ctx, metav1.ListOptions{}) + if err != nil || len(pods.Items) == 0 { + return false + } + for _, pod := range pods.Items { + if strings.Contains(pod.Name, "netobserv-controller-manager") && pod.Status.Phase == "Running" { + return true + } + } + return false + }, 5*time.Minute, 15*time.Second).Should(o.BeTrue(), + "netobserv-controller-manager should be Running") + + g.By("checking FLP pods are running") + o.Eventually(func() bool { + flpPods, err := client.CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=flowlogs-pipeline", + }) + if err != nil || len(flpPods.Items) == 0 { + return false + } + for _, pod := range flpPods.Items { + if pod.Status.Phase != "Running" { + return false + } + } + return true + }, 5*time.Minute, 15*time.Second).Should(o.BeTrue(), + "FLP pods should be Running in %s", netobservNamespace) +} + +var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Feature:NetObserv][Serial]", g.Ordered, func() { + oc := exutil.NewCLIWithoutNamespace("netobserv-policy-e2e") + + g.BeforeAll(func(ctx context.Context) { + isSingleNode, err := exutil.IsSingleNode(ctx, oc.AdminConfigClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isSingleNode { + g.Skip("NetObserv is not expected on single node clusters") + } + + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isMicroShift { + g.Skip("NetObserv is not supported on MicroShift") + } + + hasAccess, err := hasNetworkConfigWriteAccess(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + if !hasAccess { + g.Skip("The test is not permitted to modify the cluster/network.config.openshift.io resource") + } + }) + + g.AfterAll(func(ctx context.Context) { + g.By("restoring networkObservability to default (unset)") + patch := []byte(`{"spec":{"networkObservability":null}}`) + _, err := oc.AdminConfigClient().ConfigV1().Networks().Patch( + ctx, clusterConfig, types.MergePatchType, patch, + metav1.PatchOptions{FieldManager: netobservFieldManager}, + ) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("waiting for NetObserv to be reinstalled after restoring defaults") + verifyNetObservHealthy(ctx, oc.AdminKubeClient()) + }) + + g.Context("InstallAndEnable policy", func() { + g.It("should reinstall the operator after manual uninstall when policy is InstallAndEnable", func(ctx context.Context) { + g.By("setting networkObservability.installationPolicy to InstallAndEnable") + netConfigApply := applyconfigv1.Network(clusterConfig).WithSpec( + applyconfigv1.NetworkSpec().WithNetworkObservability( + applyconfigv1.NetworkObservabilitySpec().WithInstallationPolicy( + configv1.NetworkObservabilityInstallAndEnable, + ), + ), + ) + _, err := oc.AdminConfigClient().ConfigV1().Networks().Apply(ctx, netConfigApply, + metav1.ApplyOptions{FieldManager: netobservFieldManager, Force: true}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("verifying NetObserv is healthy with InstallAndEnable policy") + verifyNetObservHealthy(ctx, oc.AdminKubeClient()) + + g.By("manually uninstalling NetObserv operator") + uninstallNetObserv(ctx, oc, oc.AdminKubeClient()) + + g.By("verifying NetObserv gets automatically reinstalled") + verifyNetObservHealthy(ctx, oc.AdminKubeClient()) + }) + }) + + g.Context("NoAction policy", func() { + g.It("should not reinstall the operator after manual uninstall when policy is NoAction", func(ctx context.Context) { + g.By("setting networkObservability.installationPolicy to NoAction") + netConfigApply := applyconfigv1.Network(clusterConfig).WithSpec( + applyconfigv1.NetworkSpec().WithNetworkObservability( + applyconfigv1.NetworkObservabilitySpec().WithInstallationPolicy( + configv1.NetworkObservabilityNoAction, + ), + ), + ) + _, err := oc.AdminConfigClient().ConfigV1().Networks().Apply(ctx, netConfigApply, + metav1.ApplyOptions{FieldManager: netobservFieldManager, Force: true}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("manually uninstalling NetObserv operator") + uninstallNetObserv(ctx, oc, oc.AdminKubeClient()) + + g.By("verifying NetObserv does NOT get reinstalled") + verifyNetObservNotInstalled(ctx, oc.AdminKubeClient()) + }) + }) +}) From b36ea1792ce682c4ebdde9ba00f587227d4f4133 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Wed, 15 Jul 2026 14:18:27 -0400 Subject: [PATCH 10/15] uninstall test --- .../networking/network_observability.go | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index 885ae35c0a28..d25ffb52997b 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -15,18 +15,18 @@ import ( exutil "github.com/openshift/origin/test/extended/util" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" ) const ( - netobservOperatorNamespace = "netobserv-operator" - netobservNamespace = "netobserv" - netobservPrivilegedNS = "netobserv-privileged" - flowCollectorName = "cluster" - flpMetricsPort = "9401" - netobservFieldManager = "netobserv-policy-e2e" + netobservOperatorNamespace = "netobserv-operator" + netobservNamespace = "netobserv" + netobservPrivilegedNS = "netobserv-privileged" + netobservClusterExtensionName = "netobserv-operator" + flowCollectorName = "cluster" + flpMetricsPort = "9401" + netobservFieldManager = "netobserv-policy-e2e" ) type flowCollectorCondition struct { @@ -362,14 +362,9 @@ func uninstallNetObserv(ctx context.Context, oc *exutil.CLI, client kubernetes.I "flowcollector", flowCollectorName, "--ignore-not-found", ).Output() - g.By("removing NetObserv operator subscription") + g.By("removing NetObserv ClusterExtension") _, _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args( - "subscription", "--all", "-n", netobservOperatorNamespace, "--ignore-not-found", - ).Output() - - g.By("removing NetObserv CSVs") - _, _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args( - "csv", "--all", "-n", netobservOperatorNamespace, "--ignore-not-found", + "clusterextension", netobservClusterExtensionName, "--ignore-not-found", ).Output() g.By("waiting for operator pods to terminate") @@ -500,15 +495,19 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe }) g.AfterAll(func(ctx context.Context) { - g.By("restoring networkObservability to default (unset)") - patch := []byte(`{"spec":{"networkObservability":null}}`) - _, err := oc.AdminConfigClient().ConfigV1().Networks().Patch( - ctx, clusterConfig, types.MergePatchType, patch, - metav1.PatchOptions{FieldManager: netobservFieldManager}, + g.By("restoring networkObservability to InstallAndEnable") + netConfigApply := applyconfigv1.Network(clusterConfig).WithSpec( + applyconfigv1.NetworkSpec().WithNetworkObservability( + applyconfigv1.NetworkObservabilitySpec().WithInstallationPolicy( + configv1.NetworkObservabilityInstallAndEnable, + ), + ), ) + _, err := oc.AdminConfigClient().ConfigV1().Networks().Apply(ctx, netConfigApply, + metav1.ApplyOptions{FieldManager: netobservFieldManager, Force: true}) o.Expect(err).NotTo(o.HaveOccurred()) - g.By("waiting for NetObserv to be reinstalled after restoring defaults") + g.By("waiting for NetObserv to be reinstalled") verifyNetObservHealthy(ctx, oc.AdminKubeClient()) }) From 52133ae63cf594312826d431c1dbf0388d0229a4 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Thu, 16 Jul 2026 08:09:57 -0400 Subject: [PATCH 11/15] run update --- test/extended/networking/network_observability.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index d25ffb52997b..b99e9dcee5b8 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -20,13 +20,13 @@ import ( ) const ( - netobservOperatorNamespace = "netobserv-operator" - netobservNamespace = "netobserv" - netobservPrivilegedNS = "netobserv-privileged" + netobservOperatorNamespace = "netobserv-operator" + netobservNamespace = "netobserv" + netobservPrivilegedNS = "netobserv-privileged" netobservClusterExtensionName = "netobserv-operator" - flowCollectorName = "cluster" - flpMetricsPort = "9401" - netobservFieldManager = "netobserv-policy-e2e" + flowCollectorName = "cluster" + flpMetricsPort = "9401" + netobservFieldManager = "netobserv-policy-e2e" ) type flowCollectorCondition struct { From d1e723ceb34dfba99d62e06d68aad0297d5738f4 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Thu, 16 Jul 2026 14:10:44 -0400 Subject: [PATCH 12/15] error handling --- .../networking/network_observability.go | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index b99e9dcee5b8..97fb5ee6fead 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -24,6 +24,7 @@ const ( netobservNamespace = "netobserv" netobservPrivilegedNS = "netobserv-privileged" netobservClusterExtensionName = "netobserv-operator" + netobservControllerManager = "netobserv-controller-manager" flowCollectorName = "cluster" flpMetricsPort = "9401" netobservFieldManager = "netobserv-policy-e2e" @@ -55,11 +56,9 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe "Network observability namespace %q should not exist on single node clusters (err: %v)", netobservNamespace, err) g.By("checking that the FlowCollector CRD is not installed") - crdOutput, crdErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("crd", "flowcollectors.flows.netobserv.io").Output() + _, crdErr := oc.AdminKubeClient().Discovery().ServerResourcesForGroupVersion("flows.netobserv.io/v1beta2") o.Expect(crdErr).To(o.HaveOccurred(), - "FlowCollector CRD should not be installed on single node clusters, but found: %s", crdOutput) - o.Expect(strings.Contains(crdOutput, "NotFound") || strings.Contains(crdOutput, "not found")).To(o.BeTrue(), - "expected not-found error for FlowCollector CRD, got: %s", crdOutput) + "FlowCollector API group should not be available on single node clusters") }) g.Context("health checks", func() { @@ -111,7 +110,7 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe found := false for _, pod := range pods.Items { - if strings.Contains(pod.Name, "netobserv-controller-manager") { + if strings.Contains(pod.Name, netobservControllerManager) { o.Expect(string(pod.Status.Phase)).To(o.Equal("Running"), "netobserv-controller-manager pod should be Running, got %s", pod.Status.Phase) found = true @@ -208,7 +207,7 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe g.It("should not have excessive errors in operator logs", func(ctx context.Context) { logOutput, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args( "-n", netobservOperatorNamespace, - "deployment/netobserv-controller-manager", + fmt.Sprintf("deployment/%s", netobservControllerManager), "--tail=50", ).Output() o.Expect(err).NotTo(o.HaveOccurred(), "failed to fetch operator logs") @@ -246,14 +245,15 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe }) g.It("should have FLP producing non-zero flow data", func(ctx context.Context) { - flpPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: "app=flowlogs-pipeline", - }) - o.Expect(err).NotTo(o.HaveOccurred(), "failed to list FLP pods in %s", netobservNamespace) - o.Expect(flpPods.Items).NotTo(o.BeEmpty(), "no FLP pods found in %s", netobservNamespace) - - flpPod := flpPods.Items[0].Name o.Eventually(func() bool { + flpPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=flowlogs-pipeline", + }) + if err != nil || len(flpPods.Items) == 0 { + framework.Logf("FLP pods not available yet in %s", netobservNamespace) + return false + } + flpPod := flpPods.Items[0].Name metricsOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args( "-n", netobservNamespace, flpPod, "--", "curl", "-s", fmt.Sprintf("http://localhost:%s/metrics", flpMetricsPort), @@ -358,14 +358,20 @@ func countResources(oc *exutil.CLI, resource, namespace string) (int, error) { func uninstallNetObserv(ctx context.Context, oc *exutil.CLI, client kubernetes.Interface) { g.By("removing FlowCollector CR") - _, _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args( + fcOutput, fcErr := oc.AsAdmin().WithoutNamespace().Run("delete").Args( "flowcollector", flowCollectorName, "--ignore-not-found", ).Output() + if fcErr != nil { + framework.Logf("warning: failed to delete FlowCollector %s: %v (output: %s)", flowCollectorName, fcErr, fcOutput) + } g.By("removing NetObserv ClusterExtension") - _, _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args( - "clusterextension", netobservClusterExtensionName, "--ignore-not-found", + ceOutput, ceErr := oc.AsAdmin().WithoutNamespace().Run("delete").Args( + "clusterextension", netobservClusterExtensionName, "--ignore-not-found", "--wait=true", "--timeout=2m", ).Output() + if ceErr != nil { + framework.Logf("warning: failed to delete ClusterExtension %s: %v (output: %s)", netobservClusterExtensionName, ceErr, ceOutput) + } g.By("waiting for operator pods to terminate") o.Eventually(func() bool { @@ -373,7 +379,13 @@ func uninstallNetObserv(ctx context.Context, oc *exutil.CLI, client kubernetes.I if err != nil { return false } - return len(pods.Items) == 0 + for _, pod := range pods.Items { + if strings.Contains(pod.Name, netobservControllerManager) { + framework.Logf("operator pod %s still present (phase=%s)", pod.Name, pod.Status.Phase) + return false + } + } + return true }, 5*time.Minute, 10*time.Second).Should(o.BeTrue(), "operator pods should terminate after uninstall") @@ -407,10 +419,11 @@ func verifyNetObservNotInstalled(ctx context.Context, client kubernetes.Interfac o.Consistently(func() bool { pods, err := client.CoreV1().Pods(netobservOperatorNamespace).List(ctx, metav1.ListOptions{}) if err != nil { - return true + framework.Logf("failed to list pods in %s: %v", netobservOperatorNamespace, err) + return false } for _, pod := range pods.Items { - if strings.Contains(pod.Name, "netobserv-controller-manager") && pod.Status.Phase == "Running" { + if strings.Contains(pod.Name, netobservControllerManager) && pod.Status.Phase == "Running" { framework.Logf("unexpected running controller-manager pod: %s", pod.Name) return false } @@ -419,7 +432,8 @@ func verifyNetObservNotInstalled(ctx context.Context, client kubernetes.Interfac LabelSelector: "app=flowlogs-pipeline", }) if err != nil { - return true + framework.Logf("failed to list FLP pods in %s: %v", netobservNamespace, err) + return false } if len(flpPods.Items) > 0 { framework.Logf("unexpected FLP pods found: %d", len(flpPods.Items)) @@ -445,7 +459,7 @@ func verifyNetObservHealthy(ctx context.Context, client kubernetes.Interface) { return false } for _, pod := range pods.Items { - if strings.Contains(pod.Name, "netobserv-controller-manager") && pod.Status.Phase == "Running" { + if strings.Contains(pod.Name, netobservControllerManager) && pod.Status.Phase == "Running" { return true } } From fd682aae30472628e8dd0d451aa947f559e5e8f3 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Tue, 21 Jul 2026 23:00:53 -0400 Subject: [PATCH 13/15] Only InstallAndEnable test --- .../networking/network_observability.go | 55 +++---------------- 1 file changed, 7 insertions(+), 48 deletions(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index 97fb5ee6fead..8e04c6f5613b 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -508,8 +508,8 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe } }) - g.AfterAll(func(ctx context.Context) { - g.By("restoring networkObservability to InstallAndEnable") + g.It("should have NetObserv installed and healthy with InstallAndEnable policy", func(ctx context.Context) { + g.By("setting networkObservability.installationPolicy to InstallAndEnable") netConfigApply := applyconfigv1.Network(clusterConfig).WithSpec( applyconfigv1.NetworkSpec().WithNetworkObservability( applyconfigv1.NetworkObservabilitySpec().WithInstallationPolicy( @@ -521,54 +521,13 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe metav1.ApplyOptions{FieldManager: netobservFieldManager, Force: true}) o.Expect(err).NotTo(o.HaveOccurred()) - g.By("waiting for NetObserv to be reinstalled") + g.By("verifying NetObserv is healthy with InstallAndEnable policy") verifyNetObservHealthy(ctx, oc.AdminKubeClient()) - }) - - g.Context("InstallAndEnable policy", func() { - g.It("should reinstall the operator after manual uninstall when policy is InstallAndEnable", func(ctx context.Context) { - g.By("setting networkObservability.installationPolicy to InstallAndEnable") - netConfigApply := applyconfigv1.Network(clusterConfig).WithSpec( - applyconfigv1.NetworkSpec().WithNetworkObservability( - applyconfigv1.NetworkObservabilitySpec().WithInstallationPolicy( - configv1.NetworkObservabilityInstallAndEnable, - ), - ), - ) - _, err := oc.AdminConfigClient().ConfigV1().Networks().Apply(ctx, netConfigApply, - metav1.ApplyOptions{FieldManager: netobservFieldManager, Force: true}) - o.Expect(err).NotTo(o.HaveOccurred()) - g.By("verifying NetObserv is healthy with InstallAndEnable policy") - verifyNetObservHealthy(ctx, oc.AdminKubeClient()) + g.By("manually uninstalling NetObserv operator") + uninstallNetObserv(ctx, oc, oc.AdminKubeClient()) - g.By("manually uninstalling NetObserv operator") - uninstallNetObserv(ctx, oc, oc.AdminKubeClient()) - - g.By("verifying NetObserv gets automatically reinstalled") - verifyNetObservHealthy(ctx, oc.AdminKubeClient()) - }) - }) - - g.Context("NoAction policy", func() { - g.It("should not reinstall the operator after manual uninstall when policy is NoAction", func(ctx context.Context) { - g.By("setting networkObservability.installationPolicy to NoAction") - netConfigApply := applyconfigv1.Network(clusterConfig).WithSpec( - applyconfigv1.NetworkSpec().WithNetworkObservability( - applyconfigv1.NetworkObservabilitySpec().WithInstallationPolicy( - configv1.NetworkObservabilityNoAction, - ), - ), - ) - _, err := oc.AdminConfigClient().ConfigV1().Networks().Apply(ctx, netConfigApply, - metav1.ApplyOptions{FieldManager: netobservFieldManager, Force: true}) - o.Expect(err).NotTo(o.HaveOccurred()) - - g.By("manually uninstalling NetObserv operator") - uninstallNetObserv(ctx, oc, oc.AdminKubeClient()) - - g.By("verifying NetObserv does NOT get reinstalled") - verifyNetObservNotInstalled(ctx, oc.AdminKubeClient()) - }) + g.By("verifying NetObserv does NOT get reinstalled") + verifyNetObservNotInstalled(ctx, oc.AdminKubeClient()) }) }) From 6fcbf23f7af613035bce0c644ce327a9c68c50d9 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Mon, 27 Jul 2026 12:05:43 -0400 Subject: [PATCH 14/15] Removed Prometheus tests --- .../networking/network_observability.go | 72 +++++++------------ 1 file changed, 24 insertions(+), 48 deletions(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index 8e04c6f5613b..641ad9b4dd6b 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -26,7 +26,6 @@ const ( netobservClusterExtensionName = "netobserv-operator" netobservControllerManager = "netobserv-controller-manager" flowCollectorName = "cluster" - flpMetricsPort = "9401" netobservFieldManager = "netobserv-policy-e2e" ) @@ -72,7 +71,7 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) o.Expect(err).NotTo(o.HaveOccurred(), "failed to determine if cluster is MicroShift") if isMicroShift { - g.Skip("NetObserv is not supported on MicroShift") + g.Skip("FlowCollector API group should not be available on MicroShift clusters") } }) @@ -182,13 +181,12 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe } }) - g.It("should have console plugin healthy if deployed [apigroup:console.openshift.io]", func(ctx context.Context) { + g.It("should have console plugin healthy [apigroup:console.openshift.io]", func(ctx context.Context) { pluginPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ LabelSelector: "app=netobserv-plugin", }) - if err != nil || len(pluginPods.Items) == 0 { - g.Skip("console plugin not deployed") - } + o.Expect(err).NotTo(o.HaveOccurred(), "failed to list console plugin pods in %s", netobservNamespace) + o.Expect(pluginPods.Items).NotTo(o.BeEmpty(), "expected console plugin pods in %s", netobservNamespace) for _, pod := range pluginPods.Items { o.Expect(string(pod.Status.Phase)).To(o.Equal("Running"), @@ -204,6 +202,19 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe "consoleplugin resource name mismatch") }) + g.It("should have static plugin pods running", func(ctx context.Context) { + staticPluginPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservOperatorNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=netobserv-plugin-static", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to list static plugin pods in %s", netobservOperatorNamespace) + o.Expect(staticPluginPods.Items).NotTo(o.BeEmpty(), "expected static plugin pods in %s", netobservOperatorNamespace) + + for _, pod := range staticPluginPods.Items { + o.Expect(string(pod.Status.Phase)).To(o.Equal("Running"), + "static plugin pod %s should be Running, got %s", pod.Name, pod.Status.Phase) + } + }) + g.It("should not have excessive errors in operator logs", func(ctx context.Context) { logOutput, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args( "-n", netobservOperatorNamespace, @@ -213,16 +224,19 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe o.Expect(err).NotTo(o.HaveOccurred(), "failed to fetch operator logs") errorCount := 0 + var errorLines []string for _, line := range strings.Split(logOutput, "\n") { if strings.Contains(line, "\"level\":\"error\"") || strings.Contains(line, "level=error") { errorCount++ + errorLines = append(errorLines, line) } } o.Expect(errorCount).To(o.BeNumerically("<=", 5), - "found %d error-level log entries in the last 50 operator log lines (threshold: 5)", errorCount) + "found %d error-level log entries in the last 50 operator log lines (threshold: 5):\n%s", + errorCount, strings.Join(errorLines, "\n")) }) - g.It("should have monitoring resources deployed", func(ctx context.Context) { + g.It("should have Prometheus scraping non-zero NetObserv metrics", func(ctx context.Context) { g.By("checking ServiceMonitors exist") smCount, err := countResources(oc, "servicemonitor", netobservNamespace) o.Expect(err).NotTo(o.HaveOccurred(), "failed to list ServiceMonitors in %s", netobservNamespace) @@ -242,46 +256,8 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe o.Expect(rulesCount).To(o.BeNumerically(">", 0), "expected at least one PrometheusRule in %s", netobservNamespace) framework.Logf("Found %d PrometheusRule(s) in %s", rulesCount, netobservNamespace) - }) - g.It("should have FLP producing non-zero flow data", func(ctx context.Context) { - o.Eventually(func() bool { - flpPods, err := oc.AdminKubeClient().CoreV1().Pods(netobservNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: "app=flowlogs-pipeline", - }) - if err != nil || len(flpPods.Items) == 0 { - framework.Logf("FLP pods not available yet in %s", netobservNamespace) - return false - } - flpPod := flpPods.Items[0].Name - metricsOutput, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args( - "-n", netobservNamespace, flpPod, "--", - "curl", "-s", fmt.Sprintf("http://localhost:%s/metrics", flpMetricsPort), - ).Output() - if err != nil { - framework.Logf("failed to query FLP metrics endpoint") - return false - } - - for _, line := range strings.Split(metricsOutput, "\n") { - if strings.HasPrefix(line, "netobserv_ingest_flows_processed") && !strings.HasPrefix(line, "#") { - parts := strings.Fields(line) - if len(parts) >= 2 { - val, err := strconv.ParseFloat(parts[len(parts)-1], 64) - if err == nil && val > 0 { - framework.Logf("FLP processed flows metric: %v", val) - return true - } - } - } - } - framework.Logf("netobserv_ingest_flows_processed metric is zero or not found") - return false - }, 3*time.Minute, 10*time.Second).Should(o.BeTrue(), - "FLP should show non-zero netobserv_ingest_flows_processed metric") - }) - - g.It("should have Prometheus scraping non-zero NetObserv metrics", func(ctx context.Context) { + g.By("querying Prometheus for non-zero flow metrics") promPods, err := oc.AdminKubeClient().CoreV1().Pods("openshift-monitoring").List(ctx, metav1.ListOptions{ LabelSelector: "app.kubernetes.io/name=prometheus", }) @@ -498,7 +474,7 @@ var _ = g.Describe("[sig-network][OCPFeatureGate:NetworkObservabilityInstall][Fe isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) o.Expect(err).NotTo(o.HaveOccurred()) if isMicroShift { - g.Skip("NetObserv is not supported on MicroShift") + g.Skip("FlowCollector API group should not be available on MicroShift clusters") } hasAccess, err := hasNetworkConfigWriteAccess(oc) From 040a2788a92969d094a6c79842fdb38cb82bd806 Mon Sep 17 00:00:00 2001 From: Kapil Jain Date: Fri, 14 Aug 2026 11:20:27 -0400 Subject: [PATCH 15/15] openshift namespace --- test/extended/networking/network_observability.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/extended/networking/network_observability.go b/test/extended/networking/network_observability.go index 641ad9b4dd6b..42f785789f45 100644 --- a/test/extended/networking/network_observability.go +++ b/test/extended/networking/network_observability.go @@ -20,9 +20,9 @@ import ( ) const ( - netobservOperatorNamespace = "netobserv-operator" - netobservNamespace = "netobserv" - netobservPrivilegedNS = "netobserv-privileged" + netobservOperatorNamespace = "openshift-netobserv-operator" + netobservNamespace = "openshift-network-observability" + netobservPrivilegedNS = "openshift-network-observability-privileged" netobservClusterExtensionName = "netobserv-operator" netobservControllerManager = "netobserv-controller-manager" flowCollectorName = "cluster"