From 40e88e8012c71c918bc416406babcf64442de9a9 Mon Sep 17 00:00:00 2001 From: David Fridrich Date: Wed, 14 Oct 2026 11:43:34 +0200 Subject: [PATCH] test(e2e): add FUNC_BIN/WithImage helpers and a middleware-rebuild spec - test/utils/func.go: FUNC_BIN env override to point at a specific func binary (e.g. extracted from the operator image / provided by CI), and a WithImage deploy option to deploy a pre-built image (--build=false --push=false), used to seed a function's initial deployed state. - test/e2e/func_middleware_fixture_test.go: a middleware-rebuild spec that deploys a pre-built stale-middleware image (via STALE_FUNCTION_IMAGE) and asserts the operator detects it and rebuilds to current. Image ref is not hardcoded; the spec skips when STALE_FUNCTION_IMAGE is unset. Currently also skipped pending knative/func#4081 (registry auth secret type mismatch on operator rebuild). --- Makefile | 11 +- .../notificationserver/server.go | 8 +- test/e2e/func_middleware_rebuild_test.go | 122 ++++++++++++++++++ test/utils/func.go | 78 ++++++++--- 4 files changed, 197 insertions(+), 22 deletions(-) create mode 100644 test/e2e/func_middleware_rebuild_test.go diff --git a/Makefile b/Makefile index 24c45f76..348844e0 100644 --- a/Makefile +++ b/Makefile @@ -127,9 +127,15 @@ test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v e2e) -coverprofile cover.out cd api && go test ./... -coverprofile cover.out +# STALE_FUNCTION_IMAGE is a pre-built, publicly pullable function image whose +# "middleware-version" label is older than the operator's current version. The +# middleware-rebuild e2e spec seeds its deployment from this image and expects +# the operator to rebuild from source. The spec skips itself when unset. +STALE_FUNCTION_IMAGE ?= + .PHONY: test-e2e ## Run e2e tests. test-e2e: ginkgo - $(GINKGO) -v --timeout=1h --label-filter="!bundle" --fail-fast --flake-attempts=2 -p ./test/e2e/ + STALE_FUNCTION_IMAGE=$(STALE_FUNCTION_IMAGE) $(GINKGO) -v --timeout=1h --label-filter="!bundle" --fail-fast --flake-attempts=2 -p ./test/e2e/ .PHONY: test-e2e-bundle ## Run bundle e2e tests. test-e2e-bundle: operator-sdk docker-build docker-push bundle bundle-build bundle-push install-olm-in-cluster ginkgo @@ -281,6 +287,9 @@ deploy: manifests kustomize ## Deploy func-operator controller to the K8s cluste $(MAKE) patch-registry-cert $(KUBECTL) wait deployment --all --timeout=120s --for=condition=Available -n func-operator-system +.PHONY: deploy-full +deploy-full: docker-build docker-push deploy ## Build and push the manager image, then deploy it. Run as a single invocation so all steps share one IMG tag (VERSION embeds a build timestamp). + .PHONY: deploy-source-objectbucket deploy-source-objectbucket: manifests kustomize ## Deploy objectbucket-notifications-adapter to the K8s cluster. cd config/sources/objectbucket/manager && $(KUSTOMIZE) edit set image source-objectbucket-adapter=${IMG_SOURCE_OBJECTBUCKET} diff --git a/internal/objectbucketsource/notificationserver/server.go b/internal/objectbucketsource/notificationserver/server.go index 5880ba7f..6c2b9931 100644 --- a/internal/objectbucketsource/notificationserver/server.go +++ b/internal/objectbucketsource/notificationserver/server.go @@ -82,7 +82,9 @@ func (s *NotificationServer) superviseRun(ctx context.Context, changes <-chan st for { select { case <-ctx.Done(): - <-errCh + if err := <-errCh; err != nil { + log.Error(err, "notification runner stopped with error during shutdown") + } return true case err := <-errCh: // The runner exited on its own (fatal setup error). Restart it, @@ -113,7 +115,9 @@ func (s *NotificationServer) superviseRun(ctx context.Context, changes <-chan st "old-group-id", snap.settings.KafkaNotificationsGroupID, "new-group-id", newSnap.settings.KafkaNotificationsGroupID, "kafka-credentials-changed", kafkaCredsChanged) cancel() - <-errCh + if err := <-errCh; err != nil { + log.Error(err, "notification runner stopped with error during restart") + } return false } } diff --git a/test/e2e/func_middleware_rebuild_test.go b/test/e2e/func_middleware_rebuild_test.go new file mode 100644 index 00000000..1c4031d9 --- /dev/null +++ b/test/e2e/func_middleware_rebuild_test.go @@ -0,0 +1,122 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os" + "time" + + functionsdevv1alpha1 "github.com/functions-dev/func-operator/api/v1alpha1" + "github.com/functions-dev/func-operator/test/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// STALE_FUNCTION_IMAGE must point at a pre-built, publicly pullable function image +// whose "middleware-version" label is older than the operator's current version. +// The operator reads that label off the deployed image, finds it stale, and +// rebuilds the function from source. The spec is skipped when it is unset. +// +// Why a pre-built image (and not a build step): the middleware version is baked +// by the func image, which defaults to a floating "latest" tag, so there's +// no way to produce a *stale* build on the fly. The image is built once as +// `FROM ` + `LABEL middleware-version=` and stored in +// a durable public repo. The image ref is intentionally not hardcoded here so the +// same test works across upstream/downstream with each CI supplying its own image. +func staleFunctionImage() string { + return os.Getenv("STALE_FUNCTION_IMAGE") +} + +var _ = Describe("Middleware Rebuild", func() { + + Context("with a function deployed from a stale-middleware image", func() { + var repoURL string + var repoDir string + var functionName, functionNamespace string + + BeforeEach(func() { + // Opt-in: requires a pre-built stale-middleware image. + if staleFunctionImage() == "" { + Skip("set STALE_FUNCTION_IMAGE to a function image whose middleware-version label is stale") + } + + username, password, _, cleanup, err := repoProvider.CreateRandomUser() + Expect(err).NotTo(HaveOccurred()) + utils.DeferCleanupOnSuccess(cleanup) + + _, repoURL, cleanup, err = repoProvider.CreateRandomRepo(username, false) + Expect(err).NotTo(HaveOccurred()) + utils.DeferCleanupOnSuccess(cleanup) + + repoDir, err = utils.InitializeRepoWithFunction(repoURL, username, password, "go") + Expect(err).NotTo(HaveOccurred()) + utils.DeferCleanupOnSuccess(os.RemoveAll, repoDir) + + functionNamespace, err = utils.GetTestNamespace() + Expect(err).NotTo(HaveOccurred()) + utils.DeferCleanupOnSuccess(cleanupNamespaces, functionNamespace) + + // Seed the deployment with the pre-built stale image (func skips build + // because the ref carries a digest), so the operator sees outdated + // middleware and must rebuild from the repo. + out, err := utils.RunFuncDeploy(repoDir, + utils.WithNamespace(functionNamespace), + utils.WithImage(staleFunctionImage())) + Expect(err).NotTo(HaveOccurred()) + _, _ = fmt.Fprint(GinkgoWriter, out) + + utils.DeferCleanupOnSuccess(func() { + _, _ = utils.RunFunc("delete", "--path", repoDir, "--namespace", functionNamespace) + }) + + err = utils.CommitAndPush(repoDir, "Update func.yaml after deploy", "func.yaml") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + logFailedTestDetails(functionName, functionNamespace) + }) + + It("should rebuild the function and mark middleware up to date", func() { + function := &functionsdevv1alpha1.Function{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "my-function-", + Namespace: functionNamespace, + }, + Spec: functionsdevv1alpha1.FunctionSpec{ + Repository: functionsdevv1alpha1.FunctionSpecRepository{ + URL: repoURL, + }, + }, + } + + err := k8sClient.Create(ctx, function) + Expect(err).NotTo(HaveOccurred()) + utils.DeferCleanupOnSuccess(func() { + _, _ = utils.RunCmd("kubectl", "delete", "function", function.Name, "--namespace", function.Namespace) + }) + functionName = function.Name + + // The operator detects the stale middleware label, rebuilds from the + // repo, and the function becomes ready and up to date. + Eventually(functionBecomesReady(functionName, functionNamespace)).Should(Succeed()) + Eventually(functionMiddlewareUpToDate(functionName, functionNamespace), 2*time.Minute).Should(Succeed()) + }) + }) +}) diff --git a/test/utils/func.go b/test/utils/func.go index b18dd8a7..ed3fb173 100644 --- a/test/utils/func.go +++ b/test/utils/func.go @@ -29,10 +29,21 @@ import ( ginkgo "github.com/onsi/ginkgo/v2" ) +// funcBin returns the func binary to invoke. It honors the FUNC_BIN environment +// variable (an absolute path or a name on PATH), defaulting to "func" on PATH. +// This lets manual runs point at a local build and CI point at a binary extracted +// from the operator image. +func funcBin() string { + if b := os.Getenv("FUNC_BIN"); b != "" { + return b + } + return "func" +} + // RunFunc executes the func CLI with the current/latest version func RunFunc(command string, args ...string) (string, error) { allArgs := append([]string{command}, args...) - cmd := exec.Command("func", allArgs...) + cmd := exec.Command(funcBin(), allArgs...) return Run(cmd) } @@ -49,6 +60,39 @@ func RunFuncWithVersion(version string, command string, args ...string) (string, return Run(cmd) } +// funcDeployArgs assembles the `func deploy` flags from the resolved options. +func funcDeployArgs(functionDir string, opts *FuncDeployOptions) []string { + args := []string{"--path", functionDir} + + // --image and --registry are mutually exclusive in func. With a pre-built + // image (digest) func skips build/push, so the registry flags don't apply. + if opts.Image != "" { + // Deploy a pre-built image as-is: no build, no push. + args = append(args, "--image", opts.Image, "--build=false", "--push=false") + } else { + args = append(args, + "--registry", opts.Registry, + fmt.Sprintf("--registry-insecure=%t", opts.RegistryInsecure), + ) + } + + if opts.Namespace != "" { + args = append(args, "--namespace", opts.Namespace) + } + + // A pre-built image with a digest disables build/push, so the builder is + // irrelevant in that case. + if opts.Builder != "" && opts.Image == "" { + args = append(args, "--builder", opts.Builder) + } + + if opts.Deployer != "" && opts.Deployer != "knative" { + args = append(args, "--deployer", opts.Deployer) + } + + return args +} + // RunFuncDeploy runs func deploy with retry logic for transient network errors func RunFuncDeploy(functionDir string, optFns ...FuncDeployOption) (string, error) { opts := &FuncDeployOptions{ @@ -63,23 +107,7 @@ func RunFuncDeploy(functionDir string, optFns ...FuncDeployOption) (string, erro optFn(opts) } - args := []string{ - "--path", functionDir, - "--registry", opts.Registry, - fmt.Sprintf("--registry-insecure=%t", opts.RegistryInsecure), - } - - if opts.Namespace != "" { - args = append(args, "--namespace", opts.Namespace) - } - - if opts.Builder != "" { - args = append(args, "--builder", opts.Builder) - } - - if opts.Deployer != "" && opts.Deployer != "knative" { - args = append(args, "--deployer", opts.Deployer) - } + args := funcDeployArgs(functionDir, opts) // When using the pack builder, create a per-deploy PACK_HOME to prevent // parallel builds from corrupting the shared ~/.pack/volume-keys.toml. @@ -118,7 +146,7 @@ func RunFuncDeploy(functionDir string, optFns ...FuncDeployOption) (string, erro return "", err } } else { - funcBinary = "func" + funcBinary = funcBin() } cmd := exec.Command(funcBinary, append([]string{"deploy"}, args...)...) @@ -143,6 +171,10 @@ type FuncDeployOptions struct { Deployer string CliVersion string EnvVars map[string]string + // Image, when set to a ref including a digest, makes func deploy the given + // pre-built image and skip the build/push phases. Used to seed a function + // with a specific (e.g. stale-middleware) image. + Image string } type FuncDeployOption func(*FuncDeployOptions) @@ -165,6 +197,14 @@ func WithDeployer(deployer string) FuncDeployOption { } } +// WithImage deploys a pre-built image (ref should include a digest so func skips +// build/push), rather than building from source. +func WithImage(image string) FuncDeployOption { + return func(o *FuncDeployOptions) { + o.Image = image + } +} + func WithDeployCliVersion(version string) FuncDeployOption { return func(opts *FuncDeployOptions) { opts.CliVersion = version