From a91c071ff4517e85450e891d456180d92a68587f Mon Sep 17 00:00:00 2001 From: henderkes Date: Sun, 11 Oct 2026 18:26:49 +0200 Subject: [PATCH 1/2] fix 503's when php runtime shuts down mid-incoming-request, hand them off to the new config runtime --- caddy/app.go | 7 + caddy/module.go | 242 +++- caddy/reload_test.go | 2526 +++++++++++++++++++++++++++++++++++++++++ cgi.go | 2 +- frankenphp.go | 5 +- reload_queue_test.go | 118 ++ scaling.go | 2 +- server.go | 11 +- threadregular.go | 10 + threadregular_test.go | 2 +- worker.go | 27 +- 11 files changed, 2942 insertions(+), 10 deletions(-) create mode 100644 caddy/reload_test.go create mode 100644 reload_queue_test.go diff --git a/caddy/app.go b/caddy/app.go index 8dd40490d6..86e9e71c4e 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -23,6 +23,7 @@ import ( var ( options []frankenphp.Option optionsMU sync.RWMutex + activeApp atomic.Pointer[FrankenPHPApp] ) // EXPERIMENTAL: RegisterWorkers provides a way for extensions to register frankenphp.Workers @@ -68,6 +69,7 @@ type FrankenPHPApp struct { modules []*FrankenPHPModule httpApp *caddyhttp.App hasStarted atomic.Bool + next atomic.Pointer[FrankenPHPApp] started chan any } @@ -182,8 +184,12 @@ func (f *FrankenPHPApp) Start() error { // if FrankenPHP is currently running, shut it down first // this will happen in admin API reloads and caddy tests + if old := activeApp.Swap(f); old != nil { + old.next.Store(f) + } frankenphp.Shutdown() if err := frankenphp.Init(f.opts...); err != nil { + activeApp.CompareAndSwap(f, nil) return err } @@ -193,6 +199,7 @@ func (f *FrankenPHPApp) Start() error { } func (f *FrankenPHPApp) Stop() error { + activeApp.CompareAndSwap(f, nil) if f.logger.Enabled(f.ctx, slog.LevelInfo) { f.logger.LogAttrs(f.ctx, slog.LevelInfo, "FrankenPHP stopped 🐘") } diff --git a/caddy/module.go b/caddy/module.go index 20dcec9ee2..34fc402e51 100644 --- a/caddy/module.go +++ b/caddy/module.go @@ -5,8 +5,10 @@ import ( "errors" "fmt" "log/slog" + "maps" "net/http" "path/filepath" + "reflect" "runtime" "slices" "strconv" @@ -18,7 +20,11 @@ import ( "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyauth" "github.com/caddyserver/caddy/v2/modules/caddyhttp/fileserver" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/headers" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/intercept" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy" "github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite" "github.com/dunglas/frankenphp" "github.com/dunglas/frankenphp/internal/fastabs" @@ -36,6 +42,7 @@ import ( type FrankenPHPModule struct { mercureContext hotReloadContext + GeneratedRewrite bool `json:"generated_rewrite,omitempty"` // Marks the php_server try_files pipeline. // Root sets the root folder to the site. Default: `root` directive, or the path of the public directory of the embed app it exists. Root string `json:"root,omitempty"` @@ -187,10 +194,13 @@ func needReplacement(s string) bool { // ServeHTTP implements caddyhttp.MiddlewareHandler. func (f *FrankenPHPModule) ServeHTTP(w http.ResponseWriter, r *http.Request, _ caddyhttp.Handler) error { if !f.app.hasStarted.Load() { - // stall any incoming request if FrankenPHP has not started yet, blocking for up to 10 seconds select { case <-f.app.started: - case <-time.After(10 * time.Second): + case <-f.app.ctx.Done(): + case <-r.Context().Done(): + return r.Context().Err() + } + if !f.app.hasStarted.Load() || f.app.ctx.Err() != nil { return caddyhttp.Error(http.StatusServiceUnavailable, frankenphp.ErrNotRunning) } } @@ -227,6 +237,9 @@ func (f *FrankenPHPModule) ServeHTTP(w http.ResponseWriter, r *http.Request, _ c err := f.server.ServeHTTP(w, r, opts...) + if err != nil && errors.Is(err, frankenphp.ErrNotRunning) { + return f.requestReload(w, r, f.app.next.Load()) + } if _, rejected := errors.AsType[frankenphp.ErrRejected](err); err != nil && !rejected { return caddyhttp.Error(http.StatusInternalServerError, err) } @@ -234,6 +247,230 @@ func (f *FrankenPHPModule) ServeHTTP(w http.ResponseWriter, r *http.Request, _ c return nil } +func (f *FrankenPHPModule) requestReload(w http.ResponseWriter, r *http.Request, app *FrankenPHPApp) error { + unavailable := caddyhttp.Error(http.StatusServiceUnavailable, frankenphp.ErrNotRunning) + for ; app != nil && app != f.app; app = app.next.Load() { + select { + case <-app.started: + case <-app.ctx.Done(): + case <-r.Context().Done(): + return r.Context().Err() + } + if app.next.Load() == nil { + key, _ := f.reloadKey() + for _, target := range app.modules { + if other, srv := target.reloadKey(); key != "" && other == key { + for wrapper, ok := w.(interface { + Unwrap() http.ResponseWriter + }); ok; wrapper, ok = w.(interface { + Unwrap() http.ResponseWriter + }) { + w = wrapper.Unwrap() + } + r, original, rec := + r.Clone(r.Context()), + r.Context().Value(caddyhttp.OriginalRequestCtxKey).(http.Request), + caddyhttp.NewResponseRecorder(w, nil, nil) + r.Method, r.RequestURI, r.RemoteAddr, r.URL = + original.Method, original.RequestURI, + original.RemoteAddr, new(*original.URL) + replay, r := + srv.Routes.Compile(caddyhttp.HandlerFunc( + func(http.ResponseWriter, *http.Request) error { + return unavailable + }, + )), + caddyhttp.PrepareRequest(r, caddy.NewReplacer(), rec, srv) + if (!app.hasStarted.Load() || + app.ctx.Err() != nil || + replay.ServeHTTP(rec, r) != nil) && rec.Status() < 200 { + w.WriteHeader(http.StatusServiceUnavailable) + } + return nil + } + } + } + } + return unavailable +} + +func (f *FrankenPHPModule) reloadKey() (string, *caddyhttp.Server) { + if httpApp := f.app.httpApp; httpApp != nil { + for _, srv := range httpApp.Servers { + lists, callers, v := + map[string]caddyhttp.RouteList{"routes": srv.Routes}, + []string{}, + reflect.ValueOf(srv).Elem().FieldByName("trustedProxies") + if srv.Errors != nil { + lists["errors"+reloadRouteKeys( + srv.Routes, "", "", make(map[*FrankenPHPModule][2]string), false, f.app, + )] = srv.Errors.Routes + } + for name, route := range srv.NamedRoutes { + lists["named:"+name+reloadRouteKeys( + srv.Routes, "", "", make(map[*FrankenPHPModule][2]string), false, f.app, + )], callers = caddyhttp.RouteList{*route}, append(callers, reloadRouteKeys( + caddyhttp.RouteList{*route}, "", "", + make(map[*FrankenPHPModule][2]string), false, f.app, + )) + } + for name, routes := range lists { + keys := make(map[*FrankenPHPModule][2]string) + reloadRouteKeys(routes, "", "", keys, true, f.app) + if pair, found := keys[f]; found { + if !v.IsValid() || slices.ContainsFunc( + f.app.modules, + func(module *FrankenPHPModule) bool { + return module != f && keys[module] == pair + }, + ) { + return "", nil // Empty keys also fail closed on Caddy field renames. + } + return string(caddyconfig.JSON([]any{ + slices.Sorted(slices.Values(srv.Listen)), + name, + pair, + slices.Sorted(slices.Values(callers)), + srv.ExpectedUnderscoreHeaders, + srv.ExpectedDotHeaders, + srv.StrictSNIHost, + srv.TrustedProxiesRaw, + fmt.Sprintf("%#v", v.Elem()), + srv.ClientIPHeaders, + srv.TrustedProxiesStrict, + srv.TrustedProxiesUnix, + }, nil)), srv + } + } + } + } + return "", nil +} + +func reloadRouteKeys( + routes caddyhttp.RouteList, + scope, before string, + keys map[*FrankenPHPModule][2]string, + allowConsume bool, + app *FrankenPHPApp, +) (prefix string) { + cut := ^uint(0) // No pending consumable rewrite. + for i, route := range routes { + b, _ := json.Marshal(reloadMatchers(route.MatcherSets)) + var owner *FrankenPHPModule + if i+1 < len(routes) && len(routes[i+1].Handlers) == 1 { + owner, _ = routes[i+1].Handlers[0].(*FrankenPHPModule) + } + for _, h := range route.Handlers { + config, err := json.Marshal(h) + signature := string(b) + string(config) + rw, isRewrite := h.(*rewrite.Rewrite) + if owner != nil && (isRewrite && owner.GeneratedRewrite && + rw.URI == "{http.matchers.file.relative}{http.matchers.file.remainder}" || + owner.ServerIndex != 0 && slices.ContainsFunc(owner.Workers, func(w workerConfig) bool { + return len(w.MatchPath) != 0 + }) && caddy.GetModuleName(h) == "file_server") { + signature = strings.ReplaceAll( + signature, + `"root":`+string(caddyconfig.JSON(owner.Root, nil)), + `"root":"{php.root}"`, + ) + } + if isRewrite { + if allowConsume && len(route.Handlers) == 1 && owner != nil && + routes[i+1].Group == "" && strings.HasPrefix(rw.URI, "/") && + !strings.ContainsAny(rw.URI, "{}?#%\\*[]\x00\r\n\t") && + caddyhttp.CleanPath(rw.URI, !strings.Contains(rw.URI, "//")) == rw.URI && + string(config) == `{"uri":`+string(caddyconfig.JSON(rw.URI, nil))+`}` && + (len(routes[i+1].MatcherSets) == 0 || + string(caddyconfig.JSON( + reloadMatchers(routes[i+1].MatcherSets), nil, + )) == `[{"path":[`+string(caddyconfig.JSON( + strings.ToLower(rw.URI), nil, + ))+`]}]`) { + cut = uint(len(prefix)) + } + prefix += signature + } else if module, ok := h.(*FrankenPHPModule); ok { + keys[module] = [2]string{scope + prefix + string(b), before + prefix + string(b)} + prefix, cut = prefix[:min(cut, uint(len(prefix)))], ^uint(0) + } else if fs, ok := h.(*fileserver.FileServer); ok && allowConsume && + len(route.MatcherSets) == 0 && route.Group == "" && !fs.PassThru { + prefix = "" + } else if sub, ok := h.(*caddyhttp.Subroute); ok { + if key := reloadRouteKeys( + sub.Routes, + scope+string(b), + before+prefix+string(b), + keys, + allowConsume && sub.Errors == nil, + app, + ); key != "" { + prefix += string(b) + key + } + if sub.Errors != nil { + reloadRouteKeys( + sub.Errors.Routes, + scope+string(b)+"errors", + before+prefix+string(b)+"errors", + keys, + false, + app, + ) + } + } else if auth, _ := h.(*caddyauth.Authentication); !slices.Contains( + []string{"vars", "static_response", "invoke"}, caddy.GetModuleName(h), + ) && (auth == nil || slices.ContainsFunc( + slices.Collect(maps.Keys(auth.Providers)), + func(name string) bool { + return name != "http_basic" + }, + )) { + hdr, _ := h.(*headers.Handler) + inter, _ := h.(*intercept.Intercept) + if proxy, ok := h.(*reverseproxy.Handler); err != nil || + ok && len(proxy.HandleResponse) > 0 || + auth != nil || + inter != nil && slices.ContainsFunc( + inter.HandleResponse, + func(rh caddyhttp.ResponseHandler) bool { + return rh.Routes != nil + }, + ) || + hdr != nil && hdr.Request != nil || + caddy.GetModuleName(h) == "tracing" || + reflect.ValueOf(h).Kind() != reflect.Pointer || + caddy.GetModuleName(h) == "request_body" && + reflect.ValueOf(h).Elem().FieldByName("Set").String() != "" { + signature = fmt.Sprintf("%p:%T", app, h) + } + prefix += caddy.GetModuleName(h) + signature + } + } + } + return +} + +func reloadMatchers(sets caddyhttp.MatcherSets) []map[string]any { + config := make([]map[string]any, len(sets)) + for i, set := range sets { + config[i] = make(map[string]any, len(set)) + for _, m := range set { + switch m := m.(type) { + case *caddyhttp.MatchNot: + config[i]["not"] = reloadMatchers(m.MatcherSets) + case *caddyhttp.MatchHost: + config[i]["host"] = slices.Sorted(slices.Values(*m)) + case caddyhttp.MatchHost: + config[i]["host"] = slices.Sorted(slices.Values(m)) + default: + config[i][caddy.GetModuleName(m)] = m + } + } + } + return config +} + // UnmarshalCaddyfile implements caddyfile.Unmarshaler. func (f *FrankenPHPModule) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { for d.Next() { @@ -573,6 +810,7 @@ func parsePhpServer(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error) HandlersRaw: []json.RawMessage{caddyconfig.JSONModuleObject(rewriteHandler, "handler", "rewrite", nil)}, } + phpsrv.GeneratedRewrite = true routes = append(routes, rewriteRoute) } diff --git a/caddy/reload_test.go b/caddy/reload_test.go new file mode 100644 index 0000000000..d5157a244b --- /dev/null +++ b/caddy/reload_test.go @@ -0,0 +1,2526 @@ +package caddy + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/http/httptrace" + "net/textproto" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyauth" + _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/gzip" + _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/zstd" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/fileserver" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/headers" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/intercept" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite" + _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/templates" + _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/timeouts" + _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/tracing" + _ "github.com/dunglas/caddy-cbrotli" + "github.com/dunglas/frankenphp" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "golang.org/x/crypto/bcrypt" +) + +type reloadShutdownBarrier struct { + entered chan struct{} + release chan struct{} + held sync.Once + freed sync.Once +} + +func (b *reloadShutdownBarrier) shutdown(int) { + b.held.Do(func() { + close(b.entered) + <-b.release + }) +} + +func (b *reloadShutdownBarrier) unblock() { b.freed.Do(func() { close(b.release) }) } + +func reloadTestRequest(path, body string) *http.Request { + r := httptest.NewRequest(http.MethodPost, "http://example.com"+path, strings.NewReader(body)) + original := r.Clone(r.Context()) + ctx := context.WithValue(r.Context(), caddy.ReplacerCtxKey, caddy.NewReplacer()) + ctx = context.WithValue(ctx, caddyhttp.OriginalRequestCtxKey, *original) + r = r.WithContext(ctx) + r.URL.Path = "/echo.php" + return r +} + +func setReloadTestRoutes(t *testing.T, app *FrankenPHPApp, paths []string, reverseScopeOrder bool) { + t.Helper() + hosts := caddyhttp.MatchHost{"example.com", "alias.example.com"} + listeners := []string{"127.0.0.1:9080", "127.0.0.1:9081"} + if reverseScopeOrder { + hosts = caddyhttp.MatchHost{"alias.example.com", "example.com"} + listeners = []string{"127.0.0.1:9081", "127.0.0.1:9080"} + } + var routes caddyhttp.RouteList + for i, module := range app.modules { + path := caddyhttp.MatchPath{paths[i] + "/*"} + rw := &rewrite.Rewrite{URI: "/echo.php"} + require.NoError(t, rw.Provision(caddy.Context{})) + inner := caddyhttp.RouteList{ + {Handlers: []caddyhttp.MiddlewareHandler{rw}}, + {Handlers: []caddyhttp.MiddlewareHandler{module}}, + } + require.NoError(t, inner.ProvisionHandlers(caddy.Context{}, nil)) + routes = append(routes, caddyhttp.Route{ + MatcherSets: caddyhttp.MatcherSets{{hosts, path}}, + Handlers: []caddyhttp.MiddlewareHandler{&caddyhttp.Subroute{Routes: inner}}, + }) + } + require.NoError(t, routes.ProvisionHandlers(caddy.Context{}, nil)) + app.httpApp = &caddyhttp.App{Servers: map[string]*caddyhttp.Server{ + "srv0": {Listen: listeners, Routes: routes}, + }} +} + +// Exercise the real PHP shutdown/init and Caddy module handoff without using +// sockets: keeping old modules models requests accepted by the old HTTP config. +func TestReloadHandoffPreservesConcurrentBodiesAndServerScope(t *testing.T) { + if testing.Short() { + t.Skip("requires the PHP runtime") + } + for _, workerMode := range []bool{false, true} { + t.Run(fmt.Sprintf("worker=%t", workerMode), func(t *testing.T) { + root := t.TempDir() + script := ` Date: Sun, 11 Oct 2026 22:20:55 +0200 Subject: [PATCH 2/2] fix tests --- caddy/reload_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/caddy/reload_test.go b/caddy/reload_test.go index d5157a244b..7ca3ecee53 100644 --- a/caddy/reload_test.go +++ b/caddy/reload_test.go @@ -105,7 +105,8 @@ func TestReloadHandoffPreservesConcurrentBodiesAndServerScope(t *testing.T) { } for _, workerMode := range []bool{false, true} { t.Run(fmt.Sprintf("worker=%t", workerMode), func(t *testing.T) { - root := t.TempDir() + root, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) script := `