Add composite MySQL safety checks and stable recovery

.gitignore

@@ -4,5 +4,5 @@ dist/
 main
 conf/freno.local.conf.json
 .vendor/
-freno
+/freno
 .idea/

README.md

@@ -35,6 +35,9 @@ Clients (application, scripts, jobs) are expected to consult with `freno`. `fren
 
 Read more about [freno and MySQL throttling](doc/mysql.md)
 
+Before enabling required dependencies or recovery, read the
+[MySQL safety behavior changes and rollout warnings](doc/mysql.md#behavior-changes).
+
 ### Use cases
 
 `freno` is useful for bulk operations: massive loading/archiving tasks, schema migrations, mass updates. Such operations typically walk through thousands to millions of rows and may cause undesired effects such as MySQL replication lags. By breaking these tasks to small subtasks (e.g. `100` rows at a time), and by consulting `freno` before applying each such subtask, we are able to achieve the same result without ill effect to the database and to the application that uses it.

cmd/freno/main.go

@@ -3,7 +3,10 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	gohttp "net/http"
+	"os"
+	"sort"
 	"strings"
 
 	"github.com/github/freno/pkg/config"
@@ -96,6 +99,11 @@ func main() {
 
 	switch {
 	case *http:
+		if !*quiet {
+			if err := logMySQLSafetyConfiguration(config.Settings().Stores.MySQL, os.Stderr); err != nil {
+				log.Fatalf("Error reporting MySQL safety configuration: %s", err)
+			}
+		}
 		err := httpServe()
 		log.Errore(err)
 	default:
@@ -116,6 +124,32 @@ func loadConfiguration(configFile string) {
 	}
 }
 
+func logMySQLSafetyConfiguration(settings config.MySQLConfigurationSettings, output io.Writer) error {
+	clusterNames := make([]string, 0, len(settings.Clusters))
+	for name, cluster := range settings.Clusters {
+		if len(cluster.RequiredClusters) > 0 || len(cluster.ShadowRequiredClusters) > 0 || cluster.RecoveryDurationMillis > 0 || cluster.FailOnNoHosts {
+			clusterNames = append(clusterNames, name)
+		}
+	}
+	sort.Strings(clusterNames)
+
+	for _, name := range clusterNames {
+		cluster := settings.Clusters[name]
+		recoveryThreshold := cluster.ThrottleThreshold
+		if cluster.RecoveryThreshold != nil {
+			recoveryThreshold = *cluster.RecoveryThreshold
+		}
+		// Safety notices must be visible at the default ERROR log level, without changing other log filtering.
+		if _, err := fmt.Fprintf(output,
+			"freno MySQL safety configuration: cluster=%q RequiredClusters=%q ShadowRequiredClusters=%q RecoveryDurationMillis=%d RecoveryThreshold=%g ThrottleThreshold=%g FailOnNoHosts=%t\n",
+			name, cluster.RequiredClusters, cluster.ShadowRequiredClusters, cluster.RecoveryDurationMillis, recoveryThreshold, cluster.ThrottleThreshold, cluster.FailOnNoHosts,
+		); err != nil {
+			return fmt.Errorf("cannot log MySQL safety configuration for cluster %q: %w", name, err)
+		}
+	}
+	return nil
+}
+
 func httpServe() error {
 	throttler := throttle.NewThrottler()
 	log.Infof("Starting consensus service")

cmd/freno/main_test.go

@@ -0,0 +1,111 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/github/freno/pkg/config"
+	"github.com/outbrain/golib/log"
+)
+
+func TestLogMySQLSafetyConfiguration(t *testing.T) {
+	originalLogLevel := log.GetLevel()
+	log.SetLevel(log.ERROR)
+	defer log.SetLevel(originalLogLevel)
+
+	recoveryThreshold := 3.0
+	settings := config.MySQLConfigurationSettings{
+		Clusters: map[string]*config.MySQLClusterConfigurationSettings{
+			"legacy": {ThrottleThreshold: 5},
+			"dependency": {
+				RequiredClusters:  []string{"primary"},
+				ThrottleThreshold: 5,
+				User:              "do-not-log-user",
+				Password:          "do-not-log-password",
+				MetricQuery:       "do-not-log-query",
+			},
+			"primary": {ThrottleThreshold: 50},
+			"shadow-only": {
+				ShadowRequiredClusters: []string{"primary"},
+				ThrottleThreshold:      5,
+			},
+			"fail-closed": {
+				FailOnNoHosts:     true,
+				ThrottleThreshold: 50,
+			},
+			"recovery": {
+				RecoveryDurationMillis: 5000,
+				RecoveryThreshold:      &recoveryThreshold,
+				ThrottleThreshold:      10,
+			},
+		},
+	}
+	var output bytes.Buffer
+	if err := logMySQLSafetyConfiguration(settings, &output); err != nil {
+		t.Fatal(err)
+	}
+	want := "freno MySQL safety configuration: cluster=\"dependency\" RequiredClusters=[\"primary\"] ShadowRequiredClusters=[] RecoveryDurationMillis=0 RecoveryThreshold=5 ThrottleThreshold=5 FailOnNoHosts=false\n" +
+		"freno MySQL safety configuration: cluster=\"fail-closed\" RequiredClusters=[] ShadowRequiredClusters=[] RecoveryDurationMillis=0 RecoveryThreshold=50 ThrottleThreshold=50 FailOnNoHosts=true\n" +
+		"freno MySQL safety configuration: cluster=\"recovery\" RequiredClusters=[] ShadowRequiredClusters=[] RecoveryDurationMillis=5000 RecoveryThreshold=3 ThrottleThreshold=10 FailOnNoHosts=false\n" +
+		"freno MySQL safety configuration: cluster=\"shadow-only\" RequiredClusters=[] ShadowRequiredClusters=[\"primary\"] RecoveryDurationMillis=0 RecoveryThreshold=5 ThrottleThreshold=5 FailOnNoHosts=false\n"
+	if got := output.String(); got != want {
+		t.Fatalf("startup safety configuration = %q, want %q", got, want)
+	}
+	if log.GetLevel() != log.ERROR {
+		t.Fatal("startup safety notices changed unrelated log filtering")
+	}
+}
+
+func TestLogMySQLSafetyConfigurationWithoutSafetySettings(t *testing.T) {
+	settings := config.MySQLConfigurationSettings{
+		Clusters: map[string]*config.MySQLClusterConfigurationSettings{
+			"legacy": {ThrottleThreshold: 5},
+		},
+	}
+	var output bytes.Buffer
+	if err := logMySQLSafetyConfiguration(settings, &output); err != nil {
+		t.Fatal(err)
+	}
+	if output.Len() != 0 {
+		t.Fatalf("unexpected safety notice for legacy configuration: %q", output.String())
+	}
+}
+
+func TestLogMySQLSafetyConfigurationEscapesNames(t *testing.T) {
+	settings := config.MySQLConfigurationSettings{
+		Clusters: map[string]*config.MySQLClusterConfigurationSettings{
+			"line\nbreak": {RequiredClusters: []string{"probe\nname"}},
+		},
+	}
+	var output bytes.Buffer
+	if err := logMySQLSafetyConfiguration(settings, &output); err != nil {
+		t.Fatal(err)
+	}
+	got := output.String()
+	if strings.Count(got, "\n") != 1 || !strings.Contains(got, `cluster="line\nbreak" RequiredClusters=["probe\nname"]`) {
+		t.Fatalf("cluster names did not stay within one escaped log line: %q", got)
+	}
+}
+
+type failingLogWriter struct {
+	err error
+}
+
+func (writer failingLogWriter) Write([]byte) (int, error) {
+	return 0, writer.err
+}
+
+func TestLogMySQLSafetyConfigurationWriteError(t *testing.T) {
+	settings := config.MySQLConfigurationSettings{
+		Clusters: map[string]*config.MySQLClusterConfigurationSettings{
+			"primary": {FailOnNoHosts: true},
+		},
+	}
+	writeErr := errors.New("log destination unavailable")
+	err := logMySQLSafetyConfiguration(settings, failingLogWriter{err: writeErr})
+	if !errors.Is(err, writeErr) || !strings.Contains(err.Error(), `"primary"`) {
+		t.Fatalf("expected contextual startup logging error, got %v", err)
+	}
+}

doc/http.md

@@ -113,6 +113,33 @@ Notes:
 
 - `/config/memcache`: show the [memcache](memcache.md) configuration used, so freno clients can use it to implement more efficient read strategies.
 
+### Read-only MySQL diagnostics
+
+`GET /explain/mysql/<cluster>` returns the source metric and all of its required
+and shadow dependencies, even when the source is healthy. It requires the
+current leader (`503` on a follower), uses exact configured names without
+`FallbackCluster`, and returns `404` for an unknown cluster.
+
+Each entry in `Metrics` includes `MetricName`, the cached `Value`, `Threshold`,
+effective `RecoveryThreshold`, `RecoveryDurationMillis`, `CacheMillis`,
+`FailOnNoHosts`, and host-exclusion settings. `Mode` is `source`, `enforced`, or
+`shadow`, relative to the requested source. Dependencies below a shadow edge
+remain shadow-only unless also reachable through an entirely required path.
+Dependency lists preserve the configured graph.
+
+`StatusCode` and `Message` describe each metric independently, not the source's
+combined admission decision. `Value` is `null` when a usable measurement is
+unavailable; recovery errors retain the observed value. Recovery may reject
+even when the value is below the throttle threshold. Probe-error details are
+redacted; credentials, SQL, connection settings, and host identities are not
+exported. The response reflects loaded policy and cached aggregates, not
+per-host values or sample timestamps.
+
+This endpoint does not run application policy, random throttling, priority or
+shared-domain checks, issue SQL, advance recovery, or increment shadow/check
+counters. A diagnostic HTTP `200` means the explanation was retrieved, **not**
+permission to write. Continue to use the regular `/check` endpoint for admission.
+
 # GET method
 
 `GET` and `HEAD` respond with same status codes. But `GET` requests compute and return additional data. Automated requests should not be interested in this data; the status code is what should guide the clients. However humans or manual requests may benefit from extra information supplied by the `GET` request.

doc/memcache.md

@@ -37,6 +37,13 @@ Optionally set `MemcachePath` (default is `"freno"`):
 - The value will be of the form `<epochmillis>:<aggregated-value>`.
   - As example, it might be `1497418678836:0.54` where `1497418678836` is the unix epoch in milliseconds, and `0.54` is the aggregated value.
   - Embedding the epoch within the value allows the app to double-check the validity of the value, or go into more granular validation.
+- For a MySQL cluster with `RequiredClusters`, the key is removed while any
+  required metric is unavailable, recovering, or above its configured
+  threshold. Direct memcache clients therefore observe the same composite
+  safety gate as HTTP clients.
+- `ShadowRequiredClusters` never removes the requested cluster's key. Its
+  [background would-reject counters](mysql.md#shadow-counters-and-limits) describe
+  prospective dependency failures during leader aggregation, not client reads.
 
 ### Runtime access to memcache configuration