diff --git a/.gitignore b/.gitignore index 3b7d16e3..f1f1f577 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,5 @@ dist/ main conf/freno.local.conf.json .vendor/ -freno +/freno .idea/ diff --git a/README.md b/README.md index 9da2df53..4d3764a1 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,9 @@ Clients (application, scripts, jobs) are expected to consult with `freno`. `fren Read more about [freno and MySQL throttling](doc/mysql.md) +Before enabling required dependencies or recovery, read the +[MySQL safety behavior changes and rollout warnings](doc/mysql.md#behavior-changes). + ### Use cases `freno` is useful for bulk operations: massive loading/archiving tasks, schema migrations, mass updates. Such operations typically walk through thousands to millions of rows and may cause undesired effects such as MySQL replication lags. By breaking these tasks to small subtasks (e.g. `100` rows at a time), and by consulting `freno` before applying each such subtask, we are able to achieve the same result without ill effect to the database and to the application that uses it. diff --git a/cmd/freno/main.go b/cmd/freno/main.go index 3bf0a7f9..fd88b2ae 100644 --- a/cmd/freno/main.go +++ b/cmd/freno/main.go @@ -3,7 +3,10 @@ package main import ( "flag" "fmt" + "io" gohttp "net/http" + "os" + "sort" "strings" "github.com/github/freno/pkg/config" @@ -96,6 +99,11 @@ func main() { switch { case *http: + if !*quiet { + if err := logMySQLSafetyConfiguration(config.Settings().Stores.MySQL, os.Stderr); err != nil { + log.Fatalf("Error reporting MySQL safety configuration: %s", err) + } + } err := httpServe() log.Errore(err) default: @@ -116,6 +124,32 @@ func loadConfiguration(configFile string) { } } +func logMySQLSafetyConfiguration(settings config.MySQLConfigurationSettings, output io.Writer) error { + clusterNames := make([]string, 0, len(settings.Clusters)) + for name, cluster := range settings.Clusters { + if len(cluster.RequiredClusters) > 0 || cluster.RecoveryDurationMillis > 0 || cluster.FailOnNoHosts { + clusterNames = append(clusterNames, name) + } + } + sort.Strings(clusterNames) + + for _, name := range clusterNames { + cluster := settings.Clusters[name] + recoveryThreshold := cluster.ThrottleThreshold + if cluster.RecoveryThreshold != nil { + recoveryThreshold = *cluster.RecoveryThreshold + } + // Safety notices must be visible at the default ERROR log level, without changing other log filtering. + if _, err := fmt.Fprintf(output, + "freno MySQL safety configuration: cluster=%q RequiredClusters=%q RecoveryDurationMillis=%d RecoveryThreshold=%g ThrottleThreshold=%g FailOnNoHosts=%t\n", + name, cluster.RequiredClusters, cluster.RecoveryDurationMillis, recoveryThreshold, cluster.ThrottleThreshold, cluster.FailOnNoHosts, + ); err != nil { + return fmt.Errorf("cannot log MySQL safety configuration for cluster %q: %w", name, err) + } + } + return nil +} + func httpServe() error { throttler := throttle.NewThrottler() log.Infof("Starting consensus service") diff --git a/cmd/freno/main_test.go b/cmd/freno/main_test.go new file mode 100644 index 00000000..593f65d0 --- /dev/null +++ b/cmd/freno/main_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "bytes" + "errors" + "strings" + "testing" + + "github.com/github/freno/pkg/config" + "github.com/outbrain/golib/log" +) + +func TestLogMySQLSafetyConfiguration(t *testing.T) { + originalLogLevel := log.GetLevel() + log.SetLevel(log.ERROR) + defer log.SetLevel(originalLogLevel) + + recoveryThreshold := 3.0 + settings := config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "legacy": {ThrottleThreshold: 5}, + "dependency": { + RequiredClusters: []string{"primary"}, + ThrottleThreshold: 5, + User: "do-not-log-user", + Password: "do-not-log-password", + MetricQuery: "do-not-log-query", + }, + "primary": {ThrottleThreshold: 50}, + "fail-closed": { + FailOnNoHosts: true, + ThrottleThreshold: 50, + }, + "recovery": { + RecoveryDurationMillis: 5000, + RecoveryThreshold: &recoveryThreshold, + ThrottleThreshold: 10, + }, + }, + } + var output bytes.Buffer + if err := logMySQLSafetyConfiguration(settings, &output); err != nil { + t.Fatal(err) + } + want := "freno MySQL safety configuration: cluster=\"dependency\" RequiredClusters=[\"primary\"] RecoveryDurationMillis=0 RecoveryThreshold=5 ThrottleThreshold=5 FailOnNoHosts=false\n" + + "freno MySQL safety configuration: cluster=\"fail-closed\" RequiredClusters=[] RecoveryDurationMillis=0 RecoveryThreshold=50 ThrottleThreshold=50 FailOnNoHosts=true\n" + + "freno MySQL safety configuration: cluster=\"recovery\" RequiredClusters=[] RecoveryDurationMillis=5000 RecoveryThreshold=3 ThrottleThreshold=10 FailOnNoHosts=false\n" + if got := output.String(); got != want { + t.Fatalf("startup safety configuration = %q, want %q", got, want) + } + if log.GetLevel() != log.ERROR { + t.Fatal("startup safety notices changed unrelated log filtering") + } +} + +func TestLogMySQLSafetyConfigurationWithoutSafetySettings(t *testing.T) { + settings := config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "legacy": {ThrottleThreshold: 5}, + }, + } + var output bytes.Buffer + if err := logMySQLSafetyConfiguration(settings, &output); err != nil { + t.Fatal(err) + } + if output.Len() != 0 { + t.Fatalf("unexpected safety notice for legacy configuration: %q", output.String()) + } +} + +func TestLogMySQLSafetyConfigurationEscapesNames(t *testing.T) { + settings := config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "line\nbreak": {RequiredClusters: []string{"probe\nname"}}, + }, + } + var output bytes.Buffer + if err := logMySQLSafetyConfiguration(settings, &output); err != nil { + t.Fatal(err) + } + got := output.String() + if strings.Count(got, "\n") != 1 || !strings.Contains(got, `cluster="line\nbreak" RequiredClusters=["probe\nname"]`) { + t.Fatalf("cluster names did not stay within one escaped log line: %q", got) + } +} + +type failingLogWriter struct { + err error +} + +func (writer failingLogWriter) Write([]byte) (int, error) { + return 0, writer.err +} + +func TestLogMySQLSafetyConfigurationWriteError(t *testing.T) { + settings := config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "primary": {FailOnNoHosts: true}, + }, + } + writeErr := errors.New("log destination unavailable") + err := logMySQLSafetyConfiguration(settings, failingLogWriter{err: writeErr}) + if !errors.Is(err, writeErr) || !strings.Contains(err.Error(), `"primary"`) { + t.Fatalf("expected contextual startup logging error, got %v", err) + } +} diff --git a/doc/memcache.md b/doc/memcache.md index 7cbe91e7..5538de2e 100644 --- a/doc/memcache.md +++ b/doc/memcache.md @@ -37,6 +37,10 @@ Optionally set `MemcachePath` (default is `"freno"`): - The value will be of the form `:`. - As example, it might be `1497418678836:0.54` where `1497418678836` is the unix epoch in milliseconds, and `0.54` is the aggregated value. - Embedding the epoch within the value allows the app to double-check the validity of the value, or go into more granular validation. +- For a MySQL cluster with `RequiredClusters`, Freno removes the source cluster's + memcache key when a required metric is unavailable, recovering, or above its + threshold. Clients must treat a missing key as unavailable and retry, defer, or + consult Freno, not as permission to proceed. ### Runtime access to memcache configuration diff --git a/doc/mysql.md b/doc/mysql.md index 38d3519d..f1232175 100644 --- a/doc/mysql.md +++ b/doc/mysql.md @@ -120,6 +120,7 @@ Noteworthy: - `prod4` chooses to (but doesn't have to) override the `ThrottleThreshold` to `0.8` seconds - `prod4` list of servers is dictated by `HAProxy`. `freno` will routinely and dynamically poll given HAProxy server for list of hosts. These will include any hosts not in `NOLB`. +- `sharded` discovers `REPLICA` tablets for the existing replication-lag checks. Vitess primary protection is not included. - `local` cluster chooses to override `User`, `Password` and `IgnoreHostsCount`. - `local` cluster defines a static list of hosts. - `sharded` list of servers is read from Vitess vtctld's HTTP `API`. To read it with vtctld's gRPC `GetTablets` instead (Vitess v23 removes the HTTP API), set `"GRPCAddress": "vtctld.example.com:15999"` and `"GRPCInsecure": true`. Only plaintext gRPC is supported. @@ -178,3 +179,178 @@ unsuccessful refresh can leave the previous inventory in place. `freno` explicitly recognizes `show global ...` statements and reads the result's numeric value. Otherwise you may provide any query that returns a single row, single numeric column. + +### Composing replica, primary, and ProxySQL protection + +#### Behavior changes + +**Read before enabling dependencies or recovery.** These settings are opt-in, but +they can reject application work as soon as the new configuration takes effect: + +- **Startup and leadership recovery:** a cluster with `RecoveryDurationMillis > 0` + must establish a full continuously healthy window after every process start or + leadership acquisition, even without `RequiredClusters`. Ordinary write checks + return `429` while a successfully collected metric is recovering; collection + errors can return `500` instead. Empty-host behavior is controlled separately by + `FailOnNoHosts`. +- **Configuration changes require a restart:** the running service does not + live-reload configuration. Applying changes, including rollback, requires + restarting the affected process. Metrics that retain a positive + `RecoveryDurationMillis` must establish a new healthy window. +- **Required dependencies fail closed:** missing runtime metrics or collection + failures, such as incorrect discovery or missing query privileges, return `500` + for checks that reach that dependency. An empty host list also fails with `500` + when the dependency has `FailOnNoHosts: true`. Overload or incomplete recovery + returns `429`; neither response permits writes. +- **Memcache decisions are affected:** when memcache publishing is configured, + aggregation deletes a cluster's published key while any required dependency is + missing, in error, recovering, or above its threshold. Previously cached client + reads are not synchronously invalidated. +- **Enforcement is not a dry run:** adding a probe and referencing it from + `RequiredClusters` in the same configuration change can immediately block work + after restart. Collect and inspect a separately named probe before adding it + to `RequiredClusters`. An unreferenced probe does not gate the source cluster + or predict its combined admission decision. +- **Coordinate client wait budgets:** a budget shorter than the remaining + recovery window can expire before recovery finishes. Agree on safe retry/defer + behavior; do not bypass checks to avoid a timeout. + +At service startup, each cluster with required dependencies, a positive +recovery duration, or fail-on-no-hosts enabled produces one safety-configuration notice on +standard error. It includes the dependency names, effective recovery and throttle +thresholds, recovery duration, and no-host policy, without credentials or query +text. These notices are visible without `--verbose`; `--quiet` suppresses them. + +#### Rejection counters + +Application checks returning `429` increment +`check.rejected.mysql...total`. The blocking cluster +comes from the response's `MetricName`, not necessarily the cluster requested +by the application. For example: + +```text +check.rejected.mysql.prod4.threshold.total +check.rejected.mysql.prod4-primary.threshold.total +check.rejected.mysql.prod4-primary.recovery.total +``` + +With the configuration below, the first counter records replica-lag rejections, +the second primary-pressure rejections, and the third primary recovery holds. +`threshold` means the value exceeded the effective threshold; `recovery` means +the healthy window is incomplete. Other `429` decisions, such as a shared-domain +hold while the local value is healthy, use `other` rather than claiming local +overload. + +Each check increments only the counter for its first blocking metric. Healthy +checks, non-`429` failures, and the internal `freno` and `freno-share-domain` +checks do not increment these counters. These are process-local cumulative +request counters that reset on restart, not counts of overloaded probe readings +or distinct overload episodes. They do not measure memcache client rejections. + +#### Configuration example + +A cluster can require additional cluster metrics to pass before `freno` permits +work. The example below enforces separately sampled primary-load and +ProxySQL-capacity checks. To collect without enforcing, leave these probes out +of `RequiredClusters`: + +```json +"Clusters": { + "prod4": { + "RequiredClusters": [ + "prod4-primary", + "prod4-proxysql" + ], + "HAProxySettings": { + "Host": "my.haproxy.mydomain.com", + "Port": 1001, + "PoolName": "my_prod4_pool" + } + }, + "prod4-primary": { + "MetricQuery": "show global status like 'Threads_running'", + "CacheMillis": 500, + "ThrottleThreshold": 50, + "RecoveryThreshold": 35, + "RecoveryDurationMillis": 5000, + "FailOnNoHosts": true, + "StaticHostsSettings": { + "Hosts": [ + "my.prod4.primary.vip.example.com:3306" + ] + } + }, + "prod4-proxysql": { + "User": "${proxysql_stats_user}", + "Password": "${proxysql_stats_password}", + "MetricQuery": "select connection_pressure_percent from operational_metrics.proxysql_capacity limit 1", + "CacheMillis": 500, + "ThrottleThreshold": 80, + "RecoveryThreshold": 60, + "RecoveryDurationMillis": 5000, + "FailOnNoHosts": true, + "StaticHostsSettings": { + "Hosts": [ + "my.prod4.proxysql.example.com:6032" + ] + } + } +} +``` + +The ProxySQL query above is illustrative: deployments must provide a scalar query or operational view whose value increases as usable connection capacity is consumed. + +- `RequiredClusters` lists additional configured metrics that must return `HTTP 200`. Missing required runtime metrics fail closed. +- `FailOnNoHosts` changes the legacy no-host response from `HTTP 200` to `HTTP 500`. Enable it for primary and ProxySQL safety probes, where an empty roster cannot prove that work is safe. +- `RecoveryThreshold` is the lower threshold that begins recovery after a cluster has throttled. It must not exceed `ThrottleThreshold`. +- `RecoveryDurationMillis` is the continuous time the metric must remain at or below `RecoveryThreshold` before checks return `HTTP 200` again. A new error or threshold violation resets the recovery window. +- A new process or leader starts configured recovery metrics in the recovering state. It must observe a complete healthy window before admitting work. +- Metric sampling remains centralized in the `freno` leader and uses the existing metric cache. Application checks read the aggregated decision rather than querying the primary or ProxySQL for every batch. +- Check responses include `MetricName`, identifying the replica, primary, or ProxySQL metric that blocked a composite decision. `recovery.mysql..active` reports whether the recovery latch is active. + +Configuration loading rejects missing `RequiredClusters` references and +dependency cycles. Add a validated probe to `RequiredClusters` only in a +separately reviewed activation change. + +### Rolling out primary safety checks + +Deploy a binary containing these settings before deploying configuration that +uses them. Verify the capabilities of the deployed binary rather than assuming +that a configuration change installs new server functionality. + +1. Deploy the binary with the existing configuration. Without required + dependencies, fail-on-no-hosts, or a recovery duration, legacy cluster + decisions retain their existing behavior. +2. Add separately named primary metrics and reference them from the application's + normal cluster only after validation. Initially leave them out of + `RequiredClusters` and inspect their aggregates. + Validate discovery, privileges, probe load, errors, and recovery gauges. + Do not add new probes and enforce them in the same deployment. +3. Review thresholds against recent traffic, known incidents, and per-host/shard + coverage. Averaged monitoring data is not equivalent to the instantaneous SQL + samples used by the safety check. Missing data must not be interpreted as zero + or healthy. +4. In a separate reviewed configuration change, add a primary dependency to + `RequiredClusters` on one cluster whose probes have + been validated. Exercise overload, probe + failure, missing hosts, leadership changes, and the HTTP and memcache paths. + Required probes fail closed; verify callers defer work rather than bypassing + protection on an error. +5. Expand only after reviewing rejection rates, recovery duration, job backlog, + application errors, and primary health against agreed success/rollback limits. + +Choose recovery durations and client wait budgets for the workload, and +coordinate restarts and leadership transitions with application operators. +Neither a longer timeout nor client-side consecutive checks guarantee recovery +during overload. + +Rollback enforcement by removing the primary dependency from the normal cluster's +`RequiredClusters`, keeping its existing replica-lag configuration. +Apply the configuration and restart the affected process; this is not an +instantaneous runtime toggle, and any remaining recovery-enabled metrics need +a new healthy window. Remove probe configurations only after removing every +reference to them. + +### Scope boundaries + +These settings provide reusable admission checks and recovery behavior. Deployments remain responsible for choosing appropriate metrics and thresholds, configuring their host-discovery sources, monitoring probe health and rejection rates, and controlling application concurrency or fallback behavior. diff --git a/pkg/base/throttle_metric.go b/pkg/base/throttle_metric.go index 8c486fce..93362e3f 100644 --- a/pkg/base/throttle_metric.go +++ b/pkg/base/throttle_metric.go @@ -18,6 +18,7 @@ type MetricResultFunc func() (metricResult MetricResult, threshold float64) var ThresholdExceededError = errors.New("Threshold exceeded") var NoHostsError = errors.New("No hosts found") +var RecoveryNotCompleteError = errors.New("Recovery period not complete") var noResultYetError = errors.New("Metric not collected yet") var NoSuchMetricError = errors.New("No such metric") @@ -63,3 +64,16 @@ func NewSimpleMetricResult(value float64) MetricResult { func (metricResult *simpleMetricResult) Get() (float64, error) { return metricResult.Value, nil } + +type errorMetricResult struct { + Value float64 + Err error +} + +func NewErrorMetricResult(value float64, err error) MetricResult { + return &errorMetricResult{Value: value, Err: err} +} + +func (metricResult *errorMetricResult) Get() (float64, error) { + return metricResult.Value, metricResult.Err +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 21216e0c..527e6951 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -3,6 +3,8 @@ package config import ( "encoding/json" "io/ioutil" + "os" + "path/filepath" "testing" "github.com/outbrain/golib/log" @@ -112,6 +114,150 @@ func TestMySQLFallbackCluster(t *testing.T) { } } +func TestMySQLRequiredClusters(t *testing.T) { + tests := []struct { + name string + clusters map[string]*MySQLClusterConfigurationSettings + wantErr string + }{ + {name: "omitted clusters"}, + {name: "empty clusters", clusters: map[string]*MySQLClusterConfigurationSettings{}}, + { + name: "valid dependencies", + clusters: map[string]*MySQLClusterConfigurationSettings{ + "replicas": {RequiredClusters: []string{"primary", "proxysql"}}, + "primary": {}, + "proxysql": {}, + }, + }, + { + name: "unknown dependency", + clusters: map[string]*MySQLClusterConfigurationSettings{ + "replicas": {RequiredClusters: []string{"primary"}}, + }, + wantErr: `Stores.MySQL.Clusters.replicas.RequiredClusters references unknown cluster "primary"`, + }, + { + name: "dependency cycle", + clusters: map[string]*MySQLClusterConfigurationSettings{ + "replicas": {RequiredClusters: []string{"primary"}}, + "primary": {RequiredClusters: []string{"replicas"}}, + }, + wantErr: `Stores.MySQL.Clusters contains a RequiredClusters cycle involving "primary"`, + }, + { + name: "nested and shared dependencies", + clusters: map[string]*MySQLClusterConfigurationSettings{ + "replicas": {RequiredClusters: []string{"required", "primary", "primary"}}, + "required": {RequiredClusters: []string{"primary"}}, + "primary": {RequiredClusters: []string{"capacity", "extra"}}, + "capacity": {}, + "extra": {RequiredClusters: []string{"capacity"}}, + }, + }, + { + name: "null cluster", + clusters: map[string]*MySQLClusterConfigurationSettings{ + "replicas": nil, + }, + wantErr: `Stores.MySQL.Clusters.replicas must not be null`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + settings := MySQLConfigurationSettings{Clusters: test.clusters} + err := settings.postReadAdjustments() + if test.wantErr == "" && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if test.wantErr != "" && (err == nil || err.Error() != test.wantErr) { + t.Fatalf("error = %v, want %q", err, test.wantErr) + } + }) + } +} + +func TestValidateRequiredClustersRejectsNullDependency(t *testing.T) { + settings := MySQLConfigurationSettings{ + Clusters: map[string]*MySQLClusterConfigurationSettings{ + "replicas": {RequiredClusters: []string{"primary"}}, + "primary": nil, + }, + } + wantErr := "Stores.MySQL.Clusters.primary must not be null" + if err := settings.validateRequiredClusters(); err == nil || err.Error() != wantErr { + t.Fatalf("error = %v, want %q", err, wantErr) + } +} + +func TestReadRejectsMisspelledRequiredCluster(t *testing.T) { + directory := t.TempDir() + settings := newConfigurationSettings() + settings.RaftDataDir = directory + settings.Stores.MySQL.Clusters = map[string]*MySQLClusterConfigurationSettings{ + "replicas": {RequiredClusters: []string{"primray"}}, + "primary": {}, + } + contents, err := json.Marshal(settings) + if err != nil { + t.Fatal(err) + } + filename := filepath.Join(directory, "freno.json") + if err := os.WriteFile(filename, contents, 0600); err != nil { + t.Fatal(err) + } + + configuration := newConfiguration() + originalSettings := configuration.settings + wantErr := `Stores.MySQL.Clusters.replicas.RequiredClusters references unknown cluster "primray"` + if err := configuration.Read(filename); err == nil || err.Error() != wantErr { + t.Fatalf("error = %v, want %q", err, wantErr) + } + if configuration.settings != originalSettings { + t.Fatal("invalid configuration replaced the active settings") + } +} + +func TestMySQLRecoveryConfiguration(t *testing.T) { + recoveryThreshold := 5.0 + tooHighRecoveryThreshold := 11.0 + tests := []struct { + name string + clusterSettings *MySQLClusterConfigurationSettings + wantErr string + }{ + {name: "disabled", clusterSettings: &MySQLClusterConfigurationSettings{ThrottleThreshold: 10}}, + {name: "valid", clusterSettings: &MySQLClusterConfigurationSettings{ThrottleThreshold: 10, RecoveryThreshold: &recoveryThreshold, RecoveryDurationMillis: 5000}}, + {name: "default recovery threshold", clusterSettings: &MySQLClusterConfigurationSettings{ThrottleThreshold: 10, RecoveryDurationMillis: 5000}}, + { + name: "negative duration", + clusterSettings: &MySQLClusterConfigurationSettings{ThrottleThreshold: 10, RecoveryDurationMillis: -1}, + wantErr: "Stores.MySQL.Clusters.primary.RecoveryDurationMillis must not be negative", + }, + { + name: "recovery threshold above throttle threshold", + clusterSettings: &MySQLClusterConfigurationSettings{ThrottleThreshold: 10, RecoveryThreshold: &tooHighRecoveryThreshold, RecoveryDurationMillis: 5000}, + wantErr: "Stores.MySQL.Clusters.primary.RecoveryThreshold must not exceed ThrottleThreshold", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + settings := MySQLConfigurationSettings{ + Clusters: map[string]*MySQLClusterConfigurationSettings{"primary": test.clusterSettings}, + } + err := settings.postReadAdjustments() + if test.wantErr == "" && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if test.wantErr != "" && (err == nil || err.Error() != test.wantErr) { + t.Fatalf("error = %v, want %q", err, test.wantErr) + } + }) + } +} + func dump(path string, contents *ConfigurationSettings) error { json, _ := json.Marshal(contents) err := ioutil.WriteFile(path, json, 0644) diff --git a/pkg/config/mysql_config.go b/pkg/config/mysql_config.go index 8e5a7bf3..25761122 100644 --- a/pkg/config/mysql_config.go +++ b/pkg/config/mysql_config.go @@ -7,6 +7,7 @@ package config import ( "fmt" "os" + "sort" ) const DefaultMySQLPort = 3306 @@ -28,6 +29,11 @@ type MySQLClusterConfigurationSettings struct { ProxySQLSettings ProxySQLConfigurationSettings // If list of servers is to be acquired via ProxySQL, provide this field VitessSettings VitessConfigurationSettings // If list of servers is to be acquired via Vitess, provide this field StaticHostsSettings StaticHostsConfigurationSettings + + RecoveryThreshold *float64 // optional threshold that must be met before a throttled cluster begins recovery + RecoveryDurationMillis int // continuous recovery duration required before writes resume + FailOnNoHosts bool // reject checks when no eligible hosts remain instead of preserving legacy fail-open behavior + RequiredClusters []string // additional MySQL cluster metrics that must pass for this cluster to pass } // Hook to implement adjustments after reading each configuration file. @@ -75,6 +81,9 @@ func (settings *MySQLConfigurationSettings) postReadAdjustments() error { if settings.Port == 0 { settings.Port = DefaultMySQLPort } + if settings.Clusters == nil { + settings.Clusters = make(map[string]*MySQLClusterConfigurationSettings) + } if settings.FallbackCluster != "" { if _, ok := settings.Clusters[settings.FallbackCluster]; !ok { return fmt.Errorf("Stores.MySQL.FallbackCluster %q does not name a configured cluster", settings.FallbackCluster) @@ -97,7 +106,10 @@ func (settings *MySQLConfigurationSettings) postReadAdjustments() error { } } - for _, clusterSettings := range settings.Clusters { + for clusterName, clusterSettings := range settings.Clusters { + if clusterSettings == nil { + return fmt.Errorf("Stores.MySQL.Clusters.%s must not be null", clusterName) + } if err := clusterSettings.postReadAdjustments(); err != nil { return err } @@ -148,6 +160,70 @@ func (settings *MySQLConfigurationSettings) postReadAdjustments() error { if !clusterSettings.VitessSettings.IsEmpty() && len(clusterSettings.VitessSettings.Cells) < 1 { clusterSettings.VitessSettings.Cells = settings.VitessCells } + if clusterSettings.RecoveryDurationMillis < 0 { + return fmt.Errorf("Stores.MySQL.Clusters.%s.RecoveryDurationMillis must not be negative", clusterName) + } + if clusterSettings.RecoveryDurationMillis > 0 { + recoveryThreshold := clusterSettings.ThrottleThreshold + if clusterSettings.RecoveryThreshold != nil { + recoveryThreshold = *clusterSettings.RecoveryThreshold + } + if recoveryThreshold > clusterSettings.ThrottleThreshold { + return fmt.Errorf( + "Stores.MySQL.Clusters.%s.RecoveryThreshold must not exceed ThrottleThreshold", + clusterName, + ) + } + } + } + return settings.validateRequiredClusters() +} + +func (settings *MySQLConfigurationSettings) validateRequiredClusters() error { + const ( + unvisited = iota + visiting + visited + ) + states := make(map[string]int) + + var visit func(string) error + visit = func(clusterName string) error { + switch states[clusterName] { + case visiting: + return fmt.Errorf("Stores.MySQL.Clusters contains a RequiredClusters cycle involving %q", clusterName) + case visited: + return nil + } + + states[clusterName] = visiting + cluster := settings.Clusters[clusterName] + if cluster == nil { + return fmt.Errorf("Stores.MySQL.Clusters.%s must not be null", clusterName) + } + for _, name := range cluster.RequiredClusters { + if _, ok := settings.Clusters[name]; !ok { + return fmt.Errorf("Stores.MySQL.Clusters.%s.RequiredClusters references unknown cluster %q", clusterName, name) + } + if err := visit(name); err != nil { + return err + } + } + states[clusterName] = visited + return nil + } + + clusterNames := make([]string, 0, len(settings.Clusters)) + for clusterName := range settings.Clusters { + clusterNames = append(clusterNames, clusterName) + } + sort.Strings(clusterNames) + for _, clusterName := range clusterNames { + if states[clusterName] == unvisited { + if err := visit(clusterName); err != nil { + return err + } + } } return nil } diff --git a/pkg/http/api.go b/pkg/http/api.go index d0bd34d1..0983f2b3 100644 --- a/pkg/http/api.go +++ b/pkg/http/api.go @@ -366,7 +366,7 @@ type discardWriter struct{ io.Writer } func (discardWriter) Header() http.Header { return make(http.Header) } func (discardWriter) WriteHeader(int) {} -// Metrics exports process metrics, excluding aggregate and probe metrics on followers. +// Metrics exports process metrics, excluding leader-only metrics on followers. // Filtering happens here because published expvar values cannot be removed. func (api *APIImpl) Metrics(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { metricsHandler.ServeHTTP(discardWriter{Writer: io.Discard}, r) diff --git a/pkg/http/api_test.go b/pkg/http/api_test.go index 40eb7c63..c769a69d 100644 --- a/pkg/http/api_test.go +++ b/pkg/http/api_test.go @@ -50,6 +50,7 @@ func TestRoutes(t *testing.T) { {http.MethodGet, "/lb-check", http.StatusOK}, {http.MethodGet, "/config/memcache", http.StatusOK}, {http.MethodGet, "/debug/metrics", http.StatusOK}, + {http.MethodGet, "/explain/mysql/source", http.StatusNotFound}, } for _, route := range expectedRoutes { r, _ := http.NewRequest(route.verb, route.path, nil) diff --git a/pkg/throttle/api_dependencies_test.go b/pkg/throttle/api_dependencies_test.go new file mode 100644 index 00000000..f04ccf77 --- /dev/null +++ b/pkg/throttle/api_dependencies_test.go @@ -0,0 +1,140 @@ +package throttle_test + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/github/freno/pkg/base" + "github.com/github/freno/pkg/config" + frenohttp "github.com/github/freno/pkg/http" + "github.com/github/freno/pkg/throttle" +) + +func requestHTTPCheck(t *testing.T, server *httptest.Server) (int, throttle.CheckResult) { + t.Helper() + response, err := server.Client().Get(server.URL + "/check/test-app/mysql/replicas") + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var result throttle.CheckResult + if err := json.NewDecoder(response.Body).Decode(&result); err != nil { + t.Fatal(err) + } + if result.StatusCode != response.StatusCode { + t.Fatalf("HTTP status %d differs from response status %d", response.StatusCode, result.StatusCode) + } + return response.StatusCode, result +} + +func TestHTTPMySQLPrimaryDependencies(t *testing.T) { + for _, test := range []struct { + name string + dependencyField string + wantStatus int + wantMetric string + wantValue float64 + wantThreshold float64 + }{ + {"required primary blocks", "RequiredClusters", http.StatusTooManyRequests, "mysql/primary", 60, 50}, + {"removed shadow field is ignored", "ShadowRequiredClusters", http.StatusOK, "mysql/replicas", 1, 5}, + } { + t.Run(test.name, func(t *testing.T) { + defer config.Reset() + directory := t.TempDir() + contents := fmt.Sprintf(`{ + "RaftDataDir": %q, + "Stores": {"MySQL": {"Clusters": { + "replicas": {"ThrottleThreshold": 5, %q: ["primary"]}, + "primary": {"ThrottleThreshold": 50} + }}} + }`, directory, test.dependencyField) + filename := filepath.Join(directory, "freno.json") + if err := os.WriteFile(filename, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + if err := config.Instance().Read(filename); err != nil { + t.Fatal(err) + } + + throttler := throttle.NewThrottler() + throttle.SetTestMySQLClusterMetric(throttler, "replicas", 1, 5) + throttle.SetTestMySQLClusterMetric(throttler, "primary", 60, 50) + router := frenohttp.ConfigureRoutes(frenohttp.NewAPIImpl(throttle.NewThrottlerCheck(throttler), nil)) + server := httptest.NewServer(router) + defer server.Close() + + status, result := requestHTTPCheck(t, server) + if status != test.wantStatus || result.MetricName != test.wantMetric || + result.Value != test.wantValue || result.Threshold != test.wantThreshold { + t.Fatalf("unexpected HTTP check: %+v", result) + } + t.Logf("%s: HTTP %d, metric=%s, value=%g, threshold=%g", + test.dependencyField, status, result.MetricName, result.Value, result.Threshold) + }) + } +} + +func TestHTTPMySQLPrimaryRecoveryAfterProbeError(t *testing.T) { + defer config.Reset() + recoveryThreshold := 40.0 + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "replicas": {ThrottleThreshold: 5, RequiredClusters: []string{"primary"}}, + "primary": { + ThrottleThreshold: 50, + RecoveryThreshold: &recoveryThreshold, + RecoveryDurationMillis: 5000, + FailOnNoHosts: true, + }, + }, + } + throttler := throttle.NewThrottler() + throttle.SetTestMySQLClusterMetric(throttler, "replicas", 1, 5) + throttle.SetTestMySQLClusterMetric(throttler, "primary", 60, 50) + router := frenohttp.ConfigureRoutes(frenohttp.NewAPIImpl(throttle.NewThrottlerCheck(throttler), nil)) + server := httptest.NewServer(router) + defer server.Close() + + start := time.Date(2026, time.November, 16, 0, 0, 0, 0, time.UTC) + probeError := errors.New("test probe failed") + for _, step := range []struct { + name string + elapsed time.Duration + metric base.MetricResult + wantStatus int + wantMessage string + }{ + {"overloaded", 0, base.NewSimpleMetricResult(60), http.StatusTooManyRequests, base.ThresholdExceededError.Error()}, + {"healthy window starts", time.Second, base.NewSimpleMetricResult(35), http.StatusTooManyRequests, base.RecoveryNotCompleteError.Error()}, + {"healthy window progresses", 3 * time.Second, base.NewSimpleMetricResult(35), http.StatusTooManyRequests, base.RecoveryNotCompleteError.Error()}, + {"probe fails", 4 * time.Second, base.NewErrorMetricResult(0, probeError), http.StatusInternalServerError, probeError.Error()}, + {"fresh healthy window starts", 5 * time.Second, base.NewSimpleMetricResult(35), http.StatusTooManyRequests, base.RecoveryNotCompleteError.Error()}, + {"original deadline is not sufficient", 6 * time.Second, base.NewSimpleMetricResult(35), http.StatusTooManyRequests, base.RecoveryNotCompleteError.Error()}, + {"fresh window not yet complete", 10*time.Second - time.Millisecond, base.NewSimpleMetricResult(35), http.StatusTooManyRequests, base.RecoveryNotCompleteError.Error()}, + {"fresh full window complete", 10 * time.Second, base.NewSimpleMetricResult(35), http.StatusOK, ""}, + } { + t.Run(step.name, func(t *testing.T) { + throttle.StabilizeTestMySQLClusterMetric(throttler, "primary", step.metric, start.Add(step.elapsed)) + status, result := requestHTTPCheck(t, server) + if status != step.wantStatus || result.Message != step.wantMessage { + t.Fatalf("at %s: unexpected HTTP check: %+v", step.elapsed, result) + } + wantMetric := "mysql/primary" + if status == http.StatusOK { + wantMetric = "mysql/replicas" + } + if result.MetricName != wantMetric { + t.Fatalf("metric = %q, want %q", result.MetricName, wantMetric) + } + t.Logf("at %s: HTTP %d, metric=%s, message=%q", step.elapsed, status, result.MetricName, result.Message) + }) + } +} diff --git a/pkg/throttle/check.go b/pkg/throttle/check.go index 0c1cc7b8..13b31e88 100644 --- a/pkg/throttle/check.go +++ b/pkg/throttle/check.go @@ -54,34 +54,26 @@ func (check *ThrottlerCheck) checkAppMetricResult(appName string, storeType stri threshold = flags.OverrideThreshold } value, err := metricResult.Get() + if flags.ReadCheck && flags.OverrideThreshold > 0 && errors.Is(err, base.RecoveryNotCompleteError) && value <= threshold { + err = nil + } if appName == "" { return NewCheckResult(http.StatusExpectationFailed, value, threshold, fmt.Errorf("no app indicated")) } - statusCode := http.StatusInternalServerError // 500 - - if err == base.AppDeniedError { - // app specifically not allowed to get metrics - statusCode = http.StatusExpectationFailed // 417 - } else if err == base.NoSuchMetricError { - // not collected yet, or metric does not exist - statusCode = http.StatusNotFound // 404 - } else if errors.Is(err, base.NoHostsError) { - // If no hosts we report 0 for the metric. - statusCode = http.StatusOK // 200 - } else if err != nil { - // any error - statusCode = http.StatusInternalServerError // 500 - } else if value > threshold { - // casual throttling - statusCode = http.StatusTooManyRequests // 429 - err = base.ThresholdExceededError - + failOnNoHosts := false + if storeType == "mysql" { + if clusterSettings, ok := config.Settings().Stores.MySQL.Clusters[storeName]; ok { + failOnNoHosts = clusterSettings.FailOnNoHosts + } + } + checkResult = newMetricCheckResult(value, threshold, err, failOnNoHosts) + if err == nil && value > threshold { if !flags.LowPriority && !flags.ReadCheck && appName != frenoAppName { // low priority requests will henceforth be denied go check.throttler.nonLowPriorityAppRequestsThrottled.SetDefault(metricName, true) } - } else if appName != frenoAppName && check.throttler.getShareDomainSecondsSinceHealth(metricName) >= 1 { + } else if err == nil && checkResult.StatusCode == http.StatusOK && appName != frenoAppName && check.throttler.getShareDomainSecondsSinceHealth(metricName) >= 1 { // throttling based on shared domain metric. // we exclude the "freno" app itself, or else this could turn into a snowball: this service ("a") seeing // another service ("b") as unhealthy, itself becoming unhealthy, makind b's read into a's state as unheathly, @@ -89,13 +81,51 @@ func (check *ThrottlerCheck) checkAppMetricResult(appName string, storeType stri // The "freno" app is the one to generate those health metrics. It therefore must not participate the // shared-domain dependency check., - statusCode = http.StatusTooManyRequests // 429 - err = base.ThresholdExceededError - } else { - // all good! - statusCode = http.StatusOK // 200 + checkResult = NewCheckResult(http.StatusTooManyRequests, value, threshold, base.ThresholdExceededError) } - return NewCheckResult(statusCode, value, threshold, err) + checkResult.MetricName = metricName + return checkResult +} + +func (check *ThrottlerCheck) checkMySQLCluster( + appName string, + clusterName string, + flags *CheckFlags, + checkedClusters map[string]bool, +) *CheckResult { + if checkedClusters[clusterName] { + return nil + } + checkedClusters[clusterName] = true + + metricResultFunc := func() (metricResult base.MetricResult, threshold float64) { + return check.throttler.getMySQLClusterMetrics(clusterName) + } + rootResult := check.checkAppMetricResult(appName, "mysql", clusterName, metricResultFunc, flags) + if rootResult.StatusCode != http.StatusOK { + return rootResult + } + + clusterSettings := config.Settings().Stores.MySQL.Clusters[clusterName] + for _, requiredCluster := range clusterSettings.RequiredClusters { + requiredFlags := *flags + requiredFlags.ReadCheck = false + requiredFlags.OverrideThreshold = 0 + requiredResult := check.checkMySQLCluster(appName, requiredCluster, &requiredFlags, checkedClusters) + if requiredResult == nil || requiredResult.StatusCode == http.StatusOK { + continue + } + if requiredResult.StatusCode == http.StatusNotFound { + checkResult := NewErrorCheckResult( + http.StatusInternalServerError, + fmt.Errorf("required MySQL cluster metric %q is unavailable", requiredCluster), + ) + checkResult.MetricName = fmt.Sprintf("mysql/%s", requiredCluster) + return checkResult + } + return requiredResult + } + return rootResult } // CheckAppStoreMetric @@ -114,9 +144,7 @@ func (check *ThrottlerCheck) Check(appName string, storeType string, storeName s } } if configured { - metricResultFunc = func() (metricResult base.MetricResult, threshold float64) { - return check.throttler.getMySQLClusterMetrics(storeName) - } + checkResult = check.checkMySQLCluster(appName, storeName, flags, make(map[string]bool)) } else { metricResultFunc = func() (metricResult base.MetricResult, threshold float64) { return base.NoSuchMetric, 0 @@ -125,10 +153,24 @@ func (check *ThrottlerCheck) Check(appName string, storeType string, storeName s } } if metricResultFunc == nil { - return NoSuchMetricCheckResult + if checkResult == nil { + return NoSuchMetricCheckResult + } + } else { + checkResult = check.checkAppMetricResult(appName, storeType, storeName, metricResultFunc, flags) } - checkResult = check.checkAppMetricResult(appName, storeType, storeName, metricResultFunc, flags) + if checkResult.StatusCode == http.StatusTooManyRequests && appName != frenoAppName && appName != frenoShareDmainAppName { + reason := "other" + switch { + case errors.Is(checkResult.Error, base.RecoveryNotCompleteError): + reason = "recovery" + case errors.Is(checkResult.Error, base.ThresholdExceededError) && checkResult.Value > checkResult.Threshold: + reason = "threshold" + } + metricName := strings.ReplaceAll(checkResult.MetricName, "/", ".") + metrics.GetOrRegisterCounter(fmt.Sprintf("check.rejected.%s.%s.total", metricName, reason), nil).Inc(1) + } go func(statusCode int) { metrics.GetOrRegisterCounter("check.any.total", nil).Inc(1) diff --git a/pkg/throttle/check_result.go b/pkg/throttle/check_result.go index 1a288fca..a6730eb1 100644 --- a/pkg/throttle/check_result.go +++ b/pkg/throttle/check_result.go @@ -1,16 +1,40 @@ package throttle import ( + "errors" "net/http" "github.com/github/freno/pkg/base" ) +func newMetricCheckResult(value, threshold float64, err error, failOnNoHosts bool) *CheckResult { + status := http.StatusOK + switch { + case err == base.AppDeniedError: + status = http.StatusExpectationFailed + case err == base.NoSuchMetricError: + status = http.StatusNotFound + case errors.Is(err, base.NoHostsError): + if failOnNoHosts { + status = http.StatusInternalServerError + } + case errors.Is(err, base.RecoveryNotCompleteError): + status = http.StatusTooManyRequests + case err != nil: + status = http.StatusInternalServerError + case value > threshold: + status = http.StatusTooManyRequests + err = base.ThresholdExceededError + } + return NewCheckResult(status, value, threshold, err) +} + // CheckResult is the result for an app inquiring on a metric. It also exports as JSON via the API type CheckResult struct { StatusCode int `json:"StatusCode"` Value float64 `json:"Value"` Threshold float64 `json:"Threshold"` + MetricName string `json:"MetricName,omitempty"` Error error `json:"-"` Message string `json:"Message"` } diff --git a/pkg/throttle/dependencies_test.go b/pkg/throttle/dependencies_test.go new file mode 100644 index 00000000..10d8b9d1 --- /dev/null +++ b/pkg/throttle/dependencies_test.go @@ -0,0 +1,90 @@ +package throttle + +import ( + "bufio" + "fmt" + "io" + "net" + "strings" + "testing" + "time" + + "github.com/bradfitz/gomemcache/memcache" + "github.com/github/freno/pkg/base" + "github.com/github/freno/pkg/config" + "github.com/github/freno/pkg/mysql" + "github.com/stretchr/testify/assert" +) + +func TestRequiredMySQLMemcachePublication(t *testing.T) { + for _, enforce := range []bool{false, true} { + t.Run(fmt.Sprintf("enforced=%t", enforce), func(t *testing.T) { + original := config.Settings().Stores.MySQL + t.Cleanup(func() { config.Settings().Stores.MySQL = original }) + root := &config.MySQLClusterConfigurationSettings{ThrottleThreshold: 5} + if enforce { + root.RequiredClusters = []string{"primary"} + } + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "root": root, "primary": {ThrottleThreshold: 20, FailOnNoHosts: true}, + }, + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + type publication struct { + command string + err error + } + published := make(chan publication, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + published <- publication{err: err} + return + } + defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(2 * time.Second)); err != nil { + published <- publication{err: err} + return + } + reader := bufio.NewReader(conn) + line, err := reader.ReadString('\n') + if err == nil && strings.HasPrefix(line, "set ") { + _, err = reader.ReadString('\n') + } + if err == nil { + response := "STORED\r\n" + if strings.HasPrefix(line, "delete ") { + response = "DELETED\r\n" + } + _, err = io.WriteString(conn, response) + } + published <- publication{command: line, err: err} + }() + throttler := NewThrottler() + throttler.memcacheClient = memcache.New(listener.Addr().String()) + throttler.memcachePath = "dependencies-test" + throttler.isLeader = true + key := mysql.InstanceKey{Hostname: "root", Port: 3306} + probes := mysql.Probes{key: &mysql.Probe{Key: key}} + throttler.mysqlInventory.ClustersProbes["root"] = &probes + throttler.mysqlInventory.InstanceKeyMetrics[mysql.GetClusterInstanceKey("root", &key)] = base.NewSimpleMetricResult(1) + assert.NoError(t, throttler.aggregateMySQLMetrics()) + select { + case result := <-published: + assert.NoError(t, result.err) + want := "set dependencies-test/mysql/root " + if enforce { + want = "delete dependencies-test/mysql/root" + } + assert.True(t, strings.HasPrefix(result.command, want), result.command) + case <-time.After(3 * time.Second): + t.Fatal("timed out waiting for memcache publication") + } + }) + } +} diff --git a/pkg/throttle/throttler.go b/pkg/throttle/throttler.go index 5a6606fe..89778e4e 100644 --- a/pkg/throttle/throttler.go +++ b/pkg/throttle/throttler.go @@ -2,6 +2,7 @@ package throttle import ( "encoding/json" + "errors" "fmt" "io/ioutil" "math/rand" @@ -75,11 +76,18 @@ type Throttler struct { throttledAppsMutex sync.Mutex skippedHostsMutex sync.Mutex + mysqlRecoveryMutex sync.Mutex + mysqlRecoveryState map[string]*mysqlClusterRecoveryState nonLowPriorityAppRequestsThrottled *cache.Cache httpClient *http.Client } +type mysqlClusterRecoveryState struct { + throttled bool + healthySince time.Time +} + func NewThrottler() *Throttler { throttler := &Throttler{ isLeader: false, @@ -98,6 +106,7 @@ func NewThrottler() *Throttler { recentApps: cache.New(recentAppsExpiration, time.Minute), metricsHealth: cache.New(cache.NoExpiration, 0), shareDomainMetricHealth: cache.New(5*sharedDomainCollectInterval, sharedDomainCollectInterval), + mysqlRecoveryState: make(map[string]*mysqlClusterRecoveryState), nonLowPriorityAppRequestsThrottled: cache.New(nonDeprioritizedAppMapExpiration, nonDeprioritizedAppMapInterval), @@ -146,7 +155,7 @@ func (throttler *Throttler) Operate() { case <-leaderCheckTick: { // sparse - throttler.isLeader = throttler.isLeaderFunc() + throttler.updateLeadership(throttler.isLeaderFunc()) } case <-mysqlCollectTick: { @@ -450,15 +459,24 @@ func (throttler *Throttler) aggregateMySQLMetrics() error { if !throttler.isLeader { return nil } + aggregatedMetrics := make(map[string]base.MetricResult, len(throttler.mysqlInventory.ClustersProbes)) for clusterName, probes := range throttler.mysqlInventory.ClustersProbes { - metricName := fmt.Sprintf("mysql/%s", clusterName) ignoreHostsCount := throttler.mysqlInventory.IgnoreHostsCount[clusterName] ignoreHostsThreshold := throttler.mysqlInventory.IgnoreHostsThreshold[clusterName] aggregatedMetric := aggregateMySQLProbes(probes, clusterName, throttler.mysqlInventory.InstanceKeyMetrics, throttler.mysqlInventory.ClusterInstanceHttpChecks, ignoreHostsCount, config.Settings().Stores.MySQL.IgnoreDialTcpErrors, ignoreHostsThreshold) + aggregatedMetrics[clusterName] = throttler.stabilizeMySQLMetric(clusterName, aggregatedMetric, time.Now()) + } + for clusterName, aggregatedMetric := range aggregatedMetrics { + metricName := fmt.Sprintf("mysql/%s", clusterName) go throttler.aggregatedMetrics.Set(metricName, aggregatedMetric, cache.DefaultExpiration) if throttler.memcacheClient != nil { - go func() { + requiredClustersHealthy := throttler.requiredMySQLClustersHealthy(clusterName, aggregatedMetrics, make(map[string]bool)) + go func(metricName string, aggregatedMetric base.MetricResult, requiredClustersHealthy bool) { memcacheKey := fmt.Sprintf("%s/%s", throttler.memcachePath, metricName) + if !requiredClustersHealthy { + throttler.memcacheClient.Delete(memcacheKey) + return + } value, err := aggregatedMetric.Get() if err != nil { throttler.memcacheClient.Delete(memcacheKey) @@ -467,12 +485,109 @@ func (throttler *Throttler) aggregateMySQLMetrics() error { entryVal := fmt.Sprintf("%d:%.6f", epochMillis, value) throttler.memcacheClient.Set(&memcache.Item{Key: memcacheKey, Value: []byte(entryVal), Expiration: 1}) } - }() + }(metricName, aggregatedMetric, requiredClustersHealthy) } } return nil } +func (throttler *Throttler) requiredMySQLClustersHealthy(clusterName string, aggregatedMetrics map[string]base.MetricResult, checkedClusters map[string]bool) bool { + if checkedClusters[clusterName] { + return true + } + checkedClusters[clusterName] = true + + clusterSettings, ok := config.Settings().Stores.MySQL.Clusters[clusterName] + if !ok { + return false + } + for _, requiredCluster := range clusterSettings.RequiredClusters { + requiredSettings, configured := config.Settings().Stores.MySQL.Clusters[requiredCluster] + requiredMetric, collected := aggregatedMetrics[requiredCluster] + if !configured || !collected { + return false + } + value, err := requiredMetric.Get() + if err != nil || value > requiredSettings.ThrottleThreshold { + return false + } + if !throttler.requiredMySQLClustersHealthy(requiredCluster, aggregatedMetrics, checkedClusters) { + return false + } + } + return true +} + +func (throttler *Throttler) updateLeadership(isLeader bool) { + if isLeader && !throttler.isLeader { + throttler.mysqlRecoveryMutex.Lock() + throttler.mysqlRecoveryState = make(map[string]*mysqlClusterRecoveryState) + throttler.mysqlRecoveryMutex.Unlock() + } + throttler.isLeader = isLeader +} + +func (throttler *Throttler) stabilizeMySQLMetric(clusterName string, metricResult base.MetricResult, now time.Time) base.MetricResult { + clusterSettings, ok := config.Settings().Stores.MySQL.Clusters[clusterName] + if !ok || clusterSettings.RecoveryDurationMillis <= 0 { + return metricResult + } + + value, err := metricResult.Get() + if errors.Is(err, base.NoHostsError) && !clusterSettings.FailOnNoHosts { + return metricResult + } + + throttler.mysqlRecoveryMutex.Lock() + defer throttler.mysqlRecoveryMutex.Unlock() + + state, ok := throttler.mysqlRecoveryState[clusterName] + if !ok { + // A new process or leader has no recovery history. Require a complete + // healthy window before admitting work rather than assuming the metric + // was healthy before this process began observing it. + state = &mysqlClusterRecoveryState{throttled: true} + throttler.mysqlRecoveryState[clusterName] = state + } + + if err != nil || value > clusterSettings.ThrottleThreshold { + state.throttled = true + state.healthySince = time.Time{} + metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(1) + return metricResult + } + if !state.throttled { + metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(0) + return metricResult + } + + recoveryThreshold := clusterSettings.ThrottleThreshold + if clusterSettings.RecoveryThreshold != nil { + recoveryThreshold = *clusterSettings.RecoveryThreshold + } + if value > recoveryThreshold { + state.healthySince = time.Time{} + metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(1) + return base.NewErrorMetricResult(value, base.RecoveryNotCompleteError) + } + if state.healthySince.IsZero() { + state.healthySince = now + metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(1) + return base.NewErrorMetricResult(value, base.RecoveryNotCompleteError) + } + + recoveryDuration := time.Duration(clusterSettings.RecoveryDurationMillis) * time.Millisecond + if now.Sub(state.healthySince) < recoveryDuration { + metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(1) + return base.NewErrorMetricResult(value, base.RecoveryNotCompleteError) + } + + state.throttled = false + state.healthySince = time.Time{} + metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(0) + return metricResult +} + func (throttler *Throttler) pushStatusToExpVar() { metrics.DefaultRegistry.Each(func(metricName string, _ interface{}) { if strings.HasPrefix(metricName, "throttled_states.") { diff --git a/pkg/throttle/throttler_test.go b/pkg/throttle/throttler_test.go index 9a4fa614..990231f5 100644 --- a/pkg/throttle/throttler_test.go +++ b/pkg/throttle/throttler_test.go @@ -8,6 +8,8 @@ import ( "github.com/github/freno/pkg/base" "github.com/github/freno/pkg/config" "github.com/github/freno/pkg/mysql" + "github.com/patrickmn/go-cache" + metrics "github.com/rcrowley/go-metrics" "github.com/stretchr/testify/assert" ) @@ -125,6 +127,34 @@ func Test_checkAppMetricResult(t *testing.T) { assert.EqualError(t, result.Error, base.NoHostsError.Error()) assert.Equal(t, http.StatusOK, result.StatusCode) }) + t.Run("no hosts fail closed", func(t *testing.T) { + originalSettings := config.Settings().Stores.MySQL + defer func() { + config.Settings().Stores.MySQL = originalSettings + }() + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "test-cluster": {FailOnNoHosts: true}, + }, + } + + check := NewThrottlerCheck(throttler) + metricResultFunc := func() (metricResult base.MetricResult, threshold float64) { + return base.NoHostsMetricResult, 10.0 + } + result := check.checkAppMetricResult("test-app", "mysql", "test-cluster", metricResultFunc, &CheckFlags{}) + assert.EqualError(t, result.Error, base.NoHostsError.Error()) + assert.Equal(t, http.StatusInternalServerError, result.StatusCode) + }) + t.Run("recovery period throttles", func(t *testing.T) { + check := NewThrottlerCheck(throttler) + metricResultFunc := func() (metricResult base.MetricResult, threshold float64) { + return base.NewErrorMetricResult(5.0, base.RecoveryNotCompleteError), 10.0 + } + result := check.checkAppMetricResult("test-app", "mysql", "test-cluster", metricResultFunc, &CheckFlags{}) + assert.EqualError(t, result.Error, base.RecoveryNotCompleteError.Error()) + assert.Equal(t, http.StatusTooManyRequests, result.StatusCode) + }) t.Run("low priority denied", func(t *testing.T) { check := NewThrottlerCheck(throttler) throttler.nonLowPriorityAppRequestsThrottled.SetDefault("mysql/test-cluster", true) @@ -147,6 +177,10 @@ func SetTestMySQLClusterMetric(throttler *Throttler, clusterName string, value, setTestMySQLClusterMetric(throttler, clusterName, value, threshold) } +func StabilizeTestMySQLClusterMetric(throttler *Throttler, clusterName string, metric base.MetricResult, now time.Time) { + throttler.aggregatedMetrics.SetDefault("mysql/"+clusterName, throttler.stabilizeMySQLMetric(clusterName, metric, now)) +} + func newTestMySQLFallbackCheck(fallbackCluster string) (*ThrottlerCheck, *Throttler) { config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ FallbackCluster: fallbackCluster, @@ -261,3 +295,316 @@ func TestCheckMySQLFallbackUsesCanonicalState(t *testing.T) { }) } } + +func TestCheckMySQLRequiredClusters(t *testing.T) { + originalSettings := config.Settings().Stores.MySQL + defer func() { + config.Settings().Stores.MySQL = originalSettings + }() + + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "replicas": { + RequiredClusters: []string{"primary", "proxysql"}, + }, + "primary": {}, + "proxysql": {}, + }, + } + + newCheck := func() (*ThrottlerCheck, *Throttler) { + throttler := NewThrottler() + setTestMySQLClusterMetric(throttler, "replicas", 1.0, 5.0) + setTestMySQLClusterMetric(throttler, "primary", 10.0, 50.0) + setTestMySQLClusterMetric(throttler, "proxysql", 20.0, 80.0) + return NewThrottlerCheck(throttler), throttler + } + + t.Run("all required clusters healthy", func(t *testing.T) { + check, _ := newCheck() + result := check.Check("transitions", "mysql", "replicas", "", &CheckFlags{}) + assert.Equal(t, http.StatusOK, result.StatusCode) + assert.Equal(t, 1.0, result.Value) + assert.Equal(t, 5.0, result.Threshold) + }) + + t.Run("unreferenced primary does not gate admission", func(t *testing.T) { + check, throttler := newCheck() + replicas := config.Settings().Stores.MySQL.Clusters["replicas"] + required := replicas.RequiredClusters + replicas.RequiredClusters = nil + defer func() { replicas.RequiredClusters = required }() + setTestMySQLClusterMetric(throttler, "primary", 60.0, 50.0) + result := check.Check("transitions", "mysql", "replicas", "", &CheckFlags{}) + assert.Equal(t, http.StatusOK, result.StatusCode) + assert.Equal(t, "mysql/replicas", result.MetricName) + }) + + t.Run("primary cluster blocks admission", func(t *testing.T) { + check, throttler := newCheck() + setTestMySQLClusterMetric(throttler, "primary", 60.0, 50.0) + result := check.Check("transitions", "mysql", "replicas", "", &CheckFlags{}) + assert.Equal(t, http.StatusTooManyRequests, result.StatusCode) + assert.Equal(t, 60.0, result.Value) + assert.Equal(t, 50.0, result.Threshold) + assert.Equal(t, "mysql/primary", result.MetricName) + }) + + t.Run("ProxySQL cluster blocks admission", func(t *testing.T) { + check, throttler := newCheck() + setTestMySQLClusterMetric(throttler, "proxysql", 90.0, 80.0) + result := check.Check("transitions", "mysql", "replicas", "", &CheckFlags{}) + assert.Equal(t, http.StatusTooManyRequests, result.StatusCode) + assert.Equal(t, 90.0, result.Value) + assert.Equal(t, 80.0, result.Threshold) + }) + + t.Run("read threshold override only applies to requested cluster", func(t *testing.T) { + check, throttler := newCheck() + setTestMySQLClusterMetric(throttler, "primary", 60.0, 50.0) + result := check.Check("transitions", "mysql", "replicas", "", &CheckFlags{ + ReadCheck: true, + OverrideThreshold: 100.0, + }) + assert.Equal(t, http.StatusTooManyRequests, result.StatusCode) + assert.Equal(t, 50.0, result.Threshold) + }) + + t.Run("read threshold override bypasses only the requested cluster recovery latch", func(t *testing.T) { + check, throttler := newCheck() + throttler.aggregatedMetrics.Set( + "mysql/replicas", + base.NewErrorMetricResult(45.0, base.RecoveryNotCompleteError), + cache.DefaultExpiration, + ) + result := check.Check("transitions", "mysql", "replicas", "", &CheckFlags{ + ReadCheck: true, + OverrideThreshold: 100.0, + }) + assert.Equal(t, http.StatusOK, result.StatusCode) + + throttler.aggregatedMetrics.Set( + "mysql/primary", + base.NewErrorMetricResult(40.0, base.RecoveryNotCompleteError), + cache.DefaultExpiration, + ) + result = check.Check("transitions", "mysql", "replicas", "", &CheckFlags{ + ReadCheck: true, + OverrideThreshold: 100.0, + }) + assert.Equal(t, http.StatusTooManyRequests, result.StatusCode) + assert.Equal(t, "mysql/primary", result.MetricName) + }) + + t.Run("missing required metric fails closed", func(t *testing.T) { + check, throttler := newCheck() + throttler.aggregatedMetrics.Delete("mysql/primary") + result := check.Check("transitions", "mysql", "replicas", "", &CheckFlags{OKIfNotExists: true}) + assert.Equal(t, http.StatusInternalServerError, result.StatusCode) + assert.EqualError(t, result.Error, `required MySQL cluster metric "primary" is unavailable`) + assert.Equal(t, "mysql/primary", result.MetricName) + }) +} + +func TestCheckRejectionMetrics(t *testing.T) { + originalSettings := config.Settings().Stores.MySQL + defer func() { config.Settings().Stores.MySQL = originalSettings }() + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + FallbackCluster: "rejection-replicas", + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "rejection-replicas": {RequiredClusters: []string{"rejection-primary"}}, + "rejection-primary": {}, + }, + } + + for _, test := range []struct { + name string + replicaValue float64 + primaryMetric base.MetricResult + appName string + requestedCluster string + sharedDomain bool + wantStatus int + wantCounter string + }{ + {name: "healthy", wantStatus: http.StatusOK}, + {name: "replica overload", replicaValue: 6, wantStatus: http.StatusTooManyRequests, wantCounter: "rejection-replicas.threshold"}, + {name: "primary overload", primaryMetric: base.NewSimpleMetricResult(60), wantStatus: http.StatusTooManyRequests, wantCounter: "rejection-primary.threshold"}, + {name: "both overloaded count only the first blocker", replicaValue: 6, primaryMetric: base.NewSimpleMetricResult(60), wantStatus: http.StatusTooManyRequests, wantCounter: "rejection-replicas.threshold"}, + {name: "primary recovery is not overload", primaryMetric: base.NewErrorMetricResult(35, base.RecoveryNotCompleteError), wantStatus: http.StatusTooManyRequests, wantCounter: "rejection-primary.recovery"}, + {name: "shared domain is not local overload", sharedDomain: true, wantStatus: http.StatusTooManyRequests, wantCounter: "rejection-primary.other"}, + {name: "missing required metric is not a 429", primaryMetric: base.NoSuchMetric, wantStatus: http.StatusInternalServerError}, + {name: "application policy denial is not a 429", appName: "abusing-app", wantStatus: http.StatusExpectationFailed}, + {name: "self checks are excluded", appName: frenoAppName, primaryMetric: base.NewSimpleMetricResult(60), wantStatus: http.StatusTooManyRequests}, + {name: "shared domain self checks are excluded", appName: frenoShareDmainAppName, primaryMetric: base.NewSimpleMetricResult(60), wantStatus: http.StatusTooManyRequests}, + {name: "fallback uses the actual blocking metric", requestedCluster: "unknown", primaryMetric: base.NewSimpleMetricResult(60), wantStatus: http.StatusTooManyRequests, wantCounter: "rejection-primary.threshold"}, + } { + t.Run(test.name, func(t *testing.T) { + for _, cluster := range []string{"rejection-replicas", "rejection-primary"} { + for _, reason := range []string{"threshold", "recovery", "other"} { + name := "check.rejected.mysql." + cluster + "." + reason + ".total" + metrics.DefaultRegistry.Unregister(name) + t.Cleanup(func() { metrics.DefaultRegistry.Unregister(name) }) + } + } + throttler := NewThrottler() + setTestMySQLClusterMetric(throttler, "rejection-replicas", test.replicaValue, 5) + setTestMySQLClusterMetric(throttler, "rejection-primary", 10, 50) + if test.primaryMetric != nil { + throttler.aggregatedMetrics.SetDefault("mysql/rejection-primary", test.primaryMetric) + } + if test.sharedDomain { + throttler.shareDomainMetricHealth.SetDefault("mysql/rejection-primary", &base.MetricHealth{SecondsSinceLastHealthy: 1}) + } + appName, cluster := test.appName, test.requestedCluster + if appName == "" { + appName = "test-app" + } + if cluster == "" { + cluster = "rejection-replicas" + } + check := NewThrottlerCheck(throttler) + for attempt := 0; attempt < 2; attempt++ { + result := check.Check(appName, "mysql", cluster, "", StandardCheckFlags) + assert.Equal(t, test.wantStatus, result.StatusCode) + } + for _, cluster := range []string{"rejection-replicas", "rejection-primary"} { + for _, reason := range []string{"threshold", "recovery", "other"} { + name := "check.rejected.mysql." + cluster + "." + reason + ".total" + var want int64 + if cluster+"."+reason == test.wantCounter { + want = 2 + } + assert.Equal(t, want, metrics.GetOrRegisterCounter(name, nil).Count(), name) + } + } + }) + } +} + +func TestStabilizeMySQLMetric(t *testing.T) { + originalSettings := config.Settings().Stores.MySQL + defer func() { + config.Settings().Stores.MySQL = originalSettings + }() + + recoveryThreshold := 40.0 + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "primary": { + ThrottleThreshold: 50.0, + RecoveryThreshold: &recoveryThreshold, + RecoveryDurationMillis: 5000, + }, + }, + } + + throttler := NewThrottler() + start := time.Date(2026, time.September, 23, 12, 0, 0, 0, time.UTC) + + assertMetric := func(t *testing.T, result base.MetricResult, wantValue float64, wantErr error) { + t.Helper() + value, err := result.Get() + assert.Equal(t, wantValue, value) + assert.Equal(t, wantErr, err) + } + + assertMetric(t, throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(60), start), 60, nil) + assertMetric(t, throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(45), start.Add(time.Second)), 45, base.RecoveryNotCompleteError) + assertMetric(t, throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(35), start.Add(2*time.Second)), 35, base.RecoveryNotCompleteError) + assertMetric(t, throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(35), start.Add(6*time.Second)), 35, base.RecoveryNotCompleteError) + assertMetric(t, throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(35), start.Add(7*time.Second)), 35, nil) +} + +func TestStabilizeMySQLMetricResetsRecoveryWindow(t *testing.T) { + originalSettings := config.Settings().Stores.MySQL + defer func() { + config.Settings().Stores.MySQL = originalSettings + }() + + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "primary": { + ThrottleThreshold: 50.0, + RecoveryDurationMillis: 5000, + }, + }, + } + + throttler := NewThrottler() + start := time.Date(2026, time.September, 23, 12, 0, 0, 0, time.UTC) + + throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(60), start) + throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(40), start.Add(time.Second)) + throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(55), start.Add(3*time.Second)) + result := throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(40), start.Add(7*time.Second)) + _, err := result.Get() + assert.Equal(t, base.RecoveryNotCompleteError, err) +} + +func TestStabilizeMySQLMetricRequiresHealthyWindowAfterStartup(t *testing.T) { + originalSettings := config.Settings().Stores.MySQL + defer func() { + config.Settings().Stores.MySQL = originalSettings + }() + + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "primary": { + ThrottleThreshold: 50.0, + RecoveryDurationMillis: 5000, + }, + }, + } + + throttler := NewThrottler() + start := time.Date(2026, time.September, 23, 12, 0, 0, 0, time.UTC) + + result := throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(30), start) + _, err := result.Get() + assert.Equal(t, base.RecoveryNotCompleteError, err) + + result = throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(30), start.Add(5*time.Second)) + _, err = result.Get() + assert.Nil(t, err) + assert.Equal(t, int64(0), metrics.Get("recovery.mysql.primary.active").(metrics.Gauge).Value()) +} + +func TestUpdateLeadershipResetsRecoveryStateOnRegain(t *testing.T) { + throttler := NewThrottler() + throttler.isLeader = true + throttler.mysqlRecoveryState["primary"] = &mysqlClusterRecoveryState{throttled: false} + + throttler.updateLeadership(false) + assert.Len(t, throttler.mysqlRecoveryState, 1) + + throttler.updateLeadership(true) + assert.Empty(t, throttler.mysqlRecoveryState) +} + +func TestRequiredMySQLClustersHealthyForMemcache(t *testing.T) { + originalSettings := config.Settings().Stores.MySQL + defer func() { + config.Settings().Stores.MySQL = originalSettings + }() + + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "replicas": {RequiredClusters: []string{"primary"}}, + "primary": {ThrottleThreshold: 50.0}, + }, + } + throttler := NewThrottler() + aggregatedMetrics := map[string]base.MetricResult{ + "replicas": base.NewSimpleMetricResult(1.0), + "primary": base.NewSimpleMetricResult(40.0), + } + + assert.True(t, throttler.requiredMySQLClustersHealthy("replicas", aggregatedMetrics, make(map[string]bool))) + + aggregatedMetrics["primary"] = base.NewSimpleMetricResult(60.0) + assert.False(t, throttler.requiredMySQLClustersHealthy("replicas", aggregatedMetrics, make(map[string]bool))) + + delete(aggregatedMetrics, "primary") + assert.False(t, throttler.requiredMySQLClustersHealthy("replicas", aggregatedMetrics, make(map[string]bool))) +} diff --git a/pkg/vitess/api_client_test.go b/pkg/vitess/api_client_test.go index 97a753d2..b713780f 100644 --- a/pkg/vitess/api_client_test.go +++ b/pkg/vitess/api_client_test.go @@ -11,6 +11,18 @@ import ( "vitess.io/vitess/go/vt/proto/topodata" ) +func TestTabletIsValidReplica(t *testing.T) { + replica := Tablet{Type: topodata.TabletType_REPLICA} + primary := Tablet{Type: topodata.TabletType_MASTER} + + if !replica.IsValidReplica() { + t.Fatal("expected replica tablet to remain valid") + } + if primary.IsValidReplica() { + t.Fatal("expected primary tablet not to pass replica validation") + } +} + func TestParseTablets(t *testing.T) { vitessApi := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.String() {