From 3cec506712f76d1a352785adf5d80d3e2c511ef4 Mon Sep 17 00:00:00 2001 From: Pradeep Kintali Date: Wed, 18 Nov 2026 21:05:43 -0800 Subject: [PATCH 1/2] Support opt-in HAProxy backend address discovery Preserve default hostname-based discovery while allowing selected pools to use validated backend host and port values from the HAProxy CSV addr field. Carry backend ports through status filtering and probe inventory creation, with regression tests and configuration documentation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fbc83df2-d1af-406f-b97b-51cf0aaa570d --- doc/mysql.md | 19 +++ pkg/config/haproxy_config.go | 9 +- pkg/haproxy/host.go | 1 + pkg/haproxy/parser.go | 47 +++++- pkg/haproxy/server_address_test.go | 89 ++++++++++++ pkg/haproxy/throttle.go | 10 +- pkg/throttle/haproxy_inventory_test.go | 189 +++++++++++++++++++++++++ pkg/throttle/throttler.go | 15 +- 8 files changed, 369 insertions(+), 10 deletions(-) create mode 100644 pkg/haproxy/server_address_test.go create mode 100644 pkg/throttle/haproxy_inventory_test.go diff --git a/doc/mysql.md b/doc/mysql.md index 6535869a..e5bafa73 100644 --- a/doc/mysql.md +++ b/doc/mysql.md @@ -125,6 +125,25 @@ Noteworthy: - `sharded` list of servers is read from Vitess vtctld's HTTP `API`. To read it with vtctld's gRPC `GetTablets` instead (Vitess v23 removes the HTTP API), set `"GRPCAddress": "vtctld.example.com:15999"` and `"GRPCInsecure": true`. Only plaintext gRPC is supported. +### HAProxy server addresses + +By default, HAProxy discovery uses each server's `svname` as its hostname and the +cluster's configured MySQL port. If server labels are aliases rather than +resolvable hostnames, set `"UseServerAddress": true` inside that cluster's +`HAProxySettings` to use the CSV `addr` host and port instead. For example, a server +labelled `primary` with `addr` `192.0.2.10:3307` is probed at `192.0.2.10:3307`. +Bracketed IPv6 addresses such as `[2001:db8::10]:3307` are supported. + +This setting defaults to `false` and does not change other clusters. When enabled, +`IgnoreHosts` and skipped-host checks apply to the discovered address, not the +server label. + +The CSV must contain an `addr` column and a valid host and numeric port (1-65535) +for every backend server row in the selected pool. Invalid or missing addresses +cause a discovery error, without falling back to the label. As with other HAProxy +discovery failures, an unsuccessful refresh can leave the previous inventory in +place. + ### Non lag metrics `freno` isn't necessarily about replication lag. You may choose to use different thresholds appropriate for your setup and workload. For example, you may choose to monitor the master (as opposed of the replicas) and read some metric such as `threads_running`. An example configuration would be: diff --git a/pkg/config/haproxy_config.go b/pkg/config/haproxy_config.go index e1ad89fc..fe1e16d9 100644 --- a/pkg/config/haproxy_config.go +++ b/pkg/config/haproxy_config.go @@ -49,10 +49,11 @@ func ParseHostPort(address string) (hostPort *HostPort, err error) { } type HAProxyConfigurationSettings struct { - Host string - Port int - Addresses string - PoolName string + Host string + Port int + Addresses string + PoolName string + UseServerAddress bool } func parseAddress(address string) (u *url.URL, err error) { diff --git a/pkg/haproxy/host.go b/pkg/haproxy/host.go index 3e4f47f7..c37c7971 100644 --- a/pkg/haproxy/host.go +++ b/pkg/haproxy/host.go @@ -27,6 +27,7 @@ func ToBackendHostStatus(status string) BackendHostStatus { type BackendHost struct { Hostname string + Port int Status BackendHostStatus IsTransitioning bool } diff --git a/pkg/haproxy/parser.go b/pkg/haproxy/parser.go index 7f07993c..93b3dd1c 100644 --- a/pkg/haproxy/parser.go +++ b/pkg/haproxy/parser.go @@ -3,8 +3,10 @@ package haproxy import ( "fmt" "io/ioutil" + "net" "net/http" "net/url" + "strconv" "strings" "time" @@ -72,6 +74,10 @@ func ParseStatus(fullStatus string) (status BackendHostStatus, isTransitioning b // Returned are all non-disabled hosts in given backend. Thus, a NOLB is skipped; any UP or DOWN hosts are returned. // Such list indicates the hosts which can be expected to be active, which is then the list freno will probe. func ParseHosts(csvLines []string, poolName string) (backendHosts [](*BackendHost), err error) { + return parseHosts(csvLines, poolName, false) +} + +func parseHosts(csvLines []string, poolName string, useServerAddress bool) (backendHosts []*BackendHost, err error) { if len(csvLines) < 1 { return backendHosts, HAProxyEmptyStatus } @@ -79,6 +85,7 @@ func ParseHosts(csvLines []string, poolName string) (backendHosts [](*BackendHos return backendHosts, HAProxyPartialStatus } var tokensMap map[string]int + requiredColumns := 0 poolFound := false countHosts := 0 countUpHosts := 0 @@ -87,9 +94,26 @@ func ParseHosts(csvLines []string, poolName string) (backendHosts [](*BackendHos for i, line := range csvLines { if i == 0 { tokensMap = parseHeader(csvLines[0]) + if useServerAddress { + for _, field := range []string{"pxname", "svname", "status", "addr"} { + index, ok := tokensMap[field] + if !ok { + return nil, fmt.Errorf("HAProxy CSV parsing: missing %s column", field) + } + if index+1 > requiredColumns { + requiredColumns = index + 1 + } + } + } + continue + } + if strings.TrimSpace(line) == "" { continue } tokens := strings.Split(line, ",") + if useServerAddress && len(tokens) < requiredColumns { + return nil, fmt.Errorf("HAProxy CSV parsing: incomplete row %d", i+1) + } if tokens[tokensMap["pxname"]] == poolName { poolFound = true if host := tokens[tokensMap["svname"]]; host != "BACKEND" && host != "FRONTEND" { @@ -97,7 +121,22 @@ func ParseHosts(csvLines []string, poolName string) (backendHosts [](*BackendHos status, isTransitioning := ParseStatus(tokens[tokensMap["status"]]) - backendHosts = append(backendHosts, NewBackendHost(host, status, isTransitioning)) + backendHost := NewBackendHost(host, status, isTransitioning) + if useServerAddress { + address := tokens[tokensMap["addr"]] + hostname, portText, err := net.SplitHostPort(address) + ip := net.ParseIP(hostname) + if err != nil || hostname == "" || strings.ContainsAny(hostname, " \t\r\n") || (ip != nil && ip.IsUnspecified()) { + return nil, fmt.Errorf("HAProxy CSV parsing: invalid addr %q for %s/%s", address, poolName, host) + } + port, err := strconv.Atoi(portText) + if err != nil || port <= 0 || port > 65535 { + return nil, fmt.Errorf("HAProxy CSV parsing: invalid port in addr %q for %s/%s", address, poolName, host) + } + backendHost.Hostname = hostname + backendHost.Port = port + } + backendHosts = append(backendHosts, backendHost) if isTransitioning { countTransitioningHosts++ } @@ -129,6 +168,12 @@ func ParseCsvHosts(csv string, poolName string) (backendHosts [](*BackendHost), return ParseHosts(csvLines, poolName) } +// ParseCsvHostsWithServerAddresses uses each server's addr instead of its svname label. +// Each backend server row must have a valid host:port address. +func ParseCsvHostsWithServerAddresses(csv string, poolName string) ([]*BackendHost, error) { + return parseHosts(parseLines(csv), poolName, true) +} + func toCSVUrl(u url.URL) *url.URL { u.Path = fmt.Sprintf("%s;csv;norefresh", u.Path) return &u diff --git a/pkg/haproxy/server_address_test.go b/pkg/haproxy/server_address_test.go new file mode 100644 index 00000000..76ba4944 --- /dev/null +++ b/pkg/haproxy/server_address_test.go @@ -0,0 +1,89 @@ +package haproxy + +import ( + "fmt" + "reflect" + "testing" +) + +func TestParseCsvHostsWithServerAddresses(t *testing.T) { + for _, tt := range []struct { + address string + hostname string + port int + }{ + {"192.0.2.10:3307", "192.0.2.10", 3307}, + {"[2001:db8::10]:3308", "2001:db8::10", 3308}, + {"db.example.com:3309", "db.example.com", 3309}, + } { + t.Run(tt.address, func(t *testing.T) { + csv := "# pxname,svname,status,addr\n" + + fmt.Sprintf("writers,primary,no check,%s\n", tt.address) + + "writers,BACKEND,UP,\nwriters,FRONTEND,OPEN,\n" + hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers") + if err != nil { + t.Fatal(err) + } + if len(hosts) != 1 || hosts[0].Hostname != tt.hostname || hosts[0].Port != tt.port { + t.Fatalf("hosts = %+v, want %s port %d", hosts, tt.hostname, tt.port) + } + }) + } +} + +func TestParseCsvHostsWithServerAddressesRejectsInvalidAddresses(t *testing.T) { + for _, address := range []string{ + "", "192.0.2.10", ":3306", "192.0.2.10:", "192.0.2.10:mysql", + "192.0.2.10:0", "192.0.2.10:-1", "192.0.2.10:65536", + "2001:db8::10:3306", "bad host:3306", + "0.0.0.0:3306", "[::]:3306", + } { + t.Run(address, func(t *testing.T) { + csv := "# pxname,svname,status,addr\n" + + "writers,healthy,UP,192.0.2.11:3306\n" + + fmt.Sprintf("writers,primary,no check,%s\n", address) + hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers") + if err == nil || len(hosts) != 0 { + t.Fatalf("hosts = %+v, err = %v; want an error without a partial roster", hosts, err) + } + }) + } + for _, csv := range []string{ + "# pxname,svname,status\nwriters,primary,no check\n", + "# pxname,svname,status,addr\nwriters,primary,no check\n", + "# pxname,svname,status,addr\nwriters,primary\n", + "# pxname,svname,addr\nwriters,primary,192.0.2.10:3306\n", + } { + if hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers"); err == nil || len(hosts) != 0 { + t.Errorf("CSV %q: hosts = %+v, err = %v; want a schema error", csv, hosts, err) + } + } +} + +func TestServerAddressStatusAndTransitionFiltering(t *testing.T) { + csv := "# pxname,svname,status,addr\n" + + "writers,primary,no check,192.0.2.10:3307\n" + + "writers,down,DOWN,192.0.2.11:3308\n" + + "writers,up,UP,192.0.2.12:3309\n" + + "writers,disabled,NOLB,192.0.2.13:3310\n" + + "writers,transitioning,UP 1/2,192.0.2.14:3311\n" + hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers") + if err != nil { + t.Fatal(err) + } + got := FilterThrottlerBackendHosts(hosts) + want := []*BackendHost{ + {Hostname: "192.0.2.10", Port: 3307, Status: StatusNoCheck}, + {Hostname: "192.0.2.11", Port: 3308, Status: StatusDown}, + {Hostname: "192.0.2.12", Port: 3309, Status: StatusUp}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("filtered hosts = %+v, want %+v", got, want) + } + for _, status := range []string{"UP 1/2", "DOWN 1/2"} { + csv := fmt.Sprintf("# pxname,svname,status,addr\nwriters,primary,%s,192.0.2.10:3307\n", status) + if _, err := ParseCsvHostsWithServerAddresses(csv, "writers"); err == nil { + t.Errorf("all hosts transitioning (%s): expected an error", status) + } + } +} diff --git a/pkg/haproxy/throttle.go b/pkg/haproxy/throttle.go index ae97d8ec..fc8228f4 100644 --- a/pkg/haproxy/throttle.go +++ b/pkg/haproxy/throttle.go @@ -1,6 +1,14 @@ package haproxy func FilterThrotllerHosts(backendHosts [](*BackendHost)) (hosts []string) { + for _, backendHost := range FilterThrottlerBackendHosts(backendHosts) { + hosts = append(hosts, backendHost.Hostname) + } + return hosts +} + +// FilterThrottlerBackendHosts retains eligible servers, including their discovered ports. +func FilterThrottlerBackendHosts(backendHosts []*BackendHost) (hosts []*BackendHost) { for _, backendHost := range backendHosts { hostIsRelevant := false switch backendHost.Status { @@ -14,7 +22,7 @@ func FilterThrotllerHosts(backendHosts [](*BackendHost)) (hosts []string) { hostIsRelevant = true } if hostIsRelevant { - hosts = append(hosts, backendHost.Hostname) + hosts = append(hosts, backendHost) } } return hosts diff --git a/pkg/throttle/haproxy_inventory_test.go b/pkg/throttle/haproxy_inventory_test.go new file mode 100644 index 00000000..7d05e3b3 --- /dev/null +++ b/pkg/throttle/haproxy_inventory_test.go @@ -0,0 +1,189 @@ +package throttle + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/github/freno/pkg/config" + "github.com/github/freno/pkg/mysql" +) + +func TestRefreshMySQLInventoryHAProxyServerAddress(t *testing.T) { + originalSettings := config.Settings().Stores.MySQL + defer func() { config.Settings().Stores.MySQL = originalSettings }() + + tests := []struct { + name string + options string + csv string + ignoreHosts []string + skippedHost string + want map[string][]mysql.InstanceKey + }{ + { + name: "default preserves hostname identity and exclusions", + options: `{}`, + csv: "# pxname,svname,status,addr\n" + + "writer_a,db-a.example.com,no check,192.0.2.10:3307\n" + + "writer_a,ignore-me.example.com,UP,192.0.2.11:3308\n", + ignoreHosts: []string{"ignore-me"}, + want: map[string][]mysql.InstanceKey{ + "writer_a": {{Hostname: "db-a.example.com", Port: 3316}}, + }, + }, + { + name: "disabled supports CSV without addr", + options: `{"UseServerAddress":false}`, + csv: "# pxname,svname,status\nwriter_a,db-a.example.com,no check\n", + want: map[string][]mysql.InstanceKey{ + "writer_a": {{Hostname: "db-a.example.com", Port: 3316}}, + }, + }, + { + name: "opt in keeps identical labels in different pools distinct", + options: `{"UseServerAddress":true}`, + csv: "# pxname,svname,status,addr\n" + + "writer_a,primary,no check,192.0.2.10:3307\n" + + "writer_b,primary,no check,192.0.2.11:3308\n", + want: map[string][]mysql.InstanceKey{ + "writer_a": {{Hostname: "192.0.2.10", Port: 3307}}, + "writer_b": {{Hostname: "192.0.2.11", Port: 3308}}, + }, + }, + { + name: "opt in supports IPv6", + options: `{"UseServerAddress":true}`, + csv: "# pxname,svname,status,addr\nwriter_a,primary,no check,[2001:db8::10]:3309\n", + want: map[string][]mysql.InstanceKey{ + "writer_a": {{Hostname: "2001:db8::10", Port: 3309}}, + }, + }, + { + name: "opt in applies exclusions to discovered addresses", + options: `{"UseServerAddress":true}`, + csv: "# pxname,svname,status,addr\n" + + "writer_a,primary,no check,192.0.2.10:3307\n" + + "writer_a,secondary,UP,192.0.2.11:3308\n", + ignoreHosts: []string{"192.0.2.10"}, + skippedHost: "192.0.2.11", + want: map[string][]mysql.InstanceKey{"writer_a": {}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, tt.csv) + })) + defer server.Close() + + settings := config.MySQLConfigurationSettings{ + Clusters: make(map[string]*config.MySQLClusterConfigurationSettings), + } + for clusterName := range tt.want { + cluster := &config.MySQLClusterConfigurationSettings{ + Port: 3316, + IgnoreHosts: tt.ignoreHosts, + HAProxySettings: config.HAProxyConfigurationSettings{ + Addresses: server.URL, + PoolName: clusterName, + }, + } + if err := json.Unmarshal([]byte(tt.options), &cluster.HAProxySettings); err != nil { + t.Fatal(err) + } + settings.Clusters[clusterName] = cluster + } + config.Settings().Stores.MySQL = settings + throttler := NewThrottler() + throttler.isLeader = true + throttler.mysqlClusterProbesChan = make(chan *mysql.ClusterProbes, len(tt.want)) + if tt.skippedHost != "" { + throttler.skippedHosts.SetDefault(tt.skippedHost, true) + } + if err := throttler.refreshMySQLInventory(); err != nil { + t.Fatal(err) + } + seen := make(map[string]bool) + for range tt.want { + select { + case probes := <-throttler.mysqlClusterProbesChan: + want, ok := tt.want[probes.ClusterName] + if !ok || seen[probes.ClusterName] { + t.Fatalf("unexpected cluster probes: %s", probes.ClusterName) + } + seen[probes.ClusterName] = true + if got := len(*probes.InstanceProbes); got != len(want) { + t.Errorf("%s probe count = %d, want %d", probes.ClusterName, got, len(want)) + } + for _, key := range want { + if probe := (*probes.InstanceProbes)[key]; probe == nil || probe.Key != key { + t.Errorf("%s missing probe for %+v; got %+v", probes.ClusterName, key, *probes.InstanceProbes) + } + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for HAProxy probes") + } + } + }) + } +} + +func TestRefreshMySQLInventoryHAProxyServerAddressChanges(t *testing.T) { + originalSettings := config.Settings().Stores.MySQL + defer func() { config.Settings().Stores.MySQL = originalSettings }() + + var response atomic.Value + setAddress := func(address string) { + response.Store("# pxname,svname,status,addr\nwriters,primary,no check," + address + "\n") + } + setAddress("192.0.2.10:3307") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, response.Load().(string)) + })) + defer server.Close() + cluster := &config.MySQLClusterConfigurationSettings{ + Port: 3306, + HAProxySettings: config.HAProxyConfigurationSettings{ + Addresses: server.URL, + PoolName: "writers", + }, + } + if err := json.Unmarshal([]byte(`{"UseServerAddress":true}`), &cluster.HAProxySettings); err != nil { + t.Fatal(err) + } + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{"writers": cluster}, + } + throttler := NewThrottler() + throttler.isLeader = true + throttler.mysqlClusterProbesChan = make(chan *mysql.ClusterProbes, 1) + + refresh := func(want mysql.InstanceKey) { + t.Helper() + if err := throttler.refreshMySQLInventory(); err != nil { + t.Fatal(err) + } + select { + case probes := <-throttler.mysqlClusterProbesChan: + if err := throttler.updateMySQLClusterProbes(probes); err != nil { + t.Fatal(err) + } + got := throttler.mysqlInventory.ClustersProbes["writers"] + if got == nil || len(*got) != 1 || (*got)[want] == nil { + t.Fatalf("inventory = %+v, want only %+v", got, want) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for refreshed HAProxy probes") + } + } + refresh(mysql.InstanceKey{Hostname: "192.0.2.10", Port: 3307}) + setAddress("[2001:db8::11]:3308") + time.Sleep(1100 * time.Millisecond) // HAProxy responses are cached for one second. + refresh(mysql.InstanceKey{Hostname: "2001:db8::11", Port: 3308}) +} diff --git a/pkg/throttle/throttler.go b/pkg/throttle/throttler.go index ec5b6000..5a6606fe 100644 --- a/pkg/throttle/throttler.go +++ b/pkg/throttle/throttler.go @@ -314,7 +314,11 @@ func (throttler *Throttler) refreshMySQLInventory() error { throttler.mysqlClusterThresholds.Set(clusterName, clusterSettings.ThrottleThreshold, cache.DefaultExpiration) if !clusterSettings.HAProxySettings.IsEmpty() { poolName := clusterSettings.HAProxySettings.PoolName - totalHosts := []string{} + totalHosts := []*haproxy.BackendHost{} + parseCSVHosts := haproxy.ParseCsvHosts + if clusterSettings.HAProxySettings.UseServerAddress { + parseCSVHosts = haproxy.ParseCsvHostsWithServerAddresses + } addresses, _ := clusterSettings.HAProxySettings.GetProxyAddresses() for _, u := range addresses { log.Debugf("getting haproxy data from %s", u.String()) @@ -322,8 +326,8 @@ func (throttler *Throttler) refreshMySQLInventory() error { if err != nil { return log.Errorf("Unable to get HAproxy data from %s: %+v", u.String(), err) } - if backendHosts, err := haproxy.ParseCsvHosts(csv, poolName); err == nil { - hosts := haproxy.FilterThrotllerHosts(backendHosts) + if backendHosts, err := parseCSVHosts(csv, poolName); err == nil { + hosts := haproxy.FilterThrottlerBackendHosts(backendHosts) totalHosts = append(totalHosts, hosts...) log.Debugf("Read %+v hosts from haproxy %s/#%s", len(hosts), u.String(), poolName) } else { @@ -341,7 +345,10 @@ func (throttler *Throttler) refreshMySQLInventory() error { InstanceProbes: mysql.NewProbes(), } for _, host := range totalHosts { - key := mysql.InstanceKey{Hostname: host, Port: clusterSettings.Port} + key := mysql.InstanceKey{Hostname: host.Hostname, Port: clusterSettings.Port} + if host.Port > 0 { + key.Port = host.Port + } addInstanceKey(&key, clusterName, clusterSettings, clusterProbes.InstanceProbes) } throttler.mysqlClusterProbesChan <- clusterProbes From b814ae05973014dbadbc2182d499166793b011a2 Mon Sep 17 00:00:00 2001 From: Pradeep Kintali Date: Thu, 19 Nov 2026 13:33:14 -0800 Subject: [PATCH 2/2] Fix HAProxy CSV parsing and eligible address validation Decode quoted CSV records correctly and validate backend addresses only for servers eligible for probing. Preserve transition safeguards, add parser and inventory regressions, and document the HTTP stats address prerequisite. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fbc83df2-d1af-406f-b97b-51cf0aaa570d --- doc/mysql.md | 18 ++++-- pkg/haproxy/parser.go | 57 +++++++++------- pkg/haproxy/server_address_test.go | 90 ++++++++++++++++++++++++++ pkg/haproxy/throttle.go | 24 +++---- pkg/throttle/haproxy_inventory_test.go | 10 +++ 5 files changed, 160 insertions(+), 39 deletions(-) diff --git a/doc/mysql.md b/doc/mysql.md index e5bafa73..38d3519d 100644 --- a/doc/mysql.md +++ b/doc/mysql.md @@ -138,11 +138,21 @@ This setting defaults to `false` and does not change other clusters. When enable `IgnoreHosts` and skipped-host checks apply to the discovered address, not the server label. +For HTTP stats discovery, enable `stats show-legends` in the HAProxy stats +configuration so that the CSV includes server address values. Without this +setting, the `addr` column can be present but empty. Verify that eligible servers +have populated addresses before enabling `UseServerAddress`. + The CSV must contain an `addr` column and a valid host and numeric port (1-65535) -for every backend server row in the selected pool. Invalid or missing addresses -cause a discovery error, without falling back to the label. As with other HAProxy -discovery failures, an unsuccessful refresh can leave the previous inventory in -place. +for every eligible backend server in the selected pool. Address validation skips +servers excluded by the existing status rules, such as unresolved maintenance +slots, `NOLB` servers, and transitioning `UP` servers. `DOWN` and `no check` +servers remain eligible and require valid addresses. Existing safeguards for +all-transitioning servers are unchanged. + +Invalid or missing addresses on eligible servers cause a discovery error, +without falling back to the label. As with other HAProxy discovery failures, an +unsuccessful refresh can leave the previous inventory in place. ### Non lag metrics diff --git a/pkg/haproxy/parser.go b/pkg/haproxy/parser.go index 93b3dd1c..314c844f 100644 --- a/pkg/haproxy/parser.go +++ b/pkg/haproxy/parser.go @@ -1,7 +1,9 @@ package haproxy import ( + "encoding/csv" "fmt" + "io" "io/ioutil" "net" "net/http" @@ -84,35 +86,41 @@ func parseHosts(csvLines []string, poolName string, useServerAddress bool) (back if len(csvLines) == 1 { return backendHosts, HAProxyPartialStatus } - var tokensMap map[string]int + tokensMap := parseHeader(csvLines[0]) requiredColumns := 0 + for _, field := range []string{"pxname", "svname", "status"} { + index, ok := tokensMap[field] + if !ok { + return nil, fmt.Errorf("HAProxy CSV parsing: missing %s column", field) + } + if index+1 > requiredColumns { + requiredColumns = index + 1 + } + } + addressColumn, hasAddressColumn := tokensMap["addr"] + if useServerAddress && !hasAddressColumn { + return nil, fmt.Errorf("HAProxy CSV parsing: missing addr column") + } + reader := csv.NewReader(strings.NewReader(strings.Join(csvLines[1:], "\n"))) + reader.FieldsPerRecord = -1 poolFound := false countHosts := 0 countUpHosts := 0 countTransitioningHosts := 0 countTransitioningUpHosts := 0 - for i, line := range csvLines { - if i == 0 { - tokensMap = parseHeader(csvLines[0]) - if useServerAddress { - for _, field := range []string{"pxname", "svname", "status", "addr"} { - index, ok := tokensMap[field] - if !ok { - return nil, fmt.Errorf("HAProxy CSV parsing: missing %s column", field) - } - if index+1 > requiredColumns { - requiredColumns = index + 1 - } - } - } - continue + for { + tokens, err := reader.Read() + if err == io.EOF { + break } - if strings.TrimSpace(line) == "" { + if err != nil { + return nil, fmt.Errorf("HAProxy CSV parsing: %w", err) + } + if len(tokens) == 1 && strings.TrimSpace(tokens[0]) == "" { continue } - tokens := strings.Split(line, ",") - if useServerAddress && len(tokens) < requiredColumns { - return nil, fmt.Errorf("HAProxy CSV parsing: incomplete row %d", i+1) + if len(tokens) < requiredColumns { + return nil, fmt.Errorf("HAProxy CSV parsing: incomplete row") } if tokens[tokensMap["pxname"]] == poolName { poolFound = true @@ -122,8 +130,11 @@ func parseHosts(csvLines []string, poolName string, useServerAddress bool) (back status, isTransitioning := ParseStatus(tokens[tokensMap["status"]]) backendHost := NewBackendHost(host, status, isTransitioning) - if useServerAddress { - address := tokens[tokensMap["addr"]] + if useServerAddress && backendHost.isRelevantForThrottling() { + if len(tokens) <= addressColumn { + return nil, fmt.Errorf("HAProxy CSV parsing: missing addr for %s/%s", poolName, host) + } + address := tokens[addressColumn] hostname, portText, err := net.SplitHostPort(address) ip := net.ParseIP(hostname) if err != nil || hostname == "" || strings.ContainsAny(hostname, " \t\r\n") || (ip != nil && ip.IsUnspecified()) { @@ -169,7 +180,7 @@ func ParseCsvHosts(csv string, poolName string) (backendHosts [](*BackendHost), } // ParseCsvHostsWithServerAddresses uses each server's addr instead of its svname label. -// Each backend server row must have a valid host:port address. +// Each eligible backend server row must have a valid host:port address. func ParseCsvHostsWithServerAddresses(csv string, poolName string) ([]*BackendHost, error) { return parseHosts(parseLines(csv), poolName, true) } diff --git a/pkg/haproxy/server_address_test.go b/pkg/haproxy/server_address_test.go index 76ba4944..bca78bd3 100644 --- a/pkg/haproxy/server_address_test.go +++ b/pkg/haproxy/server_address_test.go @@ -1,11 +1,101 @@ package haproxy import ( + "errors" "fmt" "reflect" "testing" ) +func TestServerAddressQuotedCSVFields(t *testing.T) { + for _, report := range []string{ + `"check passed, response OK"`, + `"check passed, response ""OK"""`, + "\"check passed,\nresponse OK\"", + } { + t.Run(report, func(t *testing.T) { + csv := "# pxname,svname,status,last_chk,addr\nwriters,primary,no check," + report + ",192.0.2.10:3307\n" + hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers") + if err != nil { + t.Fatal(err) + } + got := FilterThrottlerBackendHosts(hosts) + want := []*BackendHost{{Hostname: "192.0.2.10", Port: 3307, Status: StatusNoCheck}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("hosts = %+v, want %+v", got, want) + } + legacy, err := ParseCsvHosts(csv, "writers") + if err != nil { + t.Fatal(err) + } + if names := FilterThrotllerHosts(legacy); !reflect.DeepEqual(names, []string{"primary"}) { + t.Fatalf("legacy hosts = %v, want [primary]", names) + } + }) + } +} + +func TestServerAddressRejectsMalformedCSV(t *testing.T) { + for _, report := range []string{`"unterminated`, `unexpected"quote`} { + t.Run(report, func(t *testing.T) { + csv := "# pxname,svname,status,last_chk,addr\n" + + "writers,healthy,UP,OK,192.0.2.10:3307\n" + + "writers,broken,UP," + report + ",192.0.2.11:3307\n" + if hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers"); err == nil || len(hosts) != 0 { + t.Fatalf("hosts = %+v, err = %v; want an error without a partial roster", hosts, err) + } + }) + } +} + +func TestServerAddressIgnoresIneligibleAddresses(t *testing.T) { + for _, status := range []string{"MAINT (resolution)", "NOLB", "UP 1/2", "unknown"} { + for _, address := range []string{",", ",not-an-address", ""} { + t.Run(status+address, func(t *testing.T) { + csv := "# pxname,svname,status,addr\n" + + "writers,primary,UP,192.0.2.10:3307\n" + + "writers,unused," + status + address + "\n" + hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers") + if err != nil { + t.Fatal(err) + } + got := FilterThrottlerBackendHosts(hosts) + want := []*BackendHost{{Hostname: "192.0.2.10", Port: 3307, Status: StatusUp}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("hosts = %+v, want %+v", got, want) + } + }) + } + } + for _, status := range []string{"UP", "DOWN", "DOWN 1/2", "no check"} { + t.Run("eligible "+status, func(t *testing.T) { + csv := "# pxname,svname,status,addr\n" + + "writers,healthy,UP,192.0.2.10:3307\nwriters,missing," + status + ",\n" + if hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers"); err == nil || len(hosts) != 0 { + t.Fatalf("hosts = %+v, err = %v; want an error without a partial roster", hosts, err) + } + }) + } +} + +func TestServerAddressTransitionSafeguardsWithEmptyAddresses(t *testing.T) { + for _, tt := range []struct { + name string + rows string + want error + }{ + {"all transitioning", "writers,one,UP 1/2,\nwriters,two,NOLB 1/2,\n", HAProxyAllHostsTransitioning}, + {"all UP transitioning", "writers,one,UP 1/2,\nwriters,two,MAINT (resolution),\n", HAProxyAllUpHostsTransitioning}, + } { + t.Run(tt.name, func(t *testing.T) { + _, err := ParseCsvHostsWithServerAddresses("# pxname,svname,status,addr\n"+tt.rows, "writers") + if !errors.Is(err, tt.want) { + t.Fatalf("error = %v, want %v", err, tt.want) + } + }) + } +} + func TestParseCsvHostsWithServerAddresses(t *testing.T) { for _, tt := range []struct { address string diff --git a/pkg/haproxy/throttle.go b/pkg/haproxy/throttle.go index fc8228f4..3b70e1f6 100644 --- a/pkg/haproxy/throttle.go +++ b/pkg/haproxy/throttle.go @@ -10,20 +10,20 @@ func FilterThrotllerHosts(backendHosts [](*BackendHost)) (hosts []string) { // FilterThrottlerBackendHosts retains eligible servers, including their discovered ports. func FilterThrottlerBackendHosts(backendHosts []*BackendHost) (hosts []*BackendHost) { for _, backendHost := range backendHosts { - hostIsRelevant := false - switch backendHost.Status { - case StatusUp: - if !backendHost.IsTransitioning { - hostIsRelevant = true - } - case StatusDown: - hostIsRelevant = true - case StatusNoCheck: - hostIsRelevant = true - } - if hostIsRelevant { + if backendHost.isRelevantForThrottling() { hosts = append(hosts, backendHost) } } return hosts } + +func (host *BackendHost) isRelevantForThrottling() bool { + switch host.Status { + case StatusUp: + return !host.IsTransitioning + case StatusDown, StatusNoCheck: + return true + default: + return false + } +} diff --git a/pkg/throttle/haproxy_inventory_test.go b/pkg/throttle/haproxy_inventory_test.go index 7d05e3b3..8b6c636a 100644 --- a/pkg/throttle/haproxy_inventory_test.go +++ b/pkg/throttle/haproxy_inventory_test.go @@ -63,6 +63,16 @@ func TestRefreshMySQLInventoryHAProxyServerAddress(t *testing.T) { "writer_a": {{Hostname: "2001:db8::10", Port: 3309}}, }, }, + { + name: "opt in accepts quoted reports and unused maintenance slots", + options: `{"UseServerAddress":true}`, + csv: "# pxname,svname,status,last_chk,addr\n" + + "writer_a,primary,no check,\"check passed, response \"\"OK\"\"\",192.0.2.10:3307\n" + + "writer_a,unused,MAINT (resolution),,\n", + want: map[string][]mysql.InstanceKey{ + "writer_a": {{Hostname: "192.0.2.10", Port: 3307}}, + }, + }, { name: "opt in applies exclusions to discovered addresses", options: `{"UseServerAddress":true}`,