Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions doc/mysql.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,35 @@ Noteworthy:
- `sharded` list of servers is read from Vitess vtctld's HTTP `API`. To read it with vtctld's gRPC `GetTablets` instead (Vitess v23 removes the HTTP API), set `"GRPCAddress": "vtctld.example.com:15999"` and `"GRPCInsecure": true`. Only plaintext gRPC is supported.


### HAProxy server addresses

By default, HAProxy discovery uses each server's `svname` as its hostname and the
cluster's configured MySQL port. If server labels are aliases rather than
resolvable hostnames, set `"UseServerAddress": true` inside that cluster's
`HAProxySettings` to use the CSV `addr` host and port instead. For example, a server
labelled `primary` with `addr` `192.0.2.10:3307` is probed at `192.0.2.10:3307`.
Bracketed IPv6 addresses such as `[2001:db8::10]:3307` are supported.

This setting defaults to `false` and does not change other clusters. When enabled,
`IgnoreHosts` and skipped-host checks apply to the discovered address, not the
server label.

For HTTP stats discovery, enable `stats show-legends` in the HAProxy stats
configuration so that the CSV includes server address values. Without this
setting, the `addr` column can be present but empty. Verify that eligible servers
have populated addresses before enabling `UseServerAddress`.

The CSV must contain an `addr` column and a valid host and numeric port (1-65535)
for every eligible backend server in the selected pool. Address validation skips
servers excluded by the existing status rules, such as unresolved maintenance
slots, `NOLB` servers, and transitioning `UP` servers. `DOWN` and `no check`
servers remain eligible and require valid addresses. Existing safeguards for
all-transitioning servers are unchanged.

Invalid or missing addresses on eligible servers cause a discovery error,
without falling back to the label. As with other HAProxy discovery failures, an
unsuccessful refresh can leave the previous inventory in place.

### Non lag metrics

`freno` isn't necessarily about replication lag. You may choose to use different thresholds appropriate for your setup and workload. For example, you may choose to monitor the master (as opposed of the replicas) and read some metric such as `threads_running`. An example configuration would be:
Expand Down
9 changes: 5 additions & 4 deletions pkg/config/haproxy_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,11 @@ func ParseHostPort(address string) (hostPort *HostPort, err error) {
}

type HAProxyConfigurationSettings struct {
Host string
Port int
Addresses string
PoolName string
Host string
Port int
Addresses string
PoolName string
UseServerAddress bool
}

func parseAddress(address string) (u *url.URL, err error) {
Expand Down
1 change: 1 addition & 0 deletions pkg/haproxy/host.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ func ToBackendHostStatus(status string) BackendHostStatus {

type BackendHost struct {
Hostname string
Port int
Status BackendHostStatus
IsTransitioning bool
}
Expand Down
68 changes: 62 additions & 6 deletions pkg/haproxy/parser.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
package haproxy

import (
"encoding/csv"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -72,32 +76,78 @@ func ParseStatus(fullStatus string) (status BackendHostStatus, isTransitioning b
// Returned are all non-disabled hosts in given backend. Thus, a NOLB is skipped; any UP or DOWN hosts are returned.
// Such list indicates the hosts which can be expected to be active, which is then the list freno will probe.
func ParseHosts(csvLines []string, poolName string) (backendHosts [](*BackendHost), err error) {
return parseHosts(csvLines, poolName, false)
}

func parseHosts(csvLines []string, poolName string, useServerAddress bool) (backendHosts []*BackendHost, err error) {
if len(csvLines) < 1 {
return backendHosts, HAProxyEmptyStatus
}
if len(csvLines) == 1 {
return backendHosts, HAProxyPartialStatus
}
var tokensMap map[string]int
tokensMap := parseHeader(csvLines[0])
requiredColumns := 0
for _, field := range []string{"pxname", "svname", "status"} {
index, ok := tokensMap[field]
if !ok {
return nil, fmt.Errorf("HAProxy CSV parsing: missing %s column", field)
}
if index+1 > requiredColumns {
requiredColumns = index + 1
}
}
addressColumn, hasAddressColumn := tokensMap["addr"]
if useServerAddress && !hasAddressColumn {
return nil, fmt.Errorf("HAProxy CSV parsing: missing addr column")
}
reader := csv.NewReader(strings.NewReader(strings.Join(csvLines[1:], "\n")))
reader.FieldsPerRecord = -1
poolFound := false
countHosts := 0
countUpHosts := 0
countTransitioningHosts := 0
countTransitioningUpHosts := 0
for i, line := range csvLines {
if i == 0 {
tokensMap = parseHeader(csvLines[0])
for {
tokens, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("HAProxy CSV parsing: %w", err)
}
if len(tokens) == 1 && strings.TrimSpace(tokens[0]) == "" {
continue
}
tokens := strings.Split(line, ",")
if len(tokens) < requiredColumns {
return nil, fmt.Errorf("HAProxy CSV parsing: incomplete row")
}
if tokens[tokensMap["pxname"]] == poolName {
poolFound = true
if host := tokens[tokensMap["svname"]]; host != "BACKEND" && host != "FRONTEND" {
countHosts++

status, isTransitioning := ParseStatus(tokens[tokensMap["status"]])

backendHosts = append(backendHosts, NewBackendHost(host, status, isTransitioning))
backendHost := NewBackendHost(host, status, isTransitioning)
if useServerAddress && backendHost.isRelevantForThrottling() {
if len(tokens) <= addressColumn {
return nil, fmt.Errorf("HAProxy CSV parsing: missing addr for %s/%s", poolName, host)
}
address := tokens[addressColumn]
hostname, portText, err := net.SplitHostPort(address)
ip := net.ParseIP(hostname)
if err != nil || hostname == "" || strings.ContainsAny(hostname, " \t\r\n") || (ip != nil && ip.IsUnspecified()) {
return nil, fmt.Errorf("HAProxy CSV parsing: invalid addr %q for %s/%s", address, poolName, host)
}
port, err := strconv.Atoi(portText)
if err != nil || port <= 0 || port > 65535 {
return nil, fmt.Errorf("HAProxy CSV parsing: invalid port in addr %q for %s/%s", address, poolName, host)
}
backendHost.Hostname = hostname
backendHost.Port = port
}
backendHosts = append(backendHosts, backendHost)
if isTransitioning {
countTransitioningHosts++
}
Expand Down Expand Up @@ -129,6 +179,12 @@ func ParseCsvHosts(csv string, poolName string) (backendHosts [](*BackendHost),
return ParseHosts(csvLines, poolName)
}

// ParseCsvHostsWithServerAddresses uses each server's addr instead of its svname label.
// Each eligible backend server row must have a valid host:port address.
func ParseCsvHostsWithServerAddresses(csv string, poolName string) ([]*BackendHost, error) {
return parseHosts(parseLines(csv), poolName, true)
}

func toCSVUrl(u url.URL) *url.URL {
u.Path = fmt.Sprintf("%s;csv;norefresh", u.Path)
return &u
Expand Down
179 changes: 179 additions & 0 deletions pkg/haproxy/server_address_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
package haproxy

import (
"errors"
"fmt"
"reflect"
"testing"
)

func TestServerAddressQuotedCSVFields(t *testing.T) {
for _, report := range []string{
`"check passed, response OK"`,
`"check passed, response ""OK"""`,
"\"check passed,\nresponse OK\"",
} {
t.Run(report, func(t *testing.T) {
csv := "# pxname,svname,status,last_chk,addr\nwriters,primary,no check," + report + ",192.0.2.10:3307\n"
hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers")
if err != nil {
t.Fatal(err)
}
got := FilterThrottlerBackendHosts(hosts)
want := []*BackendHost{{Hostname: "192.0.2.10", Port: 3307, Status: StatusNoCheck}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("hosts = %+v, want %+v", got, want)
}
legacy, err := ParseCsvHosts(csv, "writers")
if err != nil {
t.Fatal(err)
}
if names := FilterThrotllerHosts(legacy); !reflect.DeepEqual(names, []string{"primary"}) {
t.Fatalf("legacy hosts = %v, want [primary]", names)
}
})
}
}

func TestServerAddressRejectsMalformedCSV(t *testing.T) {
for _, report := range []string{`"unterminated`, `unexpected"quote`} {
t.Run(report, func(t *testing.T) {
csv := "# pxname,svname,status,last_chk,addr\n" +
"writers,healthy,UP,OK,192.0.2.10:3307\n" +
"writers,broken,UP," + report + ",192.0.2.11:3307\n"
if hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers"); err == nil || len(hosts) != 0 {
t.Fatalf("hosts = %+v, err = %v; want an error without a partial roster", hosts, err)
}
})
}
}

func TestServerAddressIgnoresIneligibleAddresses(t *testing.T) {
for _, status := range []string{"MAINT (resolution)", "NOLB", "UP 1/2", "unknown"} {
for _, address := range []string{",", ",not-an-address", ""} {
t.Run(status+address, func(t *testing.T) {
csv := "# pxname,svname,status,addr\n" +
"writers,primary,UP,192.0.2.10:3307\n" +
"writers,unused," + status + address + "\n"
hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers")
if err != nil {
t.Fatal(err)
}
got := FilterThrottlerBackendHosts(hosts)
want := []*BackendHost{{Hostname: "192.0.2.10", Port: 3307, Status: StatusUp}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("hosts = %+v, want %+v", got, want)
}
})
}
}
for _, status := range []string{"UP", "DOWN", "DOWN 1/2", "no check"} {
t.Run("eligible "+status, func(t *testing.T) {
csv := "# pxname,svname,status,addr\n" +
"writers,healthy,UP,192.0.2.10:3307\nwriters,missing," + status + ",\n"
if hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers"); err == nil || len(hosts) != 0 {
t.Fatalf("hosts = %+v, err = %v; want an error without a partial roster", hosts, err)
}
})
}
}

func TestServerAddressTransitionSafeguardsWithEmptyAddresses(t *testing.T) {
for _, tt := range []struct {
name string
rows string
want error
}{
{"all transitioning", "writers,one,UP 1/2,\nwriters,two,NOLB 1/2,\n", HAProxyAllHostsTransitioning},
{"all UP transitioning", "writers,one,UP 1/2,\nwriters,two,MAINT (resolution),\n", HAProxyAllUpHostsTransitioning},
} {
t.Run(tt.name, func(t *testing.T) {
_, err := ParseCsvHostsWithServerAddresses("# pxname,svname,status,addr\n"+tt.rows, "writers")
if !errors.Is(err, tt.want) {
t.Fatalf("error = %v, want %v", err, tt.want)
}
})
}
}

func TestParseCsvHostsWithServerAddresses(t *testing.T) {
for _, tt := range []struct {
address string
hostname string
port int
}{
{"192.0.2.10:3307", "192.0.2.10", 3307},
{"[2001:db8::10]:3308", "2001:db8::10", 3308},
{"db.example.com:3309", "db.example.com", 3309},
} {
t.Run(tt.address, func(t *testing.T) {
csv := "# pxname,svname,status,addr\n" +
fmt.Sprintf("writers,primary,no check,%s\n", tt.address) +
"writers,BACKEND,UP,\nwriters,FRONTEND,OPEN,\n"
hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers")
if err != nil {
t.Fatal(err)
}
if len(hosts) != 1 || hosts[0].Hostname != tt.hostname || hosts[0].Port != tt.port {
t.Fatalf("hosts = %+v, want %s port %d", hosts, tt.hostname, tt.port)
}
})
}
}

func TestParseCsvHostsWithServerAddressesRejectsInvalidAddresses(t *testing.T) {
for _, address := range []string{
"", "192.0.2.10", ":3306", "192.0.2.10:", "192.0.2.10:mysql",
"192.0.2.10:0", "192.0.2.10:-1", "192.0.2.10:65536",
"2001:db8::10:3306", "bad host:3306",
"0.0.0.0:3306", "[::]:3306",
} {
t.Run(address, func(t *testing.T) {
csv := "# pxname,svname,status,addr\n" +
"writers,healthy,UP,192.0.2.11:3306\n" +
fmt.Sprintf("writers,primary,no check,%s\n", address)
hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers")
if err == nil || len(hosts) != 0 {
t.Fatalf("hosts = %+v, err = %v; want an error without a partial roster", hosts, err)
}
})
}
for _, csv := range []string{
"# pxname,svname,status\nwriters,primary,no check\n",
"# pxname,svname,status,addr\nwriters,primary,no check\n",
"# pxname,svname,status,addr\nwriters,primary\n",
"# pxname,svname,addr\nwriters,primary,192.0.2.10:3306\n",
} {
if hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers"); err == nil || len(hosts) != 0 {
t.Errorf("CSV %q: hosts = %+v, err = %v; want a schema error", csv, hosts, err)
}
}
}

func TestServerAddressStatusAndTransitionFiltering(t *testing.T) {
csv := "# pxname,svname,status,addr\n" +
"writers,primary,no check,192.0.2.10:3307\n" +
"writers,down,DOWN,192.0.2.11:3308\n" +
"writers,up,UP,192.0.2.12:3309\n" +
"writers,disabled,NOLB,192.0.2.13:3310\n" +
"writers,transitioning,UP 1/2,192.0.2.14:3311\n"
hosts, err := ParseCsvHostsWithServerAddresses(csv, "writers")
if err != nil {
t.Fatal(err)
}
got := FilterThrottlerBackendHosts(hosts)
want := []*BackendHost{
{Hostname: "192.0.2.10", Port: 3307, Status: StatusNoCheck},
{Hostname: "192.0.2.11", Port: 3308, Status: StatusDown},
{Hostname: "192.0.2.12", Port: 3309, Status: StatusUp},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("filtered hosts = %+v, want %+v", got, want)
}
for _, status := range []string{"UP 1/2", "DOWN 1/2"} {
csv := fmt.Sprintf("# pxname,svname,status,addr\nwriters,primary,%s,192.0.2.10:3307\n", status)
if _, err := ParseCsvHostsWithServerAddresses(csv, "writers"); err == nil {
t.Errorf("all hosts transitioning (%s): expected an error", status)
}
}
}
34 changes: 21 additions & 13 deletions pkg/haproxy/throttle.go
Original file line number Diff line number Diff line change
@@ -1,21 +1,29 @@
package haproxy

func FilterThrotllerHosts(backendHosts [](*BackendHost)) (hosts []string) {
for _, backendHost := range FilterThrottlerBackendHosts(backendHosts) {
hosts = append(hosts, backendHost.Hostname)
}
return hosts
}

// FilterThrottlerBackendHosts retains eligible servers, including their discovered ports.
func FilterThrottlerBackendHosts(backendHosts []*BackendHost) (hosts []*BackendHost) {
for _, backendHost := range backendHosts {
hostIsRelevant := false
switch backendHost.Status {
case StatusUp:
if !backendHost.IsTransitioning {
hostIsRelevant = true
}
case StatusDown:
hostIsRelevant = true
case StatusNoCheck:
hostIsRelevant = true
}
if hostIsRelevant {
hosts = append(hosts, backendHost.Hostname)
if backendHost.isRelevantForThrottling() {
hosts = append(hosts, backendHost)
}
}
return hosts
}

func (host *BackendHost) isRelevantForThrottling() bool {
switch host.Status {
case StatusUp:
return !host.IsTransitioning
case StatusDown, StatusNoCheck:
return true
default:
return false
}
}
Loading