Skip to content

Commit efc92f6

Browse files
author
Shlomi Noach
authored
Merge pull request #59 from github/haproxy-failure-impact
Handling transitioing status; limiting concurrent reads from HAProxy; named errors
2 parents 72d48d4 + cf6d8fc commit efc92f6

File tree

3 files changed

+116
-10
lines changed

3 files changed

+116
-10
lines changed

‎go/haproxy/parser.go

Lines changed: 57 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,16 @@ import (
1212

1313
var csvCache = cache.New(time.Second, time.Second)
1414

15+
var HAProxyEmptyBody error = fmt.Errorf("Haproxy GET error: empty body")
16+
var HAProxyEmptyStatus error = fmt.Errorf("Haproxy CSV parsing error: no lines found")
17+
var HAProxyPartialStatus error = fmt.Errorf("Haproxy CSV parsing error: only got partial file")
18+
var HAProxyMissingPool error = fmt.Errorf("Haproxy CSV parsing: pool not found")
19+
var HAProxyAllUpHostsTransitioning error = fmt.Errorf("Haproxy: all host marked as UP are in transition. HAProxy is likely reloading")
20+
var HAProxyAllHostsTransitioning error = fmt.Errorf("Haproxy: all hosts are in transition. HAProxy is likely reloading")
21+
22+
var MaxHTTPGetConcurrency = 2
23+
var httpGetConcurrentcyChan = make(chan bool, MaxHTTPGetConcurrency)
24+
1525
// parseHeader parses the HAPRoxy CSV header, which lists column names.
1626
// Returned is a header-to-index map
1727
func parseHeader(header string) (tokensMap map[string]int) {
@@ -36,13 +46,17 @@ func parseLines(csv string) []string {
3646
// Such list indicates the hosts which can be expected to be active, which is then the list freno will probe.
3747
func ParseHosts(csvLines []string, poolName string) (hosts []string, err error) {
3848
if len(csvLines) < 1 {
39-
return hosts, fmt.Errorf("Haproxy CSV parsing error: no lines found.")
49+
return hosts, HAProxyEmptyStatus
4050
}
4151
if len(csvLines) == 1 {
42-
return hosts, fmt.Errorf("Haproxy CSV parsing error: only found a header.")
52+
return hosts, HAProxyPartialStatus
4353
}
4454
var tokensMap map[string]int
4555
poolFound := false
56+
countHosts := 0
57+
countUpHosts := 0
58+
countTransitioningHosts := 0
59+
countTransitioningUpHosts := 0
4660
for i, line := range csvLines {
4761
if i == 0 {
4862
tokensMap = parseHeader(csvLines[0])
@@ -52,16 +66,46 @@ func ParseHosts(csvLines []string, poolName string) (hosts []string, err error)
5266
if tokens[tokensMap["pxname"]] == poolName {
5367
poolFound = true
5468
if host := tokens[tokensMap["svname"]]; host != "BACKEND" && host != "FRONTEND" {
55-
status := tokens[tokensMap["status"]]
56-
status = strings.Split(status, " ")[0]
57-
if status == "UP" || status == "DOWN" {
58-
hosts = append(hosts, host)
69+
countHosts++
70+
statusTokens := strings.Split(tokens[tokensMap["status"]], " ")
71+
// status can show up as:
72+
// `UP`
73+
// `UP 1/2` (transitioning)
74+
// `NOLB`
75+
// `DOWN`
76+
// `DOWN (agent)`
77+
// etc. See https://github.com/haproxy/haproxy/blob/a5de024d42c4113fc6e189ea1d0ba6335219e151/src/dumpstats.c#L4117-L4129
78+
isTransitioning := (len(statusTokens) > 1 && strings.Contains(statusTokens[1], "/"))
79+
if isTransitioning {
80+
countTransitioningHosts++
81+
}
82+
83+
switch status := statusTokens[0]; status {
84+
case "UP":
85+
{
86+
countUpHosts++
87+
if isTransitioning {
88+
countTransitioningUpHosts++
89+
} else {
90+
hosts = append(hosts, host)
91+
}
92+
}
93+
case "DOWN":
94+
{
95+
hosts = append(hosts, host)
96+
}
5997
}
6098
}
6199
}
62100
}
63101
if !poolFound {
64-
return hosts, fmt.Errorf("Haproxy CSV parsing error: did not find %+v pool", poolName)
102+
return hosts, HAProxyMissingPool
103+
}
104+
if countTransitioningHosts == countHosts && countHosts > 0 {
105+
return hosts, HAProxyAllHostsTransitioning
106+
}
107+
if countTransitioningUpHosts == countUpHosts && countUpHosts > 0 {
108+
return hosts, HAProxyAllUpHostsTransitioning
65109
}
66110
return hosts, nil
67111
}
@@ -75,6 +119,9 @@ func ParseCsvHosts(csv string, poolName string) (hosts []string, err error) {
75119

76120
// Read will read HAProxy URI and return with the CSV text
77121
func Read(host string, port int) (csv string, err error) {
122+
httpGetConcurrentcyChan <- true
123+
defer func() { <-httpGetConcurrentcyChan }()
124+
78125
haproxyUrl := fmt.Sprintf("http://%s:%d/;csv;norefresh", host, port)
79126

80127
if cachedCSV, found := csvCache.Get(haproxyUrl); found {
@@ -94,6 +141,9 @@ func Read(host string, port int) (csv string, err error) {
94141
}
95142

96143
csv = string(body)
144+
if csv == "" {
145+
return "", HAProxyEmptyBody
146+
}
97147
csvCache.Set(haproxyUrl, csv, cache.DefaultExpiration)
98148
return csv, nil
99149
}

‎go/haproxy/parser_test.go

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,45 @@ statsctl,FRONTEND,,,1,3,2000,21718,2788357,173364223,0,0,315,,,,,OPEN,,,,,,,,,1,
3838
statsctl,BACKEND,0,0,0,0,200,0,2788357,173364223,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,9,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,0,,,0,0,0,0,
3939
`
4040

41+
var csvTransitioning = `# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,
42+
mysqlcluster0ro,FRONTEND,,,0,0,20000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,5,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
43+
mysqlcluster0_ro_main,mysqlcluster0a-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP 1/2,10,1,0,49,6,89174,368958,,1,6,1,,0,,2,0,,0,L7OK,200,18,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
44+
mysqlcluster0_ro_main,mysqlcluster0b-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP,10,1,0,41,5,1912,1000,,1,6,2,,0,,2,0,,0,L7OK,200,24,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
45+
mysqlcluster0_ro_main,mysqlcluster0c-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB,10,1,0,0,0,1032061,0,,1,6,3,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
46+
mysqlcluster0_ro_main,mysqlcluster0d-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB,10,1,0,0,0,1032061,0,,1,6,4,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
47+
mysqlcluster0_ro_main,BACKEND,0,0,0,0,2000,0,0,0,0,0,,0,0,0,0,UP,20,2,0,,4,89174,728,,1,6,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
48+
monitoring,FRONTEND,,,0,0,2000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,8,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
49+
monitoring,BACKEND,0,0,0,0,200,0,0,0,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,8,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
50+
statsctl,FRONTEND,,,1,3,2000,21718,2788357,173364223,0,0,315,,,,,OPEN,,,,,,,,,1,9,0,,,,0,1,0,3,,,,0,21403,0,315,0,0,,1,3,21719,,,0,0,0,0,,,,,,,,
51+
statsctl,BACKEND,0,0,0,0,200,0,2788357,173364223,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,9,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,0,,,0,0,0,0,
52+
`
53+
54+
var csvTransitioningAllUp = `# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,
55+
mysqlcluster0ro,FRONTEND,,,0,0,20000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,5,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
56+
mysqlcluster0_ro_main,mysqlcluster0a-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP 1/2,10,1,0,49,6,89174,368958,,1,6,1,,0,,2,0,,0,L7OK,200,18,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
57+
mysqlcluster0_ro_main,mysqlcluster0b-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP 1/2,10,1,0,41,5,1912,1000,,1,6,2,,0,,2,0,,0,L7OK,200,24,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
58+
mysqlcluster0_ro_main,mysqlcluster0c-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB,10,1,0,0,0,1032061,0,,1,6,3,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
59+
mysqlcluster0_ro_main,mysqlcluster0d-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB,10,1,0,0,0,1032061,0,,1,6,4,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
60+
mysqlcluster0_ro_main,BACKEND,0,0,0,0,2000,0,0,0,0,0,,0,0,0,0,UP,20,2,0,,4,89174,728,,1,6,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
61+
monitoring,FRONTEND,,,0,0,2000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,8,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
62+
monitoring,BACKEND,0,0,0,0,200,0,0,0,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,8,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
63+
statsctl,FRONTEND,,,1,3,2000,21718,2788357,173364223,0,0,315,,,,,OPEN,,,,,,,,,1,9,0,,,,0,1,0,3,,,,0,21403,0,315,0,0,,1,3,21719,,,0,0,0,0,,,,,,,,
64+
statsctl,BACKEND,0,0,0,0,200,0,2788357,173364223,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,9,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,0,,,0,0,0,0,
65+
`
66+
67+
var csvTransitioningAll = `# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,
68+
mysqlcluster0ro,FRONTEND,,,0,0,20000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,5,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
69+
mysqlcluster0_ro_main,mysqlcluster0a-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP 1/2,10,1,0,49,6,89174,368958,,1,6,1,,0,,2,0,,0,L7OK,200,18,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
70+
mysqlcluster0_ro_main,mysqlcluster0b-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP 1/2,10,1,0,41,5,1912,1000,,1,6,2,,0,,2,0,,0,L7OK,200,24,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
71+
mysqlcluster0_ro_main,mysqlcluster0c-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB 2/3,10,1,0,0,0,1032061,0,,1,6,3,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
72+
mysqlcluster0_ro_main,mysqlcluster0d-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB 1/3,10,1,0,0,0,1032061,0,,1,6,4,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
73+
mysqlcluster0_ro_main,BACKEND,0,0,0,0,2000,0,0,0,0,0,,0,0,0,0,UP,20,2,0,,4,89174,728,,1,6,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
74+
monitoring,FRONTEND,,,0,0,2000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,8,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
75+
monitoring,BACKEND,0,0,0,0,200,0,0,0,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,8,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
76+
statsctl,FRONTEND,,,1,3,2000,21718,2788357,173364223,0,0,315,,,,,OPEN,,,,,,,,,1,9,0,,,,0,1,0,3,,,,0,21403,0,315,0,0,,1,3,21719,,,0,0,0,0,,,,,,,,
77+
statsctl,BACKEND,0,0,0,0,200,0,2788357,173364223,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,9,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,0,,,0,0,0,0,
78+
`
79+
4180
func init() {
4281
log.SetLevel(log.ERROR)
4382
}
@@ -69,3 +108,19 @@ func TestParseHosts(t *testing.T) {
69108
test.S(t).ExpectTrue(reflect.DeepEqual(hosts, []string{"mysqlcluster0e-dc", "mysqlcluster0f-dc", "mysqlcluster0h-dc"}))
70109
}
71110
}
111+
112+
func TestParseHostsTransitioning(t *testing.T) {
113+
{
114+
hosts, err := ParseCsvHosts(csvTransitioning, "mysqlcluster0_ro_main")
115+
test.S(t).ExpectNil(err)
116+
test.S(t).ExpectTrue(reflect.DeepEqual(hosts, []string{"mysqlcluster0b-dc"}))
117+
}
118+
{
119+
_, err := ParseCsvHosts(csvTransitioningAllUp, "mysqlcluster0_ro_main")
120+
test.S(t).ExpectEquals(err, HAProxyAllUpHostsTransitioning)
121+
}
122+
{
123+
_, err := ParseCsvHosts(csvTransitioningAll, "mysqlcluster0_ro_main")
124+
test.S(t).ExpectEquals(err, HAProxyAllHostsTransitioning)
125+
}
126+
}

‎go/throttle/throttler.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -203,11 +203,12 @@ func (throttler *Throttler) refreshMySQLInventory() error {
203203
if err != nil {
204204
return log.Errorf("Unable to get HAproxy data from %s:%d: %+v", clusterSettings.HAProxySettings.Host, clusterSettings.HAProxySettings.Port, err)
205205
}
206-
hosts, err := haproxy.ParseCsvHosts(csv, clusterSettings.HAProxySettings.PoolName)
206+
poolName := clusterSettings.HAProxySettings.PoolName
207+
hosts, err := haproxy.ParseCsvHosts(csv, poolName)
207208
if err != nil {
208-
return log.Errorf("Unable to get HAproxy hosts from %s:%d: %+v", clusterSettings.HAProxySettings.Host, clusterSettings.HAProxySettings.Port, err)
209+
return log.Errorf("Unable to get HAproxy hosts from %s:%d/#%s: %+v", clusterSettings.HAProxySettings.Host, clusterSettings.HAProxySettings.Port, poolName, err)
209210
}
210-
log.Debugf("Read %+v hosts from haproxy %s:%d/%s", len(hosts), clusterSettings.HAProxySettings.Host, clusterSettings.HAProxySettings.Port, clusterSettings.HAProxySettings.PoolName)
211+
log.Debugf("Read %+v hosts from haproxy %s:%d/#%s", len(hosts), clusterSettings.HAProxySettings.Host, clusterSettings.HAProxySettings.Port, poolName)
211212
clusterProbes := &mysql.ClusterProbes{
212213
ClusterName: clusterName,
213214
IgnoreHostsCount: clusterSettings.IgnoreHostsCount,

0 commit comments

Comments
 (0)