Skip to content

Commit cf6d8fc

Browse files
author
Shlomi Noach
committed
taking into account transitioning hosts
1 parent c675d44 commit cf6d8fc

File tree

2 files changed

+95
-4
lines changed

2 files changed

+95
-4
lines changed

‎go/haproxy/parser.go

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ var HAProxyEmptyBody error = fmt.Errorf("Haproxy GET error: empty body")
1616
var HAProxyEmptyStatus error = fmt.Errorf("Haproxy CSV parsing error: no lines found")
1717
var HAProxyPartialStatus error = fmt.Errorf("Haproxy CSV parsing error: only got partial file")
1818
var HAProxyMissingPool error = fmt.Errorf("Haproxy CSV parsing: pool not found")
19+
var HAProxyAllUpHostsTransitioning error = fmt.Errorf("Haproxy: all host marked as UP are in transition. HAProxy is likely reloading")
20+
var HAProxyAllHostsTransitioning error = fmt.Errorf("Haproxy: all hosts are in transition. HAProxy is likely reloading")
1921

2022
var MaxHTTPGetConcurrency = 2
2123
var httpGetConcurrentcyChan = make(chan bool, MaxHTTPGetConcurrency)
@@ -51,6 +53,10 @@ func ParseHosts(csvLines []string, poolName string) (hosts []string, err error)
5153
}
5254
var tokensMap map[string]int
5355
poolFound := false
56+
countHosts := 0
57+
countUpHosts := 0
58+
countTransitioningHosts := 0
59+
countTransitioningUpHosts := 0
5460
for i, line := range csvLines {
5561
if i == 0 {
5662
tokensMap = parseHeader(csvLines[0])
@@ -60,17 +66,47 @@ func ParseHosts(csvLines []string, poolName string) (hosts []string, err error)
6066
if tokens[tokensMap["pxname"]] == poolName {
6167
poolFound = true
6268
if host := tokens[tokensMap["svname"]]; host != "BACKEND" && host != "FRONTEND" {
63-
status := tokens[tokensMap["status"]]
64-
status = strings.Split(status, " ")[0]
65-
if status == "UP" || status == "DOWN" {
66-
hosts = append(hosts, host)
69+
countHosts++
70+
statusTokens := strings.Split(tokens[tokensMap["status"]], " ")
71+
// status can show up as:
72+
// `UP`
73+
// `UP 1/2` (transitioning)
74+
// `NOLB`
75+
// `DOWN`
76+
// `DOWN (agent)`
77+
// etc. See https://github.com/haproxy/haproxy/blob/a5de024d42c4113fc6e189ea1d0ba6335219e151/src/dumpstats.c#L4117-L4129
78+
isTransitioning := (len(statusTokens) > 1 && strings.Contains(statusTokens[1], "/"))
79+
if isTransitioning {
80+
countTransitioningHosts++
81+
}
82+
83+
switch status := statusTokens[0]; status {
84+
case "UP":
85+
{
86+
countUpHosts++
87+
if isTransitioning {
88+
countTransitioningUpHosts++
89+
} else {
90+
hosts = append(hosts, host)
91+
}
92+
}
93+
case "DOWN":
94+
{
95+
hosts = append(hosts, host)
96+
}
6797
}
6898
}
6999
}
70100
}
71101
if !poolFound {
72102
return hosts, HAProxyMissingPool
73103
}
104+
if countTransitioningHosts == countHosts && countHosts > 0 {
105+
return hosts, HAProxyAllHostsTransitioning
106+
}
107+
if countTransitioningUpHosts == countUpHosts && countUpHosts > 0 {
108+
return hosts, HAProxyAllUpHostsTransitioning
109+
}
74110
return hosts, nil
75111
}
76112

‎go/haproxy/parser_test.go

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,45 @@ statsctl,FRONTEND,,,1,3,2000,21718,2788357,173364223,0,0,315,,,,,OPEN,,,,,,,,,1,
3838
statsctl,BACKEND,0,0,0,0,200,0,2788357,173364223,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,9,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,0,,,0,0,0,0,
3939
`
4040

41+
var csvTransitioning = `# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,
42+
mysqlcluster0ro,FRONTEND,,,0,0,20000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,5,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
43+
mysqlcluster0_ro_main,mysqlcluster0a-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP 1/2,10,1,0,49,6,89174,368958,,1,6,1,,0,,2,0,,0,L7OK,200,18,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
44+
mysqlcluster0_ro_main,mysqlcluster0b-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP,10,1,0,41,5,1912,1000,,1,6,2,,0,,2,0,,0,L7OK,200,24,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
45+
mysqlcluster0_ro_main,mysqlcluster0c-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB,10,1,0,0,0,1032061,0,,1,6,3,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
46+
mysqlcluster0_ro_main,mysqlcluster0d-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB,10,1,0,0,0,1032061,0,,1,6,4,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
47+
mysqlcluster0_ro_main,BACKEND,0,0,0,0,2000,0,0,0,0,0,,0,0,0,0,UP,20,2,0,,4,89174,728,,1,6,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
48+
monitoring,FRONTEND,,,0,0,2000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,8,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
49+
monitoring,BACKEND,0,0,0,0,200,0,0,0,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,8,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
50+
statsctl,FRONTEND,,,1,3,2000,21718,2788357,173364223,0,0,315,,,,,OPEN,,,,,,,,,1,9,0,,,,0,1,0,3,,,,0,21403,0,315,0,0,,1,3,21719,,,0,0,0,0,,,,,,,,
51+
statsctl,BACKEND,0,0,0,0,200,0,2788357,173364223,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,9,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,0,,,0,0,0,0,
52+
`
53+
54+
var csvTransitioningAllUp = `# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,
55+
mysqlcluster0ro,FRONTEND,,,0,0,20000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,5,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
56+
mysqlcluster0_ro_main,mysqlcluster0a-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP 1/2,10,1,0,49,6,89174,368958,,1,6,1,,0,,2,0,,0,L7OK,200,18,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
57+
mysqlcluster0_ro_main,mysqlcluster0b-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP 1/2,10,1,0,41,5,1912,1000,,1,6,2,,0,,2,0,,0,L7OK,200,24,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
58+
mysqlcluster0_ro_main,mysqlcluster0c-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB,10,1,0,0,0,1032061,0,,1,6,3,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
59+
mysqlcluster0_ro_main,mysqlcluster0d-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB,10,1,0,0,0,1032061,0,,1,6,4,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
60+
mysqlcluster0_ro_main,BACKEND,0,0,0,0,2000,0,0,0,0,0,,0,0,0,0,UP,20,2,0,,4,89174,728,,1,6,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
61+
monitoring,FRONTEND,,,0,0,2000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,8,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
62+
monitoring,BACKEND,0,0,0,0,200,0,0,0,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,8,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
63+
statsctl,FRONTEND,,,1,3,2000,21718,2788357,173364223,0,0,315,,,,,OPEN,,,,,,,,,1,9,0,,,,0,1,0,3,,,,0,21403,0,315,0,0,,1,3,21719,,,0,0,0,0,,,,,,,,
64+
statsctl,BACKEND,0,0,0,0,200,0,2788357,173364223,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,9,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,0,,,0,0,0,0,
65+
`
66+
67+
var csvTransitioningAll = `# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,
68+
mysqlcluster0ro,FRONTEND,,,0,0,20000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,5,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
69+
mysqlcluster0_ro_main,mysqlcluster0a-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP 1/2,10,1,0,49,6,89174,368958,,1,6,1,,0,,2,0,,0,L7OK,200,18,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
70+
mysqlcluster0_ro_main,mysqlcluster0b-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP 1/2,10,1,0,41,5,1912,1000,,1,6,2,,0,,2,0,,0,L7OK,200,24,,,,,,,0,,,,0,0,,,,,-1,OK,,0,0,0,0,
71+
mysqlcluster0_ro_main,mysqlcluster0c-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB 2/3,10,1,0,0,0,1032061,0,,1,6,3,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
72+
mysqlcluster0_ro_main,mysqlcluster0d-dc,0,0,0,0,,0,0,0,,0,,0,0,0,0,NOLB 1/3,10,1,0,0,0,1032061,0,,1,6,4,,0,,2,0,,0,L7OKC,404,12,,,,,,,0,,,,0,0,,,,,-1,Not Found,,0,0,0,0,
73+
mysqlcluster0_ro_main,BACKEND,0,0,0,0,2000,0,0,0,0,0,,0,0,0,0,UP,20,2,0,,4,89174,728,,1,6,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
74+
monitoring,FRONTEND,,,0,0,2000,0,0,0,0,0,0,,,,,OPEN,,,,,,,,,1,8,0,,,,0,0,0,0,,,,,,,,,,,0,0,0,,,0,0,0,0,,,,,,,,
75+
monitoring,BACKEND,0,0,0,0,200,0,0,0,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,8,0,,0,,1,0,,0,,,,,,,,,,,,,,0,0,0,0,0,0,-1,,,0,0,0,0,
76+
statsctl,FRONTEND,,,1,3,2000,21718,2788357,173364223,0,0,315,,,,,OPEN,,,,,,,,,1,9,0,,,,0,1,0,3,,,,0,21403,0,315,0,0,,1,3,21719,,,0,0,0,0,,,,,,,,
77+
statsctl,BACKEND,0,0,0,0,200,0,2788357,173364223,0,0,,0,0,0,0,UP,0,0,0,,0,1032064,0,,1,9,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,0,0,0,0,0,,,0,0,0,0,
78+
`
79+
4180
func init() {
4281
log.SetLevel(log.ERROR)
4382
}
@@ -69,3 +108,19 @@ func TestParseHosts(t *testing.T) {
69108
test.S(t).ExpectTrue(reflect.DeepEqual(hosts, []string{"mysqlcluster0e-dc", "mysqlcluster0f-dc", "mysqlcluster0h-dc"}))
70109
}
71110
}
111+
112+
func TestParseHostsTransitioning(t *testing.T) {
113+
{
114+
hosts, err := ParseCsvHosts(csvTransitioning, "mysqlcluster0_ro_main")
115+
test.S(t).ExpectNil(err)
116+
test.S(t).ExpectTrue(reflect.DeepEqual(hosts, []string{"mysqlcluster0b-dc"}))
117+
}
118+
{
119+
_, err := ParseCsvHosts(csvTransitioningAllUp, "mysqlcluster0_ro_main")
120+
test.S(t).ExpectEquals(err, HAProxyAllUpHostsTransitioning)
121+
}
122+
{
123+
_, err := ParseCsvHosts(csvTransitioningAll, "mysqlcluster0_ro_main")
124+
test.S(t).ExpectEquals(err, HAProxyAllHostsTransitioning)
125+
}
126+
}

0 commit comments

Comments
 (0)