Skip to content

Commit 09a2a3a

Browse files
abhishaleman
abhi
authored and
shaleman
committed
L3 sanity stability fixes (#364)
* bgp route distribution delay * retrigger * retrigger * adding quagga route check * retrigger * retrigger * retrigger * retrigger * retrigger * retrigger * bgp sanity fix
1 parent fc546ce commit 09a2a3a

File tree

6 files changed

+78
-37
lines changed

6 files changed

+78
-37
lines changed

‎systemtests/aci_test.go

100644100755
File mode changed.

‎systemtests/basic_test.go

+5-3
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package systemtests
33
import (
44
"github.com/contiv/contivmodel/client"
55
. "gopkg.in/check.v1"
6-
"time"
76
)
87

98
var privateNetwork = &client.Network{
@@ -41,7 +40,7 @@ func (s *systemtestSuite) testBasicStartRemoveContainer(c *C, encap string) {
4140
for i := 0; i < s.iterations; i++ {
4241
containers, err := s.runContainers(s.containers, false, "private", nil)
4342
if s.fwdMode == "routing" && encap == "vlan" {
44-
time.Sleep(5 * time.Second)
43+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), containers)
4544
}
4645
c.Assert(err, IsNil)
4746
c.Assert(s.pingTest(containers), IsNil)
@@ -75,6 +74,9 @@ func (s *systemtestSuite) testBasicStartStopContainer(c *C, encap string) {
7574

7675
containers, err := s.runContainers(s.containers, false, "private", nil)
7776
c.Assert(err, IsNil)
77+
if s.fwdMode == "routing" && encap == "vlan" {
78+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), containers)
79+
}
7880

7981
for i := 0; i < s.iterations; i++ {
8082
c.Assert(s.pingTest(containers), IsNil)
@@ -97,7 +99,7 @@ func (s *systemtestSuite) testBasicStartStopContainer(c *C, encap string) {
9799
}
98100

99101
if s.fwdMode == "routing" && encap == "vlan" {
100-
time.Sleep(5 * time.Second)
102+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), containers)
101103
}
102104

103105
}

‎systemtests/bgp_test.go

+46-22
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package systemtests
22

33
import (
4+
"errors"
45
"fmt"
56
"github.com/Sirupsen/logrus"
67
"github.com/contiv/contivmodel/client"
@@ -56,7 +57,7 @@ func (s *systemtestSuite) TestBgpContainerToContainerPing(c *C) {
5657
allcontainers = append(allcontainers, containers[name]...)
5758
}
5859

59-
time.Sleep(5 * time.Second)
60+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
6061
endChan := make(chan error)
6162

6263
logrus.Infof("Running ping test ")
@@ -118,7 +119,7 @@ func (s *systemtestSuite) TestBgpContainerToNonContainerPing(c *C) {
118119
c.Assert(err, IsNil)
119120
}
120121

121-
time.Sleep(5 * time.Second)
122+
time.Sleep(15 * time.Second)
122123
endChan := make(chan error)
123124

124125
//FIXME make it variable number of quagga instances
@@ -187,7 +188,7 @@ func (s *systemtestSuite) TestBgpTriggerPeerAddDelete(c *C) {
187188
for i := 0; i < s.iterations; i++ {
188189
s.SetupBgp(c, false)
189190
s.CheckBgpConnection(c)
190-
time.Sleep(5 * time.Second)
191+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
191192
logrus.Infof("Running ping test")
192193
c.Assert(s.pingTest(allcontainers), IsNil)
193194

@@ -243,7 +244,7 @@ func (s *systemtestSuite) TestBgpTriggerLinkUpDown(c *C) {
243244
c.Assert(err, IsNil)
244245
allcontainers = append(allcontainers, containers[name]...)
245246
}
246-
time.Sleep(5 * time.Second)
247+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
247248
endChan := make(chan error)
248249

249250
logrus.Infof("Running ping test")
@@ -253,7 +254,7 @@ func (s *systemtestSuite) TestBgpTriggerLinkUpDown(c *C) {
253254
s.CheckBgpNoConnectionForaNode(c, s.vagrant.GetNode("netplugin-node1"))
254255
s.vagrant.GetNode("netplugin-node1").RunCommand("sudo ip link set eth2 up")
255256
s.CheckBgpConnectionForaNode(c, s.vagrant.GetNode("netplugin-node1"))
256-
time.Sleep(5 * time.Second)
257+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
257258
logrus.Infof("Running ping test")
258259
c.Assert(s.pingTest(allcontainers), IsNil)
259260

@@ -320,7 +321,7 @@ func (s *systemtestSuite) TestBgpTriggerLoopbackDownUp(c *C) {
320321
s.SetupBgp(c, false)
321322
s.CheckBgpConnection(c)
322323

323-
time.Sleep(5 * time.Second)
324+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
324325

325326
logrus.Infof("Running ping test")
326327
c.Assert(s.pingTest(allcontainers), IsNil)
@@ -331,7 +332,7 @@ func (s *systemtestSuite) TestBgpTriggerLoopbackDownUp(c *C) {
331332
s.vagrant.GetNode("netplugin-node1").RunCommand("sudo ip link set inb01 up")
332333
s.vagrant.GetNode("netplugin-node1").RunCommand("sudo ip addr add 50.1.1.2/24 dev inb01")
333334
s.CheckBgpConnectionForaNode(c, s.vagrant.GetNode("netplugin-node1"))
334-
time.Sleep(5 * time.Second)
335+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
335336

336337
logrus.Infof("Running ping test")
337338
c.Assert(s.pingTest(allcontainers), IsNil)
@@ -403,7 +404,7 @@ func (s *systemtestSuite) TestBgpTriggerContainerAddDelete(c *C) {
403404
allcontainers = append(allcontainers, containers[name]...)
404405
}
405406

406-
time.Sleep(5 * time.Second)
407+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
407408

408409
endChan := make(chan error)
409410

@@ -434,7 +435,7 @@ func (s *systemtestSuite) TestBgpTriggerContainerAddDelete(c *C) {
434435
}
435436
}
436437

437-
time.Sleep(5 * time.Second)
438+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
438439

439440
logrus.Infof("Running ping test")
440441
c.Assert(s.pingTest(allcontainers), IsNil)
@@ -497,7 +498,7 @@ func (s *systemtestSuite) TestBgpTriggerNetpluginRestart(c *C) {
497498
allcontainers = append(allcontainers, containers[name]...)
498499
}
499500

500-
time.Sleep(5 * time.Second)
501+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
501502

502503
endChan := make(chan error)
503504

@@ -514,7 +515,7 @@ func (s *systemtestSuite) TestBgpTriggerNetpluginRestart(c *C) {
514515
c.Assert(node.runCommandUntilNoError("pgrep netplugin"), IsNil)
515516
time.Sleep(15 * time.Second)
516517
s.CheckBgpConnection(c)
517-
time.Sleep(5 * time.Second)
518+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
518519

519520
logrus.Infof("Running ping test")
520521
c.Assert(s.pingTest(allcontainers), IsNil)
@@ -675,7 +676,7 @@ func (s *systemtestSuite) TestBgpMultiTrigger(c *C) {
675676
allcontainers = append(allcontainers, containers[name]...)
676677
}
677678

678-
time.Sleep(5 * time.Second)
679+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
679680

680681
endChan := make(chan error)
681682

@@ -750,7 +751,7 @@ func (s *systemtestSuite) TestBgpSequencePeerAddLinkDown(c *C) {
750751
c.Assert(err, IsNil)
751752
allcontainers = append(allcontainers, containers[name]...)
752753
}
753-
time.Sleep(5 * time.Second)
754+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
754755

755756
logrus.Infof("Running ping test")
756757
c.Assert(s.pingTest(allcontainers), IsNil)
@@ -805,7 +806,7 @@ func (s *systemtestSuite) TestBgpMisconfigRecovery(c *C) {
805806
c.Assert(err, IsNil)
806807
allcontainers = append(allcontainers, containers[name]...)
807808
}
808-
time.Sleep(5 * time.Second)
809+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), allcontainers)
809810
endChan := make(chan error)
810811

811812
logrus.Infof("Running ping test")
@@ -872,12 +873,12 @@ func (s *systemtestSuite) TearDownBgp(c *C) {
872873
func (s *systemtestSuite) CheckBgpConnection(c *C) error {
873874

874875
count := 0
875-
for {
876+
for i := 0; i < 100; i++ {
876877
if count == len(s.nodes) {
877878
return nil
878879
}
879880
count = 0
880-
time.Sleep(5 * time.Second)
881+
time.Sleep(3 * time.Second)
881882
for _, node := range s.nodes {
882883
out, _ := node.tbnode.RunCommandWithOutput("/opt/gopath/bin/gobgp neighbor")
883884
fmt.Println(out)
@@ -886,17 +887,18 @@ func (s *systemtestSuite) CheckBgpConnection(c *C) error {
886887
}
887888
}
888889
}
890+
return errors.New("BGP connection not established")
889891
}
890892

891893
func (s *systemtestSuite) CheckBgpNoConnection(c *C) error {
892894

893895
count := 0
894-
for {
896+
for i := 0; i < 100; i++ {
895897
if count == len(s.nodes) {
896898
return nil
897899
}
898900
count = 0
899-
time.Sleep(2 * time.Second)
901+
time.Sleep(3 * time.Second)
900902
for _, node := range s.nodes {
901903
out, _ := node.tbnode.RunCommandWithOutput("/opt/gopath/bin/gobgp neighbor")
902904
fmt.Println(out)
@@ -905,26 +907,48 @@ func (s *systemtestSuite) CheckBgpNoConnection(c *C) error {
905907
}
906908
}
907909
}
910+
return errors.New("BGP connection persists")
908911
}
909912

910913
func (s *systemtestSuite) CheckBgpConnectionForaNode(c *C, node vagrantssh.TestbedNode) error {
911-
for {
912-
time.Sleep(2 * time.Second)
914+
for i := 0; i < 100; i++ {
915+
time.Sleep(3 * time.Second)
913916
out, _ := node.RunCommandWithOutput("/opt/gopath/bin/gobgp neighbor")
914917
fmt.Println(out)
915918
if strings.Contains(out, "Establ") {
916919
return nil
917920
}
918921
}
922+
return errors.New("BGP connection not established")
919923
}
920924

921925
func (s *systemtestSuite) CheckBgpNoConnectionForaNode(c *C, node vagrantssh.TestbedNode) error {
922-
for {
923-
time.Sleep(2 * time.Second)
926+
for i := 0; i < 100; i++ {
927+
time.Sleep(3 * time.Second)
924928
out, _ := node.RunCommandWithOutput("/opt/gopath/bin/gobgp neighbor")
925929
fmt.Println(out)
926930
if !strings.Contains(out, "Establ") {
927931
return nil
928932
}
929933
}
934+
return errors.New("BGP connection persists")
935+
}
936+
937+
func (s *systemtestSuite) CheckBgpRouteDistribution(c *C, node vagrantssh.TestbedNode, containers []*container) error {
938+
count := 0
939+
for i := 0; i < 10; i++ {
940+
time.Sleep(2 * time.Second)
941+
logrus.Infof("Checking Bgp container route distribution")
942+
out, _ := node.RunCommandWithOutput("ip route")
943+
for _, cont := range containers {
944+
if strings.Contains(out, cont.eth0) {
945+
count++
946+
}
947+
if count == len(containers) {
948+
return nil
949+
}
950+
}
951+
time.Sleep(2 * time.Second)
952+
}
953+
return errors.New("Routes not distributed by BGP")
930954
}

‎systemtests/network_test.go

+11-4
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ func (s *systemtestSuite) TestInfraNetworkAddDeleteVLAN(c *C) {
2020

2121
func (s *systemtestSuite) testInfraNetworkAddDelete(c *C, encap string) {
2222

23-
if s.fwdMode == "routing" {
23+
if s.fwdMode == "routing" && encap == "vlan" {
2424
s.SetupBgp(c, false)
2525
s.CheckBgpConnection(c)
2626
}
@@ -82,7 +82,7 @@ func (s *systemtestSuite) TestNetworkAddDeleteVLAN(c *C) {
8282

8383
func (s *systemtestSuite) testNetworkAddDelete(c *C, encap string) {
8484

85-
if s.fwdMode == "routing" {
85+
if s.fwdMode == "routing" && encap == "vlan" {
8686

8787
s.SetupBgp(c, false)
8888
s.CheckBgpConnection(c)
@@ -118,7 +118,12 @@ func (s *systemtestSuite) testNetworkAddDelete(c *C, encap string) {
118118
containers[name], err = s.runContainers(numContainer, false, name, nil)
119119
c.Assert(err, IsNil)
120120
}
121-
time.Sleep(5 * time.Second)
121+
122+
if s.fwdMode == "routing" && encap == "vlan" {
123+
for _, name := range netNames {
124+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), containers[name])
125+
}
126+
}
122127

123128
endChan := make(chan error)
124129

@@ -227,7 +232,9 @@ func (s *systemtestSuite) testNetworkAddDeleteTenant(c *C, encap string) {
227232
containers[network], err = s.runContainers(numContainer, false, fmt.Sprintf("%s/%s", network, tenant), nil)
228233
mutex.Unlock()
229234
endChan <- err
230-
time.Sleep(5 * time.Second)
235+
if s.fwdMode == "routing" && encap == "vlan" {
236+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), containers[network])
237+
}
231238
endChan <- s.pingTest(containers[network])
232239
}(network, tenant, containers)
233240
}

‎systemtests/node_test.go

100644100755
File mode changed.

‎systemtests/policy_test.go

+16-8
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import (
77
"github.com/Sirupsen/logrus"
88
"github.com/contiv/contivmodel/client"
99
. "gopkg.in/check.v1"
10-
"time"
1110
)
1211

1312
func (s *systemtestSuite) TestPolicyBasicVXLAN(c *C) {
@@ -87,7 +86,9 @@ func (s *systemtestSuite) testPolicyBasic(c *C, encap string) {
8786

8887
containers, err := s.runContainers(s.containers, true, "private", groupNames)
8988
c.Assert(err, IsNil)
90-
time.Sleep(5 * time.Second)
89+
if s.fwdMode == "routing" && encap == "vlan" {
90+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), containers)
91+
}
9192

9293
c.Assert(s.startListeners(containers, []int{8000, 8001}), IsNil)
9394
c.Assert(s.checkConnections(containers, 8000), IsNil)
@@ -183,8 +184,9 @@ func (s *systemtestSuite) testPolicyAddDeleteRule(c *C, encap string) {
183184

184185
containers, err := s.runContainers(s.containers, true, "private", groupNames)
185186
c.Assert(err, IsNil)
186-
time.Sleep(5 * time.Second)
187-
187+
if s.fwdMode == "routing" && encap == "vlan" {
188+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), containers)
189+
}
188190
c.Assert(s.startListeners(containers, []int{8000, 8001}), IsNil)
189191
c.Assert(s.checkConnections(containers, 8000), IsNil)
190192
c.Assert(s.checkNoConnections(containers, 8001), IsNil)
@@ -319,7 +321,9 @@ func (s *systemtestSuite) testPolicyFromEPG(c *C, encap string) {
319321

320322
containers, err := s.runContainers(s.containers, true, "private", policyNames)
321323
c.Assert(err, IsNil)
322-
time.Sleep(5 * time.Second)
324+
if s.fwdMode == "routing" && encap == "vlan" {
325+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), containers)
326+
}
323327

324328
commonNames := []string{}
325329
for _, name := range policyNames {
@@ -328,7 +332,9 @@ func (s *systemtestSuite) testPolicyFromEPG(c *C, encap string) {
328332

329333
cmnContainers, err := s.runContainersInService(s.containers, "common", "private", commonNames)
330334
c.Assert(err, IsNil)
331-
time.Sleep(5 * time.Second)
335+
if s.fwdMode == "routing" && encap == "vlan" {
336+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), cmnContainers)
337+
}
332338

333339
c.Assert(s.startListeners(containers, []int{8000, 8001}), IsNil)
334340

@@ -420,8 +426,10 @@ func (s *systemtestSuite) testPolicyFeatures(c *C, encap string) {
420426
c.Assert(err, IsNil)
421427
container2, err := s.nodes[0].runContainer(containerSpec{serviceName: "srv2", networkName: "private"})
422428
c.Assert(err, IsNil)
423-
time.Sleep(5 * time.Second)
424-
429+
if s.fwdMode == "routing" && encap == "vlan" {
430+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), []*container{container1})
431+
s.CheckBgpRouteDistribution(c, s.vagrant.GetNode("quagga1"), []*container{container2})
432+
}
425433
c.Assert(container1.startListener(8000, "tcp"), IsNil)
426434
c.Assert(container1.startListener(8001, "tcp"), IsNil)
427435
c.Assert(container2.startListener(8000, "tcp"), IsNil)

0 commit comments

Comments
 (0)