Add a node reload test

Add node reload test; fix aci breakage

Fix formatting

minor fixes in system tests

add markers for test start/end

Author: shaleman

Vagrantfile

@@ -8,14 +8,12 @@ gopath_folder="/opt/gopath"
 
 cluster_ip_nodes = ""
 
-provision_common = <<SCRIPT
+provision_common_once = <<SCRIPT
 ## setup the environment file. Export the env-vars passed as args to 'vagrant up'
 echo Args passed: [[ $@ ]]
 echo -n "$1" > /etc/hostname
 hostname -F /etc/hostname
-/sbin/ip addr add "$2/24" dev eth1
-/sbin/ip link set eth1 up
-/sbin/ip link set eth2 up
+
 echo 'export GOPATH=#{gopath_folder}' > /etc/profile.d/envvar.sh
 echo 'export GOBIN=$GOPATH/bin' >> /etc/profile.d/envvar.sh
 echo 'export GOSRC=$GOPATH/src' >> /etc/profile.d/envvar.sh
@@ -32,15 +30,8 @@ if [[ $# -gt 9 ]] && [[ $10 != "" ]]; then
     echo "export $@" >> /etc/profile.d/envvar.sh
 fi
 
-# Enable ovs mgmt port
-(ovs-vsctl set-manager tcp:127.0.0.1:6640 && \
- ovs-vsctl set-manager ptcp:6640) || exit 1
-
- # Drop cache to workaround vboxsf problem
- echo 3 > /proc/sys/vm/drop_caches
-
- # Change ownership for gopath folder
- chown vagrant #{gopath_folder}
+# Change ownership for gopath folder
+chown vagrant #{gopath_folder}
 
 # Install specific docker version if required
 if [[ $8 != "" ]]; then
@@ -65,7 +56,7 @@ cp #{gopath_folder}/src/github.com/contiv/netplugin/scripts/docker-tcp.socket /e
 systemctl enable docker-tcp.socket
 mkdir /etc/systemd/system/docker.service.d
 echo "[Service]" | sudo tee -a /etc/systemd/system/docker.service.d/http-proxy.conf
-echo "Environment=\\\"no_proxy=$3,127.0.0.1,localhost,netmaster\\\" \\\"http_proxy=$http_proxy\\\" \\\"https_proxy=$https_proxy\\\"" | sudo tee -a /etc/systemd/system/docker.service.d/http-proxy.conf
+echo "Environment=\\\"no_proxy=$CLUSTER_NODE_IPS,127.0.0.1,localhost,netmaster\\\" \\\"http_proxy=$http_proxy\\\" \\\"https_proxy=$https_proxy\\\"" | sudo tee -a /etc/systemd/system/docker.service.d/http-proxy.conf
 sudo systemctl daemon-reload
 sudo systemctl stop docker
 systemctl start docker-tcp.socket
@@ -79,6 +70,19 @@ docker load --input #{gopath_folder}/src/github.com/contiv/netplugin/scripts/dns
 
 SCRIPT
 
+provision_common_always = <<SCRIPT
+/sbin/ip addr add "$2/24" dev eth1
+/sbin/ip link set eth1 up
+/sbin/ip link set eth2 up
+
+# Drop cache to workaround vboxsf problem
+echo 3 > /proc/sys/vm/drop_caches
+
+# Enable ovs mgmt port
+(ovs-vsctl set-manager tcp:127.0.0.1:6640 && \
+ ovs-vsctl set-manager ptcp:6640) || exit 1
+SCRIPT
+
 provision_gobgp = <<SCRIPT
 #Get gobgp binary
 wget https://cisco.box.com/shared/static/5leqlo84kjh0thty91ouotilm4ish3nz -q -O #{gopath_folder}/bin/gobgp && chmod +x #{gopath_folder}/bin/gobgp
@@ -92,6 +96,7 @@ echo "export https_proxy='$2'" >> /etc/profile.d/envvar.sh
 source /etc/profile.d/envvar.sh
 SCRIPT
 
+
 VAGRANTFILE_API_VERSION = "2"
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     if ENV['CONTIV_NODE_OS'] && ENV['CONTIV_NODE_OS'] == "ubuntu" then
@@ -108,6 +113,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         num_nodes = ENV['CONTIV_NODES'].to_i
     end
     base_ip = "192.168.2."
+    if ENV['CONTIV_IP_PREFIX'] && ENV['CONTIV_IP_PREFIX'] != "" then
+        base_ip = ENV['CONTIV_IP_PREFIX']
+    end
     node_ips = num_nodes.times.collect { |n| base_ip + "#{n+10}" }
     cluster_ip_nodes = node_ips.join(",")
 
@@ -118,7 +126,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
         quagga1.vm.box = "contiv/quagga1"
         quagga1.vm.host_name = "quagga1"
-        quagga1.vm.network :private_network, ip: "192.168.1.50", virtualbox__intnet: "true", auto_config: false
+        quagga1.vm.network :private_network, ip: base_ip + "51", virtualbox__intnet: "true", auto_config: false
         quagga1.vm.network "private_network",
                          ip: "80.1.1.200",
                          virtualbox__intnet: "contiv_orange"
@@ -134,7 +142,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
         quagga2.vm.box = "contiv/quagga2"
         quagga2.vm.host_name = "quagga2"
-        quagga2.vm.network :private_network, ip: "192.168.1.50", virtualbox__intnet: "true", auto_config: false
+        quagga2.vm.network :private_network, ip: base_ip + "52", virtualbox__intnet: "true", auto_config: false
         quagga2.vm.network "private_network",
                          ip: "70.1.1.1",
                          virtualbox__intnet: "contiv_blue"
@@ -215,14 +223,19 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
                 s.inline = "echo '#{node_ips[0]} netmaster' >> /etc/hosts; echo '#{node_addr} #{node_name}' >> /etc/hosts"
             end
             node.vm.provision "shell" do |s|
-                s.inline = provision_common
+                s.inline = provision_common_once
                 s.args = [node_name, node_addr, cluster_ip_nodes, ENV["http_proxy"] || "", ENV["https_proxy"] || "", ENV["USE_RELEASE"] || "", ENV["CONTIV_CLUSTER_STORE"] || "etcd://localhost:2379", ENV["CONTIV_DOCKER_VERSION"] || "", ENV['CONTIV_NODE_OS'] || "", *ENV['CONTIV_ENV']]
             end
             if ENV['CONTIV_L3'] then
                 node.vm.provision "shell" do |s|
                     s.inline = provision_gobgp
                 end
             end
+            node.vm.provision "shell", run: "always" do |s|
+                s.inline = provision_common_always
+                s.args = [node_name, node_addr]
+            end
+
 provision_node = <<SCRIPT
 ## start etcd with generated config
 set -x

netmaster/daemon.go

@@ -19,7 +19,6 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-	"os"
 	"sync"
 
 	"github.com/contiv/netplugin/core"
@@ -132,44 +131,8 @@ func (d *daemon) registerNetpluginNodes() error {
 	return nil
 }
 
-// registerWebuiHandler registers handlers for serving web UI
-func (d *daemon) registerWebuiHandler(router *mux.Router) {
-	// Setup the router to serve the web UI
-	goPath := os.Getenv("GOPATH")
-	if goPath != "" {
-		webPath := goPath + "/src/github.com/contiv/contivmodel/www/"
-
-		// Make sure we have the web UI files
-		_, err := os.Stat(webPath)
-		if err != nil {
-			webPath = goPath + "/src/github.com/contiv/netplugin/" +
-				"Godeps/_workspace/src/github.com/contiv/contivmodel/www/"
-			_, err := os.Stat(webPath)
-			if err != nil {
-				log.Errorf("Can not find the web UI directory")
-			}
-		}
-
-		log.Infof("Using webPath: %s", webPath)
-
-		// serve static files
-		router.PathPrefix("/web/").Handler(http.StripPrefix("/web/", http.FileServer(http.Dir(webPath))))
-
-		// Special case to serve main index.html
-		router.HandleFunc("/", func(rw http.ResponseWriter, req *http.Request) {
-			http.ServeFile(rw, req, webPath+"index.html")
-		})
-	}
-
-	// proxy Handler
-	router.PathPrefix("/proxy/").HandlerFunc(proxyHandler)
-}
-
 // registerRoutes registers HTTP route handlers
 func (d *daemon) registerRoutes(router *mux.Router) {
-	// register web ui handlers
-	d.registerWebuiHandler(router)
-
 	// Add REST routes
 	s := router.Headers("Content-Type", "application/json").Methods("Post").Subrouter()
 
@@ -218,7 +181,7 @@ func (d *daemon) endpoints(id string) ([]core.State, error) {
 	return nil, core.Errorf("Unexpected code path. Recieved error during read: %v", err)
 }
 
-// XXX: This function should be returning logical state instead of driver state
+// Returns state of networks
 func (d *daemon) networks(id string) ([]core.State, error) {
 	var (
 		err error

netmaster/objApi/infraproxy.go

@@ -106,19 +106,15 @@ func appendEpgInfo(eMap *epgMap, epgObj *contivModel.EndpointGroup, stateDriver
 	epg := epgSpec{}
 	epg.Name = epgObj.GroupName
 
-	// Get EPG id for the endpoint group
-	epgID, err := mastercfg.GetEndpointGroupID(stateDriver, epgObj.GroupName, epgObj.TenantName)
-	if err != nil {
-		log.Errorf("Error getting epgID for %+v. Err: %v", epgObj, err)
-		return err
-	}
+	// Get EPG key for the endpoint group
+	epgKey := mastercfg.GetEndpointGroupKey(epgObj.GroupName, epgObj.TenantName)
 
 	// update vlantag from EpGroupState
 	epgCfg := &mastercfg.EndpointGroupState{}
 	epgCfg.StateDriver = stateDriver
-	eErr := epgCfg.Read(strconv.Itoa(epgID))
+	eErr := epgCfg.Read(epgKey)
 	if eErr != nil {
-		log.Errorf("Error reading epg %v %v", epgObj.GroupName, eErr)
+		log.Errorf("Error reading epg %v %v", epgKey, eErr)
 		return eErr
 	}
 

netmaster/utils.go

@@ -21,7 +21,6 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
-	"strings"
 
 	"github.com/contiv/netplugin/core"
 	"github.com/contiv/netplugin/version"
@@ -47,27 +46,6 @@ func getVersion(w http.ResponseWriter, r *http.Request) {
 	return
 }
 
-// proxyHandler acts as a simple reverse proxy to access containers via http
-func proxyHandler(w http.ResponseWriter, r *http.Request) {
-	proxyURL := strings.TrimPrefix(r.URL.Path, "/proxy/")
-	log.Infof("proxy handler for %q : %s", r.URL.Path, proxyURL)
-
-	// build the proxy url
-	url, _ := url.Parse("http://" + proxyURL)
-
-	// Create a proxy for the URL
-	proxy := httputil.NewSingleHostReverseProxy(url)
-
-	// modify the request url
-	newReq := *r
-	newReq.URL = url
-
-	log.Debugf("Proxying request(%v): %+v", url, newReq)
-
-	// Serve http
-	proxy.ServeHTTP(w, &newReq)
-}
-
 // slaveProxyHandler redirects to current master
 func slaveProxyHandler(w http.ResponseWriter, r *http.Request) {
 	log.Infof("proxy handler for %q ", r.URL.Path)

systemtests/basic_test.go

@@ -115,7 +115,7 @@ func (s *systemtestSuite) TestBasicSvcDiscoveryVLAN(c *C) {
 }
 
 func (s *systemtestSuite) testBasicSvcDiscovery(c *C, encap string) {
-	if !strings.Contains(s.clusterStore, "etcd") || !s.enableDNS {
+	if !strings.Contains(s.clusterStore, "etcd") {
 		c.Skip("Skipping test")
 	}
 

systemtests/init_test.go

@@ -2,6 +2,7 @@ package systemtests
 
 import (
 	"flag"
+	"fmt"
 	"os"
 	"strconv"
 	"strings"
@@ -115,6 +116,7 @@ func (s *systemtestSuite) SetUpSuite(c *C) {
 }
 
 func (s *systemtestSuite) SetUpTest(c *C) {
+	log.Infof("============================= %s starting ==========================", c.TestName())
 
 	for _, node := range s.nodes {
 		node.cleanupContainers()
@@ -143,6 +145,13 @@ func (s *systemtestSuite) SetUpTest(c *C) {
 
 	time.Sleep(15 * time.Second)
 
+	// temporarily enable DNS for service discovery tests
+	prevDNSEnabled := s.enableDNS
+	if strings.Contains(c.TestName(), "SvcDiscovery") {
+		s.enableDNS = true
+	}
+	defer func() { s.enableDNS = prevDNSEnabled }()
+
 	for _, node := range s.nodes {
 		c.Assert(node.startNetmaster(), IsNil)
 		time.Sleep(1 * time.Second)
@@ -168,6 +177,7 @@ func (s *systemtestSuite) TearDownTest(c *C) {
 		c.Assert(node.rotateLog("netplugin"), IsNil)
 		c.Assert(node.rotateLog("netmaster"), IsNil)
 	}
+	log.Infof("============================= %s completed ==========================", c.TestName())
 }
 
 func (s *systemtestSuite) TearDownSuite(c *C) {
@@ -178,9 +188,10 @@ func (s *systemtestSuite) TearDownSuite(c *C) {
 	// Print all errors and fatal messages
 	for _, node := range s.nodes {
 		logrus.Infof("Checking for errors on %v", node.Name())
-		out, _ := node.runCommand(`for i in /tmp/_net*; do grep "error\|fatal" $i; done`)
+		out, _ := node.runCommand(`for i in /tmp/_net*; do grep "error\|fatal\|panic" $i; done`)
 		if out != "" {
-			logrus.Errorf("Errors in logfiles on %s: \n%s\n==========================\n\n", node.Name(), out)
+			logrus.Errorf("Errors in logfiles on %s: \n", node.Name())
+			fmt.Printf("%s\n==========================\n\n", out)
 		}
 	}
 

systemtests/node_test.go

@@ -2,6 +2,7 @@ package systemtests
 
 import (
 	"fmt"
+	"os/exec"
 	"regexp"
 	"strings"
 	"time"
@@ -194,10 +195,19 @@ func (n *node) runContainer(spec containerSpec) (*container, error) {
 }
 
 func (n *node) checkForNetpluginErrors() error {
-	out, _ := n.tbnode.RunCommandWithOutput(`for i in /tmp/net*; do grep "error|fatal" $i; done`)
+	out, _ := n.tbnode.RunCommandWithOutput(`for i in /tmp/net*; do grep "panic\|fatal" $i; done`)
 	if out != "" {
-		logrus.Errorf("error output in netplugin logs on %s: \n%s\n", n.Name(), out)
-		return fmt.Errorf("error output in netplugin logs")
+		logrus.Errorf("Fatal error in logs on %s: \n", n.Name())
+		fmt.Printf("%s\n==========================================\n", out)
+		return fmt.Errorf("fatal error in netplugin logs")
+	}
+
+	out, _ = n.tbnode.RunCommandWithOutput(`for i in /tmp/net*; do grep "error" $i; done`)
+	if out != "" {
+		logrus.Errorf("error output in netplugin logs on %s: \n", n.Name())
+		fmt.Printf("%s==========================================\n\n", out)
+		// FIXME: We still have some tests that are failing error check
+		// return fmt.Errorf("error output in netplugin logs")
 	}
 
 	return nil
@@ -227,3 +237,16 @@ func (n *node) checkPing(ipaddr string) error {
 	logrus.Infof("Ping from %s to %s SUCCEEDED", n.Name(), ipaddr)
 	return nil
 }
+
+func (n *node) reloadNode() error {
+	logrus.Infof("Reloading node %s", n.Name())
+
+	out, err := exec.Command("vagrant", "reload", n.Name()).CombinedOutput()
+	if err != nil {
+		logrus.Errorf("Error reloading node %s. Err: %v\n Output: %s", n.Name(), err, string(out))
+		return err
+	}
+
+	logrus.Infof("Reloaded node %s. Output:\n%s", n.Name(), string(out))
+	return nil
+}