setup swarm

Author: shaleman

Vagrantfile

@@ -31,11 +31,16 @@ source /etc/profile.d/envvar.sh
 mv /etc/resolv.conf /etc/resolv.conf.bak
 cp #{gopath_folder}/src/github.com/contiv/netplugin/resolv.conf /etc/resolv.conf
 
+# setup docker remote api
+cp #{gopath_folder}/src/github.com/contiv/netplugin/scripts/docker-tcp.socket /etc/systemd/system/docker-tcp.socket
+systemctl enable docker-tcp.socket
+
 mkdir /etc/systemd/system/docker.service.d
 echo "[Service]" | sudo tee -a /etc/systemd/system/docker.service.d/http-proxy.conf
 echo "Environment=\\\"no_proxy=192.168.0.0/16,localhost,127.0.0.0/8\\\" \\\"http_proxy=$http_proxy\\\" \\\"https_proxy=$https_proxy\\\"" | sudo tee -a /etc/systemd/system/docker.service.d/http-proxy.conf
 sudo systemctl daemon-reload
 sudo systemctl stop docker
+systemctl start docker-tcp.socket
 sudo systemctl start docker
 
 if [ $# -gt 5 ]; then
@@ -46,6 +51,13 @@ fi
 # Install experimental docker
 # curl -sSL https://experimental.docker.com/ | sh
 
+# Get swarm binary
+(wget https://cisco.box.com/shared/static/isn9te95op8qgmz9o0al45eps2zmfz7c -q -O /usr/bin/swarm &&
+chmod +x /usr/bin/swarm) || exit 1
+
+# remove duplicate docker key
+rm /etc/docker/key.json
+
 (service docker restart) || exit 1
 
 (ovs-vsctl set-manager tcp:127.0.0.1:6640 && \
@@ -122,10 +134,17 @@ set -x
 ## start consul
 (nohup consul agent -server #{consul_join_flag} #{consul_bootstrap_flag} \
  -bind=#{node_addr} -data-dir /opt/consul 0<&- &>/tmp/consul.log &) || exit 1
+
+# start swarm
+(nohup #{gopath_folder}/src/github.com/contiv/netplugin/scripts/start-swarm.sh #{node_addr} > /tmp/start-swarm.log &) || exit 1
+
 SCRIPT
             node.vm.provision "shell", run: "always" do |s|
                 s.inline = provision_node
             end
+
+            # forward netmaster port
+            # node.vm.network "forwarded_port", guest: 9999, host: 8080 + n
         end
     end
 end

netmaster/main.go

@@ -177,13 +177,29 @@ func (d *daemon) execOpts() {
 }
 
 func (d *daemon) ListenAndServe() {
-	r := mux.NewRouter()
+	router := mux.NewRouter()
 
 	// Create a new api controller
-	d.apiController = objApi.NewAPIController(r)
+	d.apiController = objApi.NewAPIController(router)
+
+	// Setup the router to serve the web UI
+	goPath := os.Getenv("GOPATH")
+	if goPath != "" {
+		webPath := goPath + "/src/github.com/contiv/objmodel/contivModel/www/"
+
+		log.Infof("Using webPath: %s", webPath)
+
+		// serve static files
+		router.PathPrefix("/web/").Handler(http.StripPrefix("/web/", http.FileServer(http.Dir(webPath))))
+
+		// Special case to serve main index.html
+		router.HandleFunc("/", func(rw http.ResponseWriter, req *http.Request) {
+			http.ServeFile(rw, req, webPath+"index.html")
+		})
+	}
 
 	// Add REST routes
-	s := r.Headers("Content-Type", "application/json").Methods("Post").Subrouter()
+	s := router.Headers("Content-Type", "application/json").Methods("Post").Subrouter()
 	s.HandleFunc(fmt.Sprintf("/%s", master.DesiredConfigRESTEndpoint),
 		post(d.desiredConfig))
 	s.HandleFunc(fmt.Sprintf("/%s", master.AddConfigRESTEndpoint),
@@ -193,7 +209,7 @@ func (d *daemon) ListenAndServe() {
 	s.HandleFunc(fmt.Sprintf("/%s", master.HostBindingConfigRESTEndpoint),
 		post(d.hostBindingsConfig))
 
-	s = r.Methods("Get").Subrouter()
+	s = router.Methods("Get").Subrouter()
 	s.HandleFunc(fmt.Sprintf("/%s/%s", master.GetEndpointRESTEndpoint, "{id}"),
 		get(false, d.endpoints))
 	s.HandleFunc(fmt.Sprintf("/%s", master.GetEndpointsRESTEndpoint),
@@ -205,7 +221,7 @@ func (d *daemon) ListenAndServe() {
 
 	log.Infof("Netmaster listening on %s", d.opts.listenURL)
 
-	if err := http.ListenAndServe(d.opts.listenURL, r); err != nil {
+	if err := http.ListenAndServe(d.opts.listenURL, router); err != nil {
 		log.Fatalf("Error listening for http requests. Error: %s", err)
 	}
 }

scripts/docker-tcp.socket

@@ -0,0 +1,10 @@
+[Unit]
+Description=Docker Socket for the API
+
+[Socket]
+ListenStream=2385
+BindIPv6Only=both
+Service=docker.service
+
+[Install]
+WantedBy=sockets.target

scripts/python/dummy.py

@@ -196,17 +196,18 @@ def testPolicyAddDeleteRule(testbed, numContainer, numIter):
 # Setup testbed
 testbed = testbedApi.testbed(addrList)
 
-time.sleep(5)
+time.sleep(15)
 
 numCntr = testbed.numNodes() * 2
+numIteration = 2
 
 # Run the test
-testBasicPolicy(testbed, numCntr, 4)
-testPolicyAddDeleteRule(testbed, numCntr, 4)
-startRemoveContainer(testbed, numCntr, 4)
+testBasicPolicy(testbed, numCntr, numIteration)
+testPolicyAddDeleteRule(testbed, numCntr, numIteration)
+startRemoveContainer(testbed, numCntr, numIteration)
 
-# FIXME: This test is not passing yet
-#startStopContainer(testbed, numCntr, 4)
+# FIXME: This test is not passing yet as native-integ mode cleanup is not working
+# startStopContainer(testbed, numCntr, 4)
 
 # Cleanup testbed
 testbed.cleanup()

scripts/python/objmodelTest.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python
+
+# Start netplugin and netmaster
+import testbedApi
+import time
+import sys
+import os
+
+# Parse command line args
+if len(sys.argv) <= 1:
+	print "Usage: " + sys.argv[0] + " <ip-addr> <ip-addr>..."
+	print "This starts netplugin and netmaster"
+	sys.exit(1)
+
+# Form address list
+addrList = []
+for addr in sys.argv[1:]:
+	addrList.append(addr)
+
+# Cleanup all state and start netplugin/netmaster
+testbed = testbedApi.testbed(addrList)
+time.sleep(2)
+
+os._exit(0)

scripts/python/sanity.py

@@ -57,7 +57,7 @@ def runCmd(self, cmd, timeout=None):
     # Start netplugin process on vagrant node
     def startNetplugin(self):
         ssh_object = self.sshConnect()
-        command = "sudo /opt/gopath/bin/netplugin > /tmp/netplugin.log 2>&1"
+        command = "sudo /opt/gopath/bin/netplugin -native-integration=true > /tmp/netplugin.log 2>&1"
         self.npThread = threading.Thread(target=ssh_exec_thread, args=(ssh_object, command))
         # npThread.setDaemon(True)
         self.npThread.start()
@@ -129,6 +129,9 @@ def __init__(self, addrList):
             # Cleanup all state before we can start
             node.stopNetmaster()
             node.stopNetplugin()
+
+        # cleanup master and slave state
+        for node in self.nodes:
             node.cleanupMaster()
             node.cleanupSlave()
 
@@ -143,17 +146,17 @@ def __init__(self, addrList):
 
     # Cleanup a testbed once test is done
     def cleanup(self):
-        # cleanup master
-        print "Cleaning up master"
-        self.nodes[0].stopNetmaster()
-        self.nodes[0].cleanupMaster()
-
         # Cleanup each node
         for node in self.nodes:
             print "Cleaning up node " + node.addr
             node.stopNetplugin()
             node.cleanupSlave()
 
+        # cleanup master
+        print "Cleaning up master"
+        self.nodes[0].stopNetmaster()
+        self.nodes[0].cleanupMaster()
+
     # Number of nodes in the testbed
     def numNodes(self):
         return len(self.nodes)

scripts/python/startPlugin.py

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+USAGE="Usage: $0 <node-addr>"
+
+if [ $# -ne 1 ]; then
+    echo $USAGE
+    exit 1
+fi
+
+node_addr=$1
+
+echo "start-swarm waiting for etcd"
+
+etcdctl cluster-health
+while [[ `etcdctl cluster-health | head -n 1` != "cluster is healthy" ]]; do
+    echo "Cluster is unhealthy";
+    sleep 1;
+    etcdctl cluster-health;
+done
+
+echo "starting swarm on " $node_addr
+
+# Start swarm discovery
+nohup /usr/bin/swarm join --advertise=$node_addr:2385 etcd://localhost:2379 > /tmp/swarm-join.log 2>&1 &
+
+# Start swarm manager
+nohup /usr/bin/swarm manage -H tcp://$node_addr:2375 etcd://localhost:2379 > /tmp/swarm-manage.log 2>&1 &