diff --git a/Makefile b/Makefile index 26c357df5..5c71aecf6 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,12 @@ demo: build clean-demo: vagrant destroy -f +start-dockerdemo: + scripts/dockerhost/start-dockerhosts + +clean-dockerdemo: + scripts/dockerhost/cleanup-dockerhosts + unit-test: build CONTIV_HOST_GOBIN=$(HOST_GOBIN) CONTIV_HOST_GOROOT=$(HOST_GOROOT) ./scripts/unittests -vagrant diff --git a/README.md b/README.md index 35bafbc7e..ebf8f4d3e 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,9 @@ In the examples directory [two_hosts_multiple_tenants.json](examples/two_hosts_m [two_hosts_multiple_tenants_mix_vlan_vxlan.json](examples/two_hosts_multiple_tenants_mix_vlan_vxlan.json) shows the creation of a multi-tenant (disjoint, overlapping) networks within a cluster. +####Trying the multi-host tests on a single machine using docker as hosts +If you cannot launch VM on your host, especially if your host is itself a VM, one can test the multi-host network by simulating hosts using docker containers. Please see [docs/Dockerhost.md](docs/Dockerhost.md) for instructions. + ####Auto-allocation of IP addresses The plugin can automatically manage the IP address pools and assign an appropriate IP address based on the subnet that was associated with the network. However this doesn't take away the flexibility to keep a specific IP address of a container, which can always be specified as shown earlier. To automatically allocate the IP address, just avoid specifying the IP address during endpoint creation or endpoint description diff --git a/docs/Dockerhost.md b/docs/Dockerhost.md new file mode 100644 index 000000000..8a0931f57 --- /dev/null +++ b/docs/Dockerhost.md @@ -0,0 +1,74 @@ +The multi node container networking can be tested on a single VM by running docker inside docker. +https://blog.docker.com/2013/09/docker-can-now-run-within-docker/ + +The outside docker containers act like physical hosts in our test and are connected using a standard linux bridge. Inside each "host container" we run a namespaced instance of docker, OVS , etcd and netplugin instance. One can now launch containers from within each "host containers" and use netplugin networking to connect them. + +Prerequisites +------------- +The following needs to be installed on the host machine +1. Docker +2. nsenter (Pls see https://github.com/jpetazzo/nsenter) +3. Bridge utils +``` + apt-get install bridge-utils +```` + +Step to launch docker hosts are : +-------------------------------- +``` + make build + CONTINV_NODES=2 make start-dockerdemo +``` + +This will start CONTIV_NODES number of containers with docker image called ubuntu_netplugin which is just ubuntu image with docker, etcd and ovs installed. + +Now start a shell within any of the "host containers" using following convenience wrapper around nsenter : +``` +scripts/dockerhost/docker-sh netplugin-node +``` + +Start netplugin, post netplugin config and launch containers inside the "host containers" the same way you do on VMs. +Note : currently the demo is working only if config is posted before containers are started .. need to debug why the reverse is not working. + +To cleanup all the docker hosts and the virtual interfaces created do + ``` + make cleanup-dockerdemo + ``` + +Example for testing TwoHostMultiVlan you can do : + +1. Launch the two host containers + + ``` + export CONTIV_NODES=2 + make start-dockerdemo + ``` + +2. Load the netplugin configuration + ``` + docker-sh netplugin-node1 + /netplugin/bin/netdcli -cfg /netplugin/examples/two_hosts_multiple_vlans_nets.json + ``` + +3. Launch container1 on host1 + + ``` + docker-sh netplugin-node1 + docker run -it --name=myContainer1 --hostname=myContainer1 ubuntu /bin/bash + ``` + +4. Launch container3 on host2 + + ``` + docker-sh netplugin-node2 + docker run -it --name=myContainer3 --hostname=myContainer1 ubuntu /bin/bash + ``` + +5. Test connectivity between the containers using ping. Go to the shell for container1 + ``` + root@myContainer1:/# ping -c3 11.1.2.2 +PING 11.1.2.2 (11.1.2.2) 56(84) bytes of data. +64 bytes from 11.1.2.2: icmp_seq=1 ttl=64 time=3.15 ms +64 bytes from 11.1.2.2: icmp_seq=2 ttl=64 time=1.36 ms + ``` + diff --git a/scripts/dockerhost/Dockerfile b/scripts/dockerhost/Dockerfile new file mode 100644 index 000000000..9d5c7b7dd --- /dev/null +++ b/scripts/dockerhost/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu +MAINTAINER Sachin Jain + +RUN apt-get update && apt-get install -y curl + +RUN cd /tmp && \ +curl -L https://github.com/coreos/etcd/releases/download/v2.0.0/etcd-v2.0.0-linux-amd64.tar.gz -o etcd-v2.0.0-linux-amd64.tar.gz && \ +tar -xzf etcd-v2.0.0-linux-amd64.tar.gz && \ +cd /usr/bin && \ +ln -s /tmp/etcd-v2.0.0-linux-amd64/etcd && \ +ln -s /tmp/etcd-v2.0.0-linux-amd64/etcdctl + +RUN curl -sSL https://get.docker.com/ubuntu/ | sh > /dev/null + +RUN apt-get install -y openvswitch-switch diff --git a/scripts/dockerhost/cleanup-dockerhosts b/scripts/dockerhost/cleanup-dockerhosts new file mode 100755 index 000000000..bf775e8fd --- /dev/null +++ b/scripts/dockerhost/cleanup-dockerhosts @@ -0,0 +1,18 @@ + +#!/bin/bash + +num_nodes=1 +if [ -n "$CONTIV_NODES" ]; +then + num_nodes=$CONTIV_NODES +fi +echo $num_nodes + +for i in `seq 1 $num_nodes`; +do + hostname="netplugin-node$i" + echo "Cleaning $hostname" + sudo docker exec $hostname service docker stop + sudo docker rm -f $hostname + sudo ip link delete $i-ext +done diff --git a/scripts/dockerhost/create_loop_devices.sh b/scripts/dockerhost/create_loop_devices.sh new file mode 100755 index 000000000..440f85e4b --- /dev/null +++ b/scripts/dockerhost/create_loop_devices.sh @@ -0,0 +1,6 @@ +#!/bin/bash +for i in `seq 9 20`; +do + sudo /bin/mknod -m640 /dev/loop$i b 7 $i + sudo /bin/chown root:disk /dev/loop$i +done diff --git a/scripts/dockerhost/docker-ip b/scripts/dockerhost/docker-ip new file mode 100755 index 000000000..0b40229cc --- /dev/null +++ b/scripts/dockerhost/docker-ip @@ -0,0 +1,3 @@ +#!/bin/sh + +sudo docker inspect --format '{{ .NetworkSettings.IPAddress }}' "$@" diff --git a/scripts/dockerhost/docker-pid b/scripts/dockerhost/docker-pid new file mode 100755 index 000000000..976fe66d8 --- /dev/null +++ b/scripts/dockerhost/docker-pid @@ -0,0 +1,3 @@ +#!/bin/sh + +sudo docker inspect --format '{{ .State.Pid }}' "$@" diff --git a/scripts/dockerhost/docker-sh b/scripts/dockerhost/docker-sh new file mode 100755 index 000000000..798e2db3d --- /dev/null +++ b/scripts/dockerhost/docker-sh @@ -0,0 +1,3 @@ +#!/bin/bash +scriptdir=`dirname "$BASH_SOURCE"` +sudo nsenter --target $($scriptdir/docker-pid $1) --mount --uts --ipc --net --pid diff --git a/scripts/dockerhost/start-dockerhosts b/scripts/dockerhost/start-dockerhosts new file mode 100755 index 000000000..0bb3399cd --- /dev/null +++ b/scripts/dockerhost/start-dockerhosts @@ -0,0 +1,60 @@ +#!/bin/bash + +scriptdir=`dirname "$BASH_SOURCE"` +echo $scriptdir + +# Create a linux bridge between containers +brctl addbr br-em1 +ip link set br-em1 up + +num_nodes=1 +if [ -n "$CONTIV_NODES" ]; +then + num_nodes=$CONTIV_NODES +fi +echo "Num nodes = "$num_nodes + +netplugin_path=$(pwd) + +# Create the docker image for hosts +sudo docker build -t ubuntu_netplugin $scriptdir + +cluster="" +first="true" +for i in `seq 1 $num_nodes`; +do + hostname="netplugin-node$i" + echo "Starting $hostname" + sudo docker run -d -i -t --name $hostname --privileged -v /var/lib/docker -v $netplugin_path:/netplugin ubuntu_netplugin bash -c "/netplugin/scripts/dockerhost/start-service.sh & bash" + sudo nsenter -t $($scriptdir/docker-pid $hostname) -n hostname $hostname + sudo ip link add $i-int type veth peer name $i-ext + sudo brctl addif br-em1 $i-ext + sudo ip link set netns $($scriptdir/docker-pid $hostname) dev $i-int + sudo nsenter -t $($scriptdir/docker-pid $hostname) -n ip link set dev $i-int name eth2 + sudo nsenter -t $($scriptdir/docker-pid $hostname) -n ip link set eth2 up + if [ $i -gt "1" ]; then + cluster=$cluster","; + fi + sudo docker exec $hostname service docker start + addr=$($scriptdir/docker-ip $hostname) + cluster=$cluster$hostname"=http://"$addr":2380" + first="false" +done + +echo $cluster + +for i in `seq 1 $num_nodes`; +do + hostname="netplugin-node$i" + echo "Starting etcd on $hostname" + addr=$($scriptdir/docker-ip $hostname) + sudo docker exec $hostname etcd -name $hostname -data-dir /opt/etcd -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -advertise-client-urls http://$addr:2379,http://$addr:4001 -initial-advertise-peer-urls http://$addr:2380 -listen-peer-urls http://$addr:2380 -initial-cluster $cluster -initial-cluster-state new & +done + +for i in `seq 1 $num_nodes`; +do + hostname="netplugin-node$i" + echo "Starting netplugin on $hostname" + sudo docker exec $hostname /netplugin/bin/netplugin -host-label host$i & +done + diff --git a/scripts/dockerhost/start-service.sh b/scripts/dockerhost/start-service.sh new file mode 100755 index 000000000..771b4f5cd --- /dev/null +++ b/scripts/dockerhost/start-service.sh @@ -0,0 +1,9 @@ +#!/bin/bash +/etc/init.d/openvswitch-switch restart +ovs-vsctl set-manager tcp:127.0.0.1:6640 +ovs-vsctl set-manager ptcp:6640 +while true +do + echo service is running >> service.log + sleep 10 +done