vagrant setup for mesos,marathon,docker and netplugin

Sukhesh Halemane · Sukhesh Halemane · commit 9c602da75559 · 2016-03-25T18:22:23.000-07:00
diff --git a/mgmtfn/mesos-docker/README.md b/mgmtfn/mesos-docker/README.md
@@ -0,0 +1,85 @@
+# Netplugin with Mesos Marathon
+
+This document explains how to use Netplugin with Mesos Marathon. Currently, netplugin supports docker containerizer with Mesos Marathon.
+
+## Getting started with Vagrant VMs
+### Prerequisits
+- Virtualbox 5.0.2 or higher
+- Vagrant 1.7.4 or higher
+- ansible 1.9.4 or higher
+
+### Step 1: Bring up the vagrant VMs
+
+```
+$ git clone https://github.com/contiv/netplugin
+$ cd netplugin/mgmtfn/mesos-docker
+$ vagrant up
+```
+
+This will bring up a two node Vagrant setup with Mesos, Marathon and docker.
+Bringing up vagrant VMs and provisioning them can take few minutes to complete since it needs to download the VM images and mesos/marathon binaries. Please be patient.
+
+### Step 2: Build netplugin binaries
+
+```
+$ vagrant ssh node1
+< Inside the VM>
+$ cd /opt/gopath/src/github.com/contiv/netplugin
+$ make host-build
+```
+
+### Step 3: Start netplugin
+
+```
+$ cd mgmtfn/mesos-docker
+$ ./startPlugin.sh
+```
+
+This will start netplugin and netmaster processes on both VMs in the setup.
+This will also create a network called `contiv` to launch marathon containers
+
+### Step 4: Launch containers
+
+`docker.json` file in mgmtfn/mesos-docker directory has an example marathon app definition.
+
+```
+
+  "container": {
+    "type": "DOCKER",
+    "docker": {
+      "image": "libmesos/ubuntu",
+      "parameters": [ { "key": "net", "value": "contiv" } ]
+    }
+  },
+  "id": "ubuntu",
+  "instances": 2,
+  "constraints": [ ["hostname", "UNIQUE", ""] ],
+  "cpus": 1,
+  "mem": 128,
+  "uris": [],
+  "cmd": "while sleep 10; do date -u +%T; done"
+}
+```
+
+This example application definition launches two ubuntu containers with a constraint that both containers be spread on different hosts.
+Note that there is a special `net` parameter used in this specification `"parameters": [ { "key": "net", "value": "contiv" } ]`. This tells docker to launch the application in contiv network that we created in step 3.
+
+You can launch this application using following command
+
+```
+$ ./launch.sh docker.json
+```
+
+Launching the container can take few minutes depending on how long it takes to pull the image.
+Once its launched, you should be able to see the containers using docker commands
+
+```
+$ docker ps
+CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS               NAMES
+2a68fed77d5a        libmesos/ubuntu     "/bin/sh -c 'while sl"   About an hour ago   Up About an hour                       mesos-cce1c91f-65fb-457d-99af-5fdd4af14f16-S1.da634e3c-1fde-479a-b100-c61a498bcbe7
+ ```
+
+## Notes
+
+ 1. Mesos and Marathon ports are port-mapped from vagrant VM to host machine. You can access them by logging into localhost:5050 and localhost:8080 respectively.
+ 2. Netmaster web-ui is port-mapped to port 9090 on the host machine
diff --git a/mgmtfn/mesos-docker/Vagrantfile b/mgmtfn/mesos-docker/Vagrantfile
@@ -0,0 +1,169 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+require 'fileutils'
+
+# netplugin_synced_gopath="/opt/golang"
+gopath_folder="/opt/gopath"
+
+ANSIBLE_GROUPS = {
+              "master" => ["node1"],
+              "nodes" => ["node2"],
+              "all_groups:children" => ["master, ""nodes"]
+            }
+
+provision_common = <<SCRIPT
+## setup the environment file. Export the env-vars passed as args to 'vagrant up'
+echo Args passed: [[ $@ ]]
+
+echo -n "$1" > /etc/hostname
+hostname -F /etc/hostname
+
+/sbin/ip addr add "$3/24" dev eth1
+/sbin/ip link set eth1 up
+/sbin/ip link set eth2 up
+
+echo 'export GOPATH=#{gopath_folder}' > /etc/profile.d/envvar.sh
+echo 'export GOBIN=$GOPATH/bin' >> /etc/profile.d/envvar.sh
+echo 'export GOSRC=$GOPATH/src' >> /etc/profile.d/envvar.sh
+echo 'export PATH=$PATH:/usr/local/go/bin:$GOBIN' >> /etc/profile.d/envvar.sh
+echo "export http_proxy='$4'" >> /etc/profile.d/envvar.sh
+echo "export https_proxy='$5'" >> /etc/profile.d/envvar.sh
+echo "export no_proxy=192.168.2.10,192.168.2.11,127.0.0.1,localhost,netmaster" >> /etc/profile.d/envvar.sh
+echo "export CLUSTER_NODE_IPS=192.168.2.10,192.168.2.11" >> /etc/profile.d/envvar.sh
+echo "export USE_RELEASE=$6" >> /etc/profile.d/envvar.sh
+
+
+source /etc/profile.d/envvar.sh
+
+# setup docker cluster store
+cp #{gopath_folder}/src/github.com/contiv/netplugin/scripts/docker.service /lib/systemd/system/docker.service
+
+# setup docker remote api
+cp #{gopath_folder}/src/github.com/contiv/netplugin/scripts/docker-tcp.socket /etc/systemd/system/docker-tcp.socket
+systemctl enable docker-tcp.socket
+
+mkdir /etc/systemd/system/docker.service.d
+echo "[Service]" | sudo tee -a /etc/systemd/system/docker.service.d/http-proxy.conf
+echo "Environment=\\\"no_proxy=192.168.2.10,192.168.2.11,127.0.0.1,localhost,netmaster\\\" \\\"http_proxy=$http_proxy\\\" \\\"https_proxy=$https_proxy\\\"" | sudo tee -a /etc/systemd/system/docker.service.d/http-proxy.conf
+sudo systemctl daemon-reload
+sudo systemctl stop docker
+systemctl start docker-tcp.socket
+sudo systemctl start docker
+
+if [ $# -gt 6 ]; then
+    shift; shift; shift; shift; shift; shift
+    echo "export $@" >> /etc/profile.d/envvar.sh
+fi
+
+
+# remove duplicate docker key
+rm /etc/docker/key.json
+
+(service docker restart) || exit 1
+
+(ovs-vsctl set-manager tcp:127.0.0.1:6640 && \
+ ovs-vsctl set-manager ptcp:6640) || exit 1
+
+docker load --input #{gopath_folder}/src/github.com/contiv/netplugin/scripts/dnscontainer.tar
+SCRIPT
+
+VAGRANTFILE_API_VERSION = "2"
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+    config.vm.box = "contiv/centos71-netplugin"
+    config.vm.box_version = "0.3.1"
+
+    num_nodes = 2
+    if ENV['CONTIV_NODES'] && ENV['CONTIV_NODES'] != "" then
+        num_nodes = ENV['CONTIV_NODES'].to_i
+    end
+    base_ip = "192.168.33."
+    node_ips = num_nodes.times.collect { |n| base_ip + "#{n+10}" }
+    node_names = num_nodes.times.collect { |n| "node#{n+1}" }
+    node_peers = []
+
+    num_nodes.times do |n|
+        node_name = node_names[n]
+        node_addr = node_ips[n]
+        node_peers += ["#{node_name}=http://#{node_addr}:2380,#{node_name}=http://#{node_addr}:7001"]
+        consul_join_flag = if n > 0 then "-join #{node_ips[0]}" else "" end
+        consul_bootstrap_flag = "-bootstrap-expect=3"
+        swarm_flag = "slave"
+        if num_nodes < 3 then
+            if n == 0 then
+                consul_bootstrap_flag = "-bootstrap"
+                swarm_flag = "master"
+            else
+                consul_bootstrap_flag = ""
+                swarm_flag = "slave"
+            end
+        end
+        config.vm.define node_name do |node|
+            # node.vm.hostname = node_name
+            # create an interface for etcd cluster
+            node.vm.network :private_network, ip: node_addr, virtualbox__intnet: "true", auto_config: false
+            # create an interface for bridged network
+            node.vm.network :private_network, ip: "0.0.0.0", virtualbox__intnet: "true", auto_config: false
+            node.vm.provider "virtualbox" do |v|
+                # make all nics 'virtio' to take benefit of builtin vlan tag
+                # support, which otherwise needs to be enabled in Intel drivers,
+                # which are used by default by virtualbox
+                v.customize ['modifyvm', :id, '--nictype1', 'virtio']
+                v.customize ['modifyvm', :id, '--nictype2', 'virtio']
+                v.customize ['modifyvm', :id, '--nictype3', 'virtio']
+                v.customize ['modifyvm', :id, '--nicpromisc2', 'allow-all']
+                v.customize ['modifyvm', :id, '--nicpromisc3', 'allow-all']
+                v.customize ['modifyvm', :id, '--paravirtprovider', "kvm"]
+            end
+
+            # mount the host directories
+            node.vm.synced_folder "../../bin", File.join(gopath_folder, "bin")
+            if ENV["GOPATH"] && ENV['GOPATH'] != ""
+              node.vm.synced_folder "../../../../../", File.join(gopath_folder, "src"), rsync: true
+            else
+              node.vm.synced_folder "../../", File.join(gopath_folder, "src/github.com/contiv/netplugin"), rsync: true
+            end
+
+            node.vm.provision "shell" do |s|
+                s.inline = "echo '#{node_ips[0]} netmaster' >> /etc/hosts; echo '#{node_addr} #{node_name}' >> /etc/hosts"
+            end
+            node.vm.provision "shell" do |s|
+                s.inline = provision_common
+                s.args = [node_name, ENV["CONTIV_NODE_OS"] || "", node_addr, ENV["http_proxy"] || "", ENV["https_proxy"] || "", ENV["USE_RELEASE"] || "", *ENV['CONTIV_ENV']]
+            end
+provision_node = <<SCRIPT
+## start etcd with generated config
+set -x
+(nohup etcd --name #{node_name} --data-dir /tmp/etcd \
+ --listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 \
+ --advertise-client-urls http://#{node_addr}:2379,http://#{node_addr}:4001 \
+ --initial-advertise-peer-urls http://#{node_addr}:2380,http://#{node_addr}:7001 \
+ --listen-peer-urls http://#{node_addr}:2380 \
+ --initial-cluster #{node_peers.join(",")} --initial-cluster-state new \
+  0<&- &>/tmp/etcd.log &) || exit 1
+
+## start consul
+(nohup consul agent -server #{consul_join_flag} #{consul_bootstrap_flag} \
+ -bind=#{node_addr} -data-dir /opt/consul 0<&- &>/tmp/consul.log &) || exit 1
+
+SCRIPT
+            node.vm.provision "shell", run: "always" do |s|
+                s.inline = provision_node
+            end
+
+            if n == (num_nodes - 1) then
+              node.vm.provision "ansible" do |ansible|
+                  ansible.playbook = "playbook.yml"
+                  ansible.groups = ANSIBLE_GROUPS
+                  ansible.limit = "all"
+              end
+            end
+            # forward netmaster port
+            if n == 0 then
+                node.vm.network "forwarded_port", guest: 5050, host: 5050
+                node.vm.network "forwarded_port", guest: 8080, host: 8080
+                node.vm.network "forwarded_port", guest: 9999, host: 9090
+            end
+        end
+    end
+end
diff --git a/mgmtfn/mesos-docker/docker.json b/mgmtfn/mesos-docker/docker.json
@@ -0,0 +1,18 @@
+{
+  "container": {
+    "type": "DOCKER",
+    "docker": {
+      "image": "libmesos/ubuntu",
+      "parameters": [
+                { "key": "net", "value": "contiv" }
+      ]
+    }
+  },
+  "id": "ubuntu",
+  "instances": 2,
+  "constraints": [ ["hostname", "UNIQUE", ""] ],
+  "cpus": 1,
+  "mem": 128,
+  "uris": [],
+  "cmd": "while sleep 10; do date -u +%T; done"
+}
diff --git a/mgmtfn/mesos-docker/launch.sh b/mgmtfn/mesos-docker/launch.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Launch a marathon job
+
+USAGE="Usage: $0 <marathon-json-file>"
+
+if [ $# -ne 1 ]; then
+    echo $USAGE
+    exit 1
+fi
+
+curl -X POST -H "Content-Type: application/json" http://localhost:8080/v2/apps -d@$1
diff --git a/mgmtfn/mesos-docker/playbook.yml b/mgmtfn/mesos-docker/playbook.yml
@@ -0,0 +1,54 @@
+---
+- hosts: master
+  remote_user: vagrant
+  become: yes
+  become_method: sudo
+  tasks:
+#      - name: upgrade system (redhat)
+#        yum:
+#          update_cache: true
+#          name: '*'
+#          state: latest
+      - name: install mesosphere yum repo
+        yum: name=http://repos.mesosphere.com/el/7/noarch/RPMS/mesosphere-el-repo-7-1.noarch.rpm state=present
+      - name: install zookeeper yum repo
+        yum: name=http://archive.cloudera.com/cdh4/one-click-install/redhat/6/x86_64/cloudera-cdh-4-0.x86_64.rpm state=present
+      - name: install zookeeper
+        yum: pkg=zookeeper,zookeeper-server state=latest
+      - name: configure zookeeper ID
+        become_user: zookeeper
+        shell: zookeeper-server-initialize --force --myid=1
+      - name: install mesos and marathon packages
+        yum: pkg=device-mapper-event-libs,mesos-0.27.0,marathon-0.14.0 state=latest
+      - name: configure containerizers
+        lineinfile: dest=/etc/mesos-slave/containerizers create=yes line="docker,mesos"
+      - name: start zookeeper
+        service: name=zookeeper-server state=started enabled=yes
+      - name: start mesos-master
+        service: name=mesos-master state=started enabled=yes
+      - name: start mesos-slave
+        service: name=mesos-slave state=started enabled=yes
+      - name: start marathon
+        service: name=marathon state=started enabled=yes
+
+
+- hosts: nodes
+  remote_user: vagrant
+  become: yes
+  become_method: sudo
+  tasks:
+#      - name: upgrade system (redhat)
+#        yum:
+#          update_cache: true
+#          name: '*'
+#          state: latest
+      - name: install mesosphere yum repo
+        yum: name=http://repos.mesosphere.com/el/7/noarch/RPMS/mesosphere-el-repo-7-1.noarch.rpm state=present
+      - name: install mesos packages
+        yum: pkg=device-mapper-event-libs,mesos-0.27.0 state=latest
+      - name: configure containerizers
+        lineinfile: dest=/etc/mesos-slave/containerizers create=yes line="docker,mesos"
+      - name: set zookeeper master
+        replace: dest=/etc/mesos/zk regexp="localhost" replace="192.168.33.10"
+      - name: start mesos-slave
+        service: name=mesos-slave state=started enabled=yes
diff --git a/mgmtfn/mesos-docker/startPlugin.sh b/mgmtfn/mesos-docker/startPlugin.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+# Start netplugin on this setup
+
+../../scripts/python/startPlugin.py -nodes 192.168.33.10,192.168.33.11
+
+# Create a network to launch docker containers
+netctl net create contiv -s 10.1.1.0/24
