Build versioned v2plugin from versioned binaries

This enables build and caching locally or in a build machine the
v2plugin before creating test environment VMs and failing much faster
on any PR issues, which is good for CI turnaround.

v2plugin_rootfs (called by host-pluginfs-create) builds a docker
container runtime with a versioned archive of netplugin binaries, then
extracts the entire filesystem from the container, which is now left as
a versioned tarball.

The versioned archive of netplugin binaries used for v2plugin can now be
created in two ways:
* run-build (dependency for demo-v2plugin) will compile then version an
  archive with built assets (runs only on the VM)
* tar target (locally or in VM)

host-pluginfs-create can be run in the VM or locally

Leaving the plugin rootfs as a tarball has a few advantages reasons:
* docker in a VM had internal issues when creating the plugin when the
  rootfs was unarchived on the host and exposed though a virtualbox
  mount, at least on OS X
* the rootfs is versioned

This adjusted host-plugin-release and demo-v2plugin to both use a new
target 'host-pluginfs-unpack'

Drive-by:
* Fix Dockerfile by adding objdb which is needed by vendored packages
* Fix config.template for the CONTIV_V2PLUGIN_NAME replacement
* gtar support for OS X
* use cp -a for maintaining owner

Signed-off-by: Chris Plock <[email protected]>

Author: chrisplo

.gitignore

@@ -43,9 +43,10 @@ cscope*
 .out.vagrant
 .etc_hosts
 
-# docker v2 plugin temp files
+# docker v2 plugin files
 install/v2plugin/rootfs
 install/v2plugin/config.json
+install/v2plugin/v2plugin-*.tar.gz
 
 # netcontain temporary build files
 scripts/netContain/bin

Dockerfile

@@ -37,6 +37,7 @@ CMD ["--help"]
 # by far, most of the compilation time is building vendor packages
 # build the vendor dependencies as a separate docker caching layer
 COPY ./vendor/ /go/src/github.com/contiv/netplugin/vendor/
+COPY ./objdb/ /go/src/github.com/contiv/netplugin/objdb/
 
 RUN GOGC=1500 go install -ldflags "-s -w" $(go list ./vendor/...)
 

Makefile

@@ -6,7 +6,6 @@
 .PHONY: all all-CI build clean default unit-test release tar checks go-version gofmt-src \
 	golint-src govet-src run-build compile-with-docker
 
-DEFAULT_DOCKER_VERSION := 1.12.6
 SHELL := /bin/bash
 EXCLUDE_DIRS := bin docs Godeps scripts vagrant vendor install
 PKG_DIRS := $(filter-out $(EXCLUDE_DIRS),$(subst /,,$(sort $(dir $(wildcard */)))))
@@ -18,19 +17,24 @@ NAME := netplugin
 # we evaluate and set the value of version once in a file and use it in 'tar'
 # and 'release' targets.
 VERSION_FILE := $(NAME)-version
+# This is treated as a string literal and is evaluated by bash in targets
+# It allows for compile target to set the version file for other targets
 VERSION := `cat $(VERSION_FILE)`
+TAR := $(shell command -v gtar || echo command -v tar || echo "Could not find tar")
 TAR_EXT := tar.bz2
-NETPLUGIN_CONTAINER_TAG := $(shell ./scripts/getGitCommit.sh)
+export NETPLUGIN_CONTAINER_TAG := $(shell ./scripts/getGitCommit.sh)
 TAR_FILENAME := $(NAME)-$(VERSION).$(TAR_EXT)
 TAR_LOC := .
-TAR_FILE := $(TAR_LOC)/$(TAR_FILENAME)
+export NETPLUGIN_TAR_FILE := $(TAR_LOC)/$(TAR_FILENAME)
+export V2PLUGIN_TAR_FILENAME := v2plugin-$(VERSION).tar.gz
 GO_MIN_VERSION := 1.7
 GO_MAX_VERSION := 1.8
 GO_VERSION := $(shell go version | cut -d' ' -f3 | sed 's/go//')
 CI_HOST_TARGETS ?= "host-unit-test host-integ-test host-build-docker-image"
 SYSTEM_TESTS_TO_RUN ?= "00SSH|Basic|Network|Policy|TestTrigger|ACIM|Netprofile"
 K8S_SYSTEM_TESTS_TO_RUN ?= "00SSH|Basic|Network|Policy"
 ACI_GW_IMAGE ?= "contiv/aci-gw:04-12-2017.2.2_1n"
+export CONTIV_V2PLUGIN_NAME ?= contiv/v2plugin:0.1
 
 all: build unit-test system-test ubuntu-tests
 
@@ -43,7 +47,7 @@ all-CI: stop clean start
 		&& cd /opt/gopath/src/github.com/contiv/netplugin \
 		&& make ${CI_HOST_TARGETS}"'
 ifdef SKIP_SYSTEM_TEST
-	echo "Skipping system tests"
+	@echo "Skipping system tests"
 else
 	make system-test
 endif
@@ -95,14 +99,15 @@ checks: go-version govet-src golint-src gofmt-src misspell-src
 checks-with-docker:
 	scripts/code_checks_in_docker.sh $(PKG_DIRS)
 
+# install binaries into GOPATH and update file netplugin-version
 compile:
 	cd $(GOPATH)/src/github.com/contiv/netplugin && \
 	NIGHTLY_RELEASE=${NIGHTLY_RELEASE} BUILD_VERSION=${BUILD_VERSION} \
-	TO_BUILD="${TO_BUILD}" VERSION_FILE=${VERSION_FILE} \
+	TO_BUILD="${TO_BUILD}" VERSION_FILE=$(VERSION_FILE) \
 	scripts/build.sh
 
 # fully prepares code for pushing to branch, includes building binaries
-run-build: deps checks clean compile
+run-build: deps checks clean compile archive
 
 compile-with-docker:
 	docker build \
@@ -129,7 +134,7 @@ update:
 # setting CONTIV_NODES=<number> while calling 'make demo' can be used to bring
 # up a cluster of <number> nodes. By default <number> = 1
 start:
-	CONTIV_DOCKER_VERSION="$${CONTIV_DOCKER_VERSION:-$(DEFAULT_DOCKER_VERSION)}" CONTIV_NODE_OS=${CONTIV_NODE_OS} vagrant up
+	CONTIV_NODE_OS=${CONTIV_NODE_OS} vagrant up
 
 # ===================================================================
 #kubernetes demo targets
@@ -213,7 +218,7 @@ ssh:
 	@vagrant ssh netplugin-node1 -c 'bash -lc "cd /opt/gopath/src/github.com/contiv/netplugin/ && bash"' || echo 'Please run "make demo"'
 
 ssh-build: start
-	vagrant ssh netplugin-node1 -c 'bash -lc "source /etc/profile.d/envvar.sh && cd /opt/gopath/src/github.com/contiv/netplugin && make run-build install-shell-completion"'
+	$(call make-on-node1, run-build install-shell-completion)
 
 unit-test: stop clean
 	./scripts/unittests -vagrant
@@ -290,63 +295,121 @@ host-restart:
 
 # create the rootfs for v2plugin. this is required for docker plugin create command
 host-pluginfs-create:
-	@echo dev: creating a docker v2plugin rootfs ...
-	sh scripts/v2plugin_rootfs.sh
+	./scripts/v2plugin_rootfs.sh
 
-# if rootfs already exists, copy newly compiled contiv binaries and start plugin on local host
-host-plugin-update:
-	@echo dev: updating docker v2plugin ...
+# remove the v2plugin from docker
+host-plugin-remove:
+	@echo dev: removing docker v2plugin ...
 	docker plugin disable ${CONTIV_V2PLUGIN_NAME}
 	docker plugin rm -f ${CONTIV_V2PLUGIN_NAME}
-	cp bin/netplugin bin/netmaster bin/netctl install/v2plugin/rootfs
+
+# add the v2plugin to docker with the current rootfs
+host-plugin-create:
+	@echo Creating docker v2plugin
 	docker plugin create ${CONTIV_V2PLUGIN_NAME} install/v2plugin
 	docker plugin enable ${CONTIV_V2PLUGIN_NAME}
 
-# cleanup all containers, plugins and start the v2plugin on all hosts
-host-plugin-restart:
+# re-deploy the v2plugin in docker with latest versioned binaries
+host-plugin-update: host-plugin-remove unarchive host-plugin-create
+	rm -rf install/v2plugin/rootfs
+
+# cleanup all containers, recreate and start the v2plugin on all hosts
+# uses the latest compile binaries
+host-plugin-restart: unarchive
 	@echo dev: restarting services...
-	cp bin/netplugin bin/netmaster bin/netctl install/v2plugin/rootfs
-	cd $(GOPATH)/src/github.com/contiv/netplugin/scripts/python && PYTHONIOENCODING=utf-8 ./startPlugin.py -nodes ${CLUSTER_NODE_IPS} -plugintype "v2plugin"
-
-# complete workflow to create rootfs, create/enable plugin and start swarm-mode
-demo-v2plugin:
-	CONTIV_V2PLUGIN_NAME="$${CONTIV_V2PLUGIN_NAME:-contiv/v2plugin:0.1}" CONTIV_DOCKER_VERSION="$${CONTIV_DOCKER_VERSION:-1.13.1}" CONTIV_DOCKER_SWARM="$${CONTIV_DOCKER_SWARM:-swarm_mode}" make ssh-build
-	vagrant ssh netplugin-node1 -c 'bash -lc "source /etc/profile.d/envvar.sh && cd /opt/gopath/src/github.com/contiv/netplugin && make host-pluginfs-create host-plugin-restart host-swarm-restart"'
-
-# release a v2 plugin
-host-plugin-release:
-	@echo dev: creating a docker v2plugin ...
-	sh scripts/v2plugin_rootfs.sh
-	docker plugin create ${CONTIV_V2PLUGIN_NAME} install/v2plugin
+	cd $(GOPATH)/src/github.com/contiv/netplugin/scripts/python \
+		&& PYTHONIOENCODING=utf-8 ./startPlugin.py -nodes ${CLUSTER_NODE_IPS} \
+			-plugintype "v2plugin"
+
+# unpack v2plugin archive created by host-pluginfs-create
+host-pluginfs-unpack:
+	# clear out old plugin completely
+	sudo rm -rf install/v2plugin/rootfs
+	mkdir -p install/v2plugin/rootfs
+	sudo tar -xf install/v2plugin/${V2PLUGIN_TAR_FILENAME} \
+		-C install/v2plugin/rootfs/ \
+		--exclude=usr/share/terminfo --exclude=dev/null \
+		--exclude=etc/terminfo/v/vt220
+
+# Runs make targets on the first netplugin vagrant node
+# this is used as a macro like $(call make-on-node1, compile checks)
+make-on-node1 = vagrant ssh netplugin-node1 -c '\
+	bash -lc "source /etc/profile.d/envvar.sh \
+	&& cd /opt/gopath/src/github.com/contiv/netplugin && make $(1)"'
+
+# Calls macro make-on-node1 but can be used as a dependecy by setting
+# the variable "node1-make-targets"
+make-on-node1-dep:
+	$(call make-on-node1, $(node1-make-targets))
+
+# assumes the v2plugin archive is available, installs the v2plugin and resets
+# everything on the vm to clean state
+v2plugin-install:
+	@echo Installing v2plugin
+	$(call make-on-node1, install-shell-completion host-pluginfs-unpack \
+		host-plugin-restart host-swarm-restart)
+
+# Just like demo-v2plugin except builds are done locally and cached
+demo-v2plugin-from-local: export CONTIV_DOCKER_VERSION ?= 1.13.1
+demo-v2plugin-from-local: export CONTIV_DOCKER_SWARM := swarm_mode
+demo-v2plugin-from-local: tar host-pluginfs-create start v2plugin-install
+
+# demo v2plugin on VMs: creates plugin assets, starts docker swarm
+# then creates and enables v2plugin
+demo-v2plugin: export CONTIV_DOCKER_VERSION ?= 1.13.1
+demo-v2plugin: export CONTIV_DOCKER_SWARM := swarm_mode
+demo-v2plugin: node1-make-targets := host-pluginfs-create
+demo-v2plugin: ssh-build make-on-node1-dep v2plugin-install
+
+# release a v2 plugin from the VM
+host-plugin-release: tar host-pluginfs-create host-pluginfs-unpack host-plugin-create
 	@echo dev: pushing ${CONTIV_V2PLUGIN_NAME} to docker hub
 	@echo dev: need docker login with user in contiv org
 	docker plugin push ${CONTIV_V2PLUGIN_NAME}
 
+# unarchive versioned binaries to bin, usually as a helper for other targets
+# uses the version stored in file 'netplugin-version' which is produced anytime
+# binaries are compiled (and that version can be set with env var
+# 'BUILD_VERSION'
+unarchive:
+	@# $(NETPLUGIN_TAR_FILE) depends on local file netplugin-version (exists in image),
+	@# but it is evaluated after we have extracted that file to local disk
+	@echo Updating bin/ with binaries versioned $(VERSION)
+	tar -xf $(NETPLUGIN_TAR_FILE) -C bin
+
+# pulls netplugin binaries from build container
+binaries-from-container:
+	docker rm netplugin-build 2>/dev/null || :
+	c_id=$$(docker create --name netplugin-build \
+		 netplugin-build:$(NETPLUGIN_CONTAINER_TAG)) && \
+	docker cp \
+		$${c_id}:/go/src/github.com/contiv/netplugin/netplugin-version ./ && \
+	for f in netplugin netmaster netctl contivk8s netcontiv; do \
+		docker cp -a $${c_id}:/go/bin/$$f bin/$$f; done && \
+	docker rm $${c_id}
+
 ##########################
 ## Packaging and Releasing
 ##########################
 
-# build tarball
-tar: compile-with-docker
-	@# $(TAR_FILE) depends on local file netplugin-version (exists in image),
-	@# but it is evaluated after we have extracted that file to local disk
-	docker rm netplugin-build || :
-	c_id=$$(docker create --name netplugin-build netplugin-build:$(NETPLUGIN_CONTAINER_TAG)) && \
-	docker cp $${c_id}:/go/src/github.com/contiv/netplugin/netplugin-version ./ && \
-	for f in netplugin netmaster netctl contivk8s netcontiv; do \
-		docker cp $${c_id}:/go/bin/$$f bin/$$f; done && \
-	docker rm $${c_id}
-	tar -jcf $(TAR_FILE) \
+archive:
+	$(TAR) --version | grep -q GNU \
+		|| (echo Please use GNU tar as \'gtar\' or \'tar\'; exit 1)
+	$(TAR) --owner=0 --group=0 -jcf $(NETPLUGIN_TAR_FILE) \
 		-C bin netplugin netmaster netctl contivk8s netcontiv \
 		-C ../scripts contrib/completion/bash/netctl get-contiv-diags
 
+# build versioned archive of netplugin binaries
+tar: compile-with-docker binaries-from-container archive
+
 clean-tar:
 	@rm -f $(TAR_LOC)/*.$(TAR_EXT)
 	@rm -f ${VERSION_FILE}
+	@rm -f install/v2plugin/v2plugin-*.tar.gz
 
 # GITHUB_USER and GITHUB_TOKEN are needed be set to run github-release
 release: tar
-	TAR_FILENAME=$(TAR_FILENAME) TAR_FILE=$(TAR_FILE) \
+	TAR_FILENAME=$(TAR_FILENAME) TAR_FILE=$(NETPLUGIN_TAR_FILE) \
 	OLD_VERSION=${OLD_VERSION} BUILD_VERSION=${BUILD_VERSION} \
 	NIGHTLY_RELEASE=${NIGHTLY_RELEASE} scripts/release.sh
 	@make clean-tar

install/v2plugin/Dockerfile

@@ -10,6 +10,12 @@ RUN mkdir -p /run/docker/plugins /etc/openvswitch /var/run/contiv/log \
  && wget https://github.com/andyshinn/alpine-pkg-glibc/releases/download/2.23-r1/glibc-2.23-r1.apk \
  && apk --no-cache add glibc-2.23-r1.apk
 
-COPY netplugin netmaster netctl startcontiv.sh /
+# copy in binaries and scripts
+ARG TAR_FILE
+ADD ${TAR_FILE} /
+COPY startcontiv.sh /
 
+# this container is never run, it is exported for docker to run as a plugin,
+# the startcontiv.sh script and the netplugin binaries are copied into the
+# plugin's rootfs after export, this avoids copying into and out of container
 ENTRYPOINT ["/startcontiv.sh"]

install/v2plugin/config.template

@@ -102,7 +102,7 @@
              "value"
           ],
           ## Do not change the default value, this will be replaced with $CONTIV_V2PLUGIN_NAME
-          "Value": "PluginName"
+          "Value": "__CONTIV_V2PLUGIN_NAME__"
        }
     ],
     "mounts": [

scripts/jenkins_cleanup.sh

@@ -1,17 +1,38 @@
 #!/bin/bash
+
 # Script to create the docker v2 plugin
 # run this script from contiv/netplugin directory
+# requires NETPLUGIN_CONTAINER_TAG for contivrootfs image
+# requires CONTIV_NETPLUGIN_NAME, the Network Driver name for requests to
+#   dockerd, should look like contiv/v2plugin:$NETPLUGIN_CONTAINER_TAG
+# requires NETPLUGIN_TAR_FILE to point to the netplugin binaries
+
+set -euxo pipefail
+
+echo "Creating rootfs for v2plugin: ${CONTIV_V2PLUGIN_NAME}"
+
+
+# config.json is docker's runtime configuration for the container
+# delete comments and replace placeholder with ${CONTIV_V2PLUGIN_NAME}
+sed '/##/d;s/__CONTIV_V2PLUGIN_NAME__/${CONTIV_V2PLUGIN_NAME}/' \
+    install/v2plugin/config.template > install/v2plugin/config.json
+
+# copy over binaries
+cp ${NETPLUGIN_TAR_FILE} install/v2plugin/
 
-echo "Creating rootfs for v2plugin ", ${CONTIV_V2PLUGIN_NAME}
-rm -rf install/v2plugin/rootfs
-rm install/v2plugin/config.json
-cat install/v2plugin/config.template | grep -v "##" > install/v2plugin/config.json
-sed -i "s%PluginName%${CONTIV_V2PLUGIN_NAME}%" install/v2plugin/config.json
-cp bin/netplugin bin/netmaster bin/netctl install/v2plugin
-docker build -t contivrootfs install/v2plugin
-id=$(docker create contivrootfs true)
-mkdir -p install/v2plugin/rootfs
-sudo docker export "${id}" | sudo tar -x -C install/v2plugin/rootfs
+DOCKER_IMAGE=contivrootfs:${NETPLUGIN_CONTAINER_TAG}
+docker build -t ${DOCKER_IMAGE} \
+    --build-arg TAR_FILE=$(basename "${NETPLUGIN_TAR_FILE}") install/v2plugin
+
+rm install/v2plugin/${NETPLUGIN_TAR_FILE}
+
+# creates a ready to run container but doesn't run it
+id=$(docker create $DOCKER_IMAGE true)
+
+# create the rootfs archive based on the created container contents
+sudo docker export "${id}" > install/v2plugin/${V2PLUGIN_TAR_FILENAME}
+
+# clean up created container
 docker rm -vf "${id}"
-docker rmi contivrootfs
-rm install/v2plugin/netplugin install/v2plugin/netmaster install/v2plugin/netctl
+
+echo netplugin\'s docker plugin rootfs is archived at install/v2plugin/${V2PLUGIN_TAR_FILENAME}