02.Version cache - docker cache build framework (sonic-net#12001)

During docker build, host files can be passed to the docker build through
docker context files. But there is no straightforward way to transfer
the files from docker build to host.

This feature provides a tricky way to pass the cache contents from docker
build to host. It tar's the cached content and encodes them as base64 format
and passes it through a log file with a special tag as 'VCSTART and VCENT'.

Slave.mk in the host, it extracts the cache contents from the log and stores them
in the cache folder. Cache contents are encoded as base64 format for
easy passing.

<!--
     Please make sure you've read and understood our contributing guidelines:
     https://github.com/Azure/SONiC/blob/gh-pages/CONTRIBUTING.md

     ** Make sure all your commits include a signature generated with `git commit -s` **

     If this is a bug fix, make sure your description includes "fixes #xxxx", or
     "closes #xxxx" or "resolves #xxxx"

     Please provide the following information:
-->

#### Why I did it

#### How I did it

#### How to verify it

Author: Kalimuthu-Velappan

Makefile.work

@@ -229,8 +229,12 @@ SLAVE_TAG = $(shell \
 
 COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
 	scripts/collect_docker_version_files.sh \
-	$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
-		target 
+		$(SLAVE_BASE_IMAGE) \
+		target \
+		$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
+		$(SLAVE_DIR) \
+		$(SLAVE_DIR)/Dockerfile
+
 OVERLAY_MODULE_CHECK := \
     lsmod | grep -q "^overlay " &>/dev/null || \
     zgrep -q 'CONFIG_OVERLAY_FS=y' /proc/config.gz &>/dev/null || \
@@ -377,7 +381,8 @@ DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \
 		    --build-arg http_proxy=$(http_proxy) \
 		    --build-arg https_proxy=$(https_proxy) \
 		    --build-arg no_proxy=$(no_proxy) \
-		    $(SLAVE_DIR) $(SPLIT_LOG) $(DOCKER_BASE_LOG)
+		    $(SLAVE_DIR) \
+		    $(SPLIT_LOG) $(DOCKER_BASE_LOG)
 
 DOCKER_BASE_PULL = docker pull \
 			$(REGISTRY_SERVER):$(REGISTRY_PORT)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)
@@ -508,7 +513,7 @@ SONIC_BUILD_INSTRUCTION :=  $(MAKE) \
 
 
 ifeq ($(filter clean,$(MAKECMDGOALS)),)
-COLLECT_BUILD_VERSION = { DBGOPT='$(DBGOPT)' scripts/collect_build_version_files.sh \$$?; }
+COLLECT_BUILD_VERSION = { scripts/collect_build_version_files.sh \$$?; }
 endif
 
 ifdef SOURCE_FOLDER

files/build_templates/build_docker_cache.j2

@@ -0,0 +1,11 @@
+# Base docker build
+FROM {{IMAGENAME}}
+
+# Copy the cache data to host
+From scratch as output
+COPY --from={{IMAGENAME}} /cache.tgz cache.tgz
+
+# Clean up the cache data
+FROM {{IMAGENAME}} as final
+RUN rm /cache.tgz
+

scripts/collect_build_version_files.sh

@@ -23,6 +23,6 @@ mkdir -p $VERSION_SLAVE_PATH
 
 scripts/versions_manager.py merge -t $VERSION_SLAVE_PATH -b $LOG_VERSION_PATH -e $POST_VERSION_PATH
 
-rm -rf $BUILD_VERSION_PATH/*
+[ -d $BUILD_VERSION_PATH ] && rm -rf $BUILD_VERSION_PATH/*
 
 exit $RET

scripts/collect_docker_version_files.sh

@@ -1,12 +1,17 @@
 #!/bin/bash
 
+set -x 
 DOCKER_IMAGE=$1
 TARGET_PATH=$2
+DOCKER_IMAGE_TAG=$3
+DOCKER_PATH=$4
+DOCKER_FILE=$5
 
 [ -z "$TARGET_PATH" ] && TARGET_PATH=./target
 
 DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE | cut -d: -f1 | sed "s/-$DOCKER_USERNAME\$//")
-DOCKER_CONTAINER=$DOCKER_IMAGE_NAME
+#Create the container specific to the user tag and slave tag
+DOCKER_CONTAINER=${DOCKER_IMAGE_TAG/:/-}
 TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME
 
 [ -d $TARGET_VERSIONS_PATH ] && rm -rf $TARGET_VERSIONS_PATH
@@ -18,8 +23,16 @@ export DOCKER_CLI_EXPERIMENTAL=enabled
 if docker container inspect $DOCKER_IMAGE > /dev/null 2>&1; then
     docker container rm $DOCKER_IMAGE > /dev/null
 fi
-docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE
+docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE_TAG
 docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/
 docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/
 docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/
+
+# Save the cache contents from docker build
+IMAGENAME=${DOCKER_IMAGE_TAG} j2 files/build_templates/build_docker_cache.j2 > ${DOCKER_FILE}.cleanup
+docker tag ${DOCKER_IMAGE_TAG} tmp-${DOCKER_IMAGE_TAG}
+DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup  --target output -o target/vcache/${DOCKER_IMAGE_NAME} ${DOCKER_PATH}
+DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup  --no-cache --target final --tag ${DOCKER_IMAGE_TAG} ${DOCKER_PATH}
+docker rmi tmp-${DOCKER_IMAGE_TAG}
+
 docker container rm $DOCKER_CONTAINER

scripts/collect_host_image_version_files.sh

@@ -3,11 +3,13 @@
 TARGET=$1
 FILESYSTEM_ROOT=$2
 VERSIONS_PATH=$TARGET/versions/host-image
+IMAGENAME="host-image"
 
 [ -d $VERSIONS_PATH ] && sudo rm -rf $VERSIONS_PATH
 mkdir -p $VERSIONS_PATH
 
-sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo
+mkdir -p target/vcache/${IMAGENAME}
+sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo ${IMAGENAME}
 
 cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/
 cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/

scripts/prepare_docker_buildinfo.sh

@@ -11,6 +11,7 @@ DISTRO=$5
 DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE")
 BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo"
 BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions"
+DOCKER_PATH=$(dirname $DOCKERFILE)
 
 [ -d $BUILDINFO_PATH ] && rm -rf $BUILDINFO_PATH
 mkdir -p $BUILDINFO_VERSION_PATH
@@ -31,8 +32,11 @@ scripts/docker_version_control.sh $@
 
 DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo
 COPY ["buildinfo", "/usr/local/share/buildinfo"]
+COPY vcache/ /sonic/target/vcache/'${IMAGENAME}'
 RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb
-RUN pre_run_buildinfo'
+ENV IMAGENAME='${IMAGENAME}'
+RUN pre_run_buildinfo '${IMAGENAME}'
+'
 
 # Add the auto-generate code if it is not added in the target Dockerfile
 if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGE; then
@@ -42,7 +46,7 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE
     awk -v text="${DOCKERFILE_PRE_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE
 
     # Append the docker build script at the end of the docker file
-    echo -e "\nRUN post_run_buildinfo" >> $TEMP_FILE
+    echo -e "\nRUN post_run_buildinfo ${IMAGENAME} " >> $TEMP_FILE
 
     cat $TEMP_FILE > $DOCKERFILE_TARGE
     rm -f $TEMP_FILE
@@ -55,3 +59,8 @@ cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH
 scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH"
 
 touch $BUILDINFO_VERSION_PATH/versions-deb
+
+# Create the cache directories
+LOCAL_CACHE_DIR=target/vcache/${IMAGENAME}
+mkdir -p ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
+chmod -f 777 ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/

slave.mk

@@ -112,6 +112,7 @@ configure :
 	$(Q)mkdir -p $(PYTHON_DEBS_PATH)
 	$(Q)mkdir -p $(PYTHON_WHEELS_PATH)
 	$(Q)mkdir -p $(DPKG_ADMINDIR_PATH)
+	$(Q)mkdir -p $(TARGET_PATH)/vcache
 	$(Q)echo $(PLATFORM) > .platform
 	$(Q)echo $(PLATFORM_ARCH) > .arch
 
@@ -934,8 +935,10 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g
 		--label Tag=$(SONIC_IMAGE_VERSION) \
 		-f $(TARGET_DOCKERFILE)/Dockerfile.buildinfo \
 		-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
+
 	if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
-	scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH)
+	scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
+
 	$(call docker-image-save,$*,$@)
 	# Clean up
 	if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
@@ -1013,6 +1016,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
 		mkdir -p $($*.gz_PATH)/files $(LOG)
 		mkdir -p $($*.gz_PATH)/python-debs $(LOG)
 		mkdir -p $($*.gz_PATH)/python-wheels $(LOG)
+		mkdir -p $(TARGET_PATH)/vcache/$* $($*.gz_PATH)/vcache $(LOG)
 		sudo mount --bind $($*.gz_DEBS_PATH) $($*.gz_PATH)/debs $(LOG)
 		sudo mount --bind $($*.gz_FILES_PATH) $($*.gz_PATH)/files $(LOG)
 		sudo mount --bind $(PYTHON_DEBS_PATH) $($*.gz_PATH)/python-debs $(LOG)
@@ -1054,9 +1058,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
 			--label Tag=$(SONIC_IMAGE_VERSION) \
 		        $($(subst -,_,$(notdir $($*.gz_PATH)))_labels) \
 			-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
+
 		if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
-		scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH)
+		scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile $(LOG)
 		if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi
+
 		$(call docker-image-save,$*,$@)
 		# Clean up
 		if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
@@ -1085,6 +1091,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
 
 		mkdir -p $($*.gz_PATH)/debs $(LOG)
 		sudo mount --bind $($*.gz_DEBS_PATH) $($*.gz_PATH)/debs $(LOG)
+		mkdir -p $(TARGET_PATH)/vcache/$*-dbg $($*.gz_PATH)/vcache $(LOG)
 		# Export variables for j2. Use path for unique variable names, e.g. docker_orchagent_debs
 		$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_dbg_debs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_DEPENDS),RDEPENDS))\n" | awk '!a[$$0]++'))
 		$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_IMAGE_PACKAGES)))\n" | awk '!a[$$0]++'))
@@ -1108,9 +1115,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
 			--label Tag=$(SONIC_IMAGE_VERSION) \
 			--file $($*.gz_PATH)/Dockerfile-dbg \
 			-t $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $(LOG)
+
 		if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
-		scripts/collect_docker_version_files.sh $(DOCKER_DBG_IMAGE_REF) $(TARGET_PATH)
+		scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH)  $($*.gz_PATH)/Dockerfile-dbg $(LOG)
 		if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi
+
 		$(call docker-image-save,$*-$(DBG_IMAGE_MARK),$@)
 		# Clean up
 		docker rmi -f $(DOCKER_IMAGE_REF) &> /dev/null || true

sonic-slave-buster/Dockerfile.j2

@@ -599,7 +599,7 @@ RUN add-apt-repository \
            $(lsb_release -cs) \
            stable"
 RUN apt-get update
-RUN apt-get install -y docker-ce=5:18.09.5~3-0~debian-buster docker-ce-cli=5:18.09.5~3-0~debian-buster
+RUN apt-get install -y docker-ce=5:20.10.21~3-0~debian-buster docker-ce-cli=5:20.10.21~3-0~debian-buster
 RUN echo "DOCKER_OPTS=\"--experimental --storage-driver=vfs {{ DOCKER_EXTRA_OPTS }}\"" >> /etc/default/docker
 RUN update-alternatives --set iptables /usr/sbin/iptables-legacy
 

src/sonic-build-hooks/scripts/buildinfo_base.sh

@@ -15,6 +15,15 @@ REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/'
 DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock
 
 . $BUILDINFO_PATH/config/buildinfo.config
+if [ -e /vcache ]; then
+	PKG_CACHE_PATH=/vcache/${IMAGENAME}
+else
+	PKG_CACHE_PATH=/sonic/target/vcache/${IMAGENAME}
+fi
+PKG_CACHE_FILE_NAME=${PKG_CACHE_PATH}/cache.tgz
+mkdir -p ${PKG_CACHE_PATH}
+
+
 
 URL_PREFIX=$(echo "${PACKAGE_URL_PREFIX}" | sed -E "s#(//[^/]*/).*#\1#")
 

src/sonic-build-hooks/scripts/post_run_buildinfo

@@ -1,12 +1,14 @@
 #!/bin/bash
 
+IMAGENAME=$1
+
 . /usr/local/share/buildinfo/scripts/buildinfo_base.sh
 
 # Collect the version files
 collect_version_files $POST_VERSION_PATH
 
-[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $POST_VERSION_PATH
-rm -rf $BUILD_VERSION_PATH/*
+#Save the cache file for exporting it to host.
+tar -C ${PKG_CACHE_PATH} --exclude=cache.tgz -zcvf /cache.tgz .
 
 # Disable the build hooks
 symlink_build_hooks -d

src/sonic-build-hooks/scripts/pre_run_buildinfo

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+IMAGENAME=$1
+
 . /usr/local/share/buildinfo/scripts/buildinfo_base.sh
 
 [ -d $DIFF_VERSION_PATH ] && rm -rf $DIFF_VERSION_PATH
@@ -15,10 +17,14 @@ update_version_files
 symlink_build_hooks
 set_reproducible_mirrors
 
+mkdir -p /var/cache/apt/archives/
+
 chmod -R a+rw $BUILDINFO_PATH
 
 if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]; then
     cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/
 fi
 
+DISTRO=${DISTRO} apt-get update && apt-get install -y rsync
+
 exit 0