[build]: Parallel building of sonic dockers using native dockerd(dood)

Makefile

@@ -12,6 +12,7 @@ ifeq ($(NOSTRETCH), 0)
 	EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch make -f Makefile.work stretch
 endif
 	BLDENV=buster make -f Makefile.work $@
+	BLDENV=buster make -f Makefile.work docker-cleanup
 
 jessie:
 	@echo "+++ Making $@ +++"
@@ -29,7 +30,7 @@ init:
 	@echo "+++ Making $@ +++"
 	make -f Makefile.work $@
 
-clean configure reset showtag sonic-slave-build sonic-slave-bash :
+clean configure reset docker-cleanup showtag sonic-slave-build sonic-slave-bash :
 	@echo "+++ Making $@ +++"
 ifeq ($(NOJESSIE), 0)
 	make -f Makefile.work $@

Makefile.work

@@ -75,6 +75,7 @@ $(shell rm -f .screen)
 MAKEFLAGS += -B
 
 CONFIGURED_ARCH := $(shell [ -f .arch ] && cat .arch || echo $(PLATFORM_ARCH))
+CONFIGURED_PLATFORM = $(if $(PLATFORM),$(PLATFORM),$(shell cat .platform 2>/dev/null))
 ifeq ($(CONFIGURED_ARCH),)
     override CONFIGURED_ARCH = amd64
 endif
@@ -115,7 +116,8 @@ $(shell BUILD_SLAVE=y scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(
 
 # Add the versions in the tag, if the version change, need to rebuild the slave
 SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* | sha1sum | awk '{print substr($$1,0,11);}')
-SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* | sha1sum | awk '{print substr($$1,0,11);}')
+SLAVE_TAG = $(shell (cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* .git/HEAD && echo $(USER)/$(PWD)/$(CONFIGURED_PLATFORM)) \
+			| sha1sum | awk '{print substr($$1,0,11);}')
 
 OVERLAY_MODULE_CHECK := \
     lsmod | grep -q "^overlay " &>/dev/null || \
@@ -125,6 +127,14 @@ OVERLAY_MODULE_CHECK := \
 
 BUILD_TIMESTAMP := $(shell date +%Y%m%d\.%H%M%S)
 
+# Create separate Docker lockfiles for saving vs. loading an image.
+ifeq ($(DOCKER_LOCKDIR),)
+override DOCKER_LOCKDIR := /tmp/docklock
+endif
+DOCKER_LOCKFILE_SAVE := $(DOCKER_LOCKDIR)/docker_save.lock
+$(shell mkdir -m 0777 -p $(DOCKER_LOCKDIR))
+$(shell [ -f $(DOCKER_LOCKFILE_SAVE) ] || (touch $(DOCKER_LOCKFILE_SAVE) && chmod 0777 $(DOCKER_LOCKFILE_SAVE)))
+
 ifeq ($(DOCKER_BUILDER_MOUNT),)
 override DOCKER_BUILDER_MOUNT := "$(PWD):/sonic"
 endif
@@ -135,6 +145,7 @@ endif
 
 DOCKER_RUN := docker run --rm=true --privileged --init \
     -v $(DOCKER_BUILDER_MOUNT) \
+    -v "$(DOCKER_LOCKDIR):$(DOCKER_LOCKDIR)" \
     -w $(DOCKER_BUILDER_WORKDIR) \
     -e "http_proxy=$(http_proxy)" \
     -e "https_proxy=$(https_proxy)" \
@@ -153,6 +164,31 @@ ifneq ($(SONIC_DPKG_CACHE_SOURCE),)
 	DOCKER_RUN += -v "$(SONIC_DPKG_CACHE_SOURCE):/dpkg_cache:rw"
 endif
 
+
+# User name and tag for "docker-*" images created by native dockerd mode.
+ifeq ($(strip $(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD)),y)
+DOCKER_USERNAME = -$(USER_LC)
+DOCKER_USERTAG = $(SLAVE_TAG)
+else
+DOCKER_USERNAME =
+DOCKER_USERTAG = latest
+endif
+
+# Define canned sequence to clean up Docker image cache.
+#   - These are the remnants from building the runtime Docker images using native (host) Docker daemon.
+#   - Image naming convention differs on a shared build system vs. non-shared.
+# $(docker-image-cleanup)
+ifeq ($(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD),y)
+define docker-image-cleanup
+    @for i in $(shell docker images --quiet --filter 'dangling=true') ; do (docker rmi -f $$i &> /dev/null || true) ; done
+    @for i in $(shell docker images --quiet docker-*$(DOCKER_USERNAME):$(DOCKER_USERTAG)) ; do (docker rmi -f $$i &> /dev/null || true) ; done
+endef
+else
+define docker-image-cleanup
+    @:
+endef
+endif
+
 ifeq ($(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD), y)
     DOCKER_RUN += -v /var/run/docker.sock:/var/run/docker.sock
 endif
@@ -219,6 +255,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            BUILD_NUMBER=$(BUILD_NUMBER) \
                            BUILD_TIMESTAMP=$(BUILD_TIMESTAMP) \
                            SONIC_IMAGE_VERSION=$(SONIC_IMAGE_VERSION) \
+                           SLAVE_TAG=$(SLAVE_TAG) \
                            ENABLE_DHCP_GRAPH_SERVICE=$(ENABLE_DHCP_GRAPH_SERVICE) \
                            ENABLE_ZTP=$(ENABLE_ZTP) \
                            SHUTDOWN_BGP_ON_START=$(SHUTDOWN_BGP_ON_START) \
@@ -240,6 +277,11 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_DPKG_CACHE_SOURCE=$(SONIC_DPKG_CACHE_SOURCE) \
                            HTTP_PROXY=$(http_proxy) \
                            HTTPS_PROXY=$(https_proxy) \
+                           USER_LC=$(USER_LC) \
+                           DOCKER_USERTAG=$(DOCKER_USERTAG) \
+                           DOCKER_LOCKDIR=$(DOCKER_LOCKDIR) \
+                           DOCKER_LOCKFILE_SAVE=$(DOCKER_LOCKFILE_SAVE) \
+                           SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD=$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) \
                            SONIC_INCLUDE_SYSTEM_TELEMETRY=$(INCLUDE_SYSTEM_TELEMETRY) \
                            SONIC_INCLUDE_RESTAPI=$(INCLUDE_RESTAPI) \
                            TELEMETRY_WRITABLE=$(TELEMETRY_WRITABLE) \
@@ -286,6 +328,9 @@ else
 	@$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?"
 endif
 
+docker-cleanup:
+	$(docker-image-cleanup)
+
 sonic-build-hooks:
 	@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
 	@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo

dockers/docker-config-engine-buster/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-base-buster
+FROM {{ docker_config_engine_buster_load_image }}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-config-engine-stretch/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-base-stretch
+FROM {{ docker_config_engine_stretch_load_image }}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-config-engine/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-base
+FROM {{ docker_config_engine_load_image }}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-database/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_database_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-dhcp-relay/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_dhcp_relay_load_image }}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-fpm-frr/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_fpm_frr_load_image }}
 
 ARG docker_container_name
 ARG frr_user_uid

dockers/docker-fpm-gobgp/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-fpm-quagga
+FROM {{ docker_fpm_gobgp_load_image }}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-fpm-quagga/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine
+FROM {{ docker_fpm_quagga_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-iccpd/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_iccpd_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-lldp/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_lldp_load_image }}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-nat/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_nat_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-orchagent/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_orchagent_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-platform-monitor/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_platform_monitor_load_image }}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-ptf-saithrift/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-ptf
+FROM {{ docker_ptf_saithrift_load_image }}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

dockers/docker-router-advertiser/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_router_advertiser_load_image }}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-sflow/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{docker_sflow_load_image}}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-snmp/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python3_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_snmp_load_image }}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-sonic-mgmt-framework/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM {{ docker_sonic_mgmt_framework_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-sonic-restapi/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-stretch
+FROM {{ docker_sonic_restapi_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

dockers/docker-sonic-telemetry/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_sonic_telemetry_load_image }}
 
 ARG docker_container_name
 ARG image_version

dockers/docker-teamd/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-buster
+FROM {{ docker_teamd_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/broadcom/docker-saiserver-brcm/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM {{ docker_saiserver_brcm_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/broadcom/docker-syncd-brcm-rpc/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-syncd-brcm
+FROM {{ docker_syncd_brcm_rpc_load_image }}
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

platform/broadcom/docker-syncd-brcm/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages %}
-FROM docker-config-engine-buster
+FROM {{ docker_syncd_brcm_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/vs/docker-gbsyncd-vs/Dockerfile.j2

@@ -1,4 +1,5 @@
-FROM docker-config-engine-buster
+ {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+FROM {{ docker_gbsyncd_vs_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/vs/docker-sonic-vs/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM {{ docker_sonic_vs_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

platform/vs/docker-syncd-vs/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-buster
+FROM {{ docker_syncd_vs_load_image }}
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

rules/config

@@ -29,7 +29,10 @@ DEFAULT_BUILD_LOG_TIMESTAMP = none
 # SONIC_USE_DOCKER_BUILDKIT = y
 
 # SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD - use native dockerd for build.
-# If set to y SONiC build container will use native dockerd instead of dind for faster build
+# If set to y SONiC build container will use native dockerd instead of dind for faster build.
+# Special handling of the docker image file names is needed to avoid conflicts with
+# other SONiC build jobs on the same server. This requires changes to the Dockerfile.j2 FROM statement
+# in the dockers/ and platform/ subdirs to use a variable reference instead of an explicit image name.
 # SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD = y
 
 # SONIC_CONFIG_ENABLE_COLORS - enable colored output in build system.

rules/functions

@@ -165,3 +165,7 @@ sudo mount -t overlay overlay -olowerdir=/var/lib/dpkg,upperdir=$$upperdir,workd
 export SONIC_DPKG_ADMINDIR=$$mergedir
 trap "sudo umount $$mergedir && rm -rf $$mergedir $$upperdir $$workdir" EXIT
 endef
+
+TO_LOWER = $(shell echo $(1) | tr '[:upper:]' '[:lower:]')
+TO_UPPER = $(shell echo $(1) | tr  '[:lower:]' '[:upper:]')
+