From 363ecfc3fed197957e1cd9559c2e6df3f7f4d689 Mon Sep 17 00:00:00 2001
From: Liu Shilong <shilongliu@microsoft.com>
Date: Mon, 19 Dec 2022 12:18:36 +0800
Subject: [PATCH] [build] Add retry when make SONiC image to improve success
 rate. (#12325)

Why I did it
Makefile needs some dependencies from the Internet. It will fail for network related issue.
Retries will fix most of these issues.

How I did it
Add retries when running commands which maybe related with networking.

How to verify it
---
 .azure-pipelines/template-variables.yml |  4 ++++
 Makefile                                |  9 +++++----
 scripts/run_with_retry                  | 17 +++++++++++++++++
 3 files changed, 26 insertions(+), 4 deletions(-)
 create mode 100755 scripts/run_with_retry

diff --git a/.azure-pipelines/template-variables.yml b/.azure-pipelines/template-variables.yml
index f28768e39efe..027dfee9986a 100644
--- a/.azure-pipelines/template-variables.yml
+++ b/.azure-pipelines/template-variables.yml
@@ -1,2 +1,6 @@
 variables:
   DEFAULT_CONTAINER_REGISTRY: 'publicmirror.azurecr.io'
+  COMMON_LIB_BUILD_ENVS: 'bullseye'
+  SONIC_SLAVE_DOCKER_DRIVER: 'overlay2'
+  SONIC_BUILD_RETRY_COUNT: 3
+  SONIC_BUILD_RETRY_INTERVAL: 600
diff --git a/Makefile b/Makefile
index ace13582cbef..2f312cc7f1aa 100644
--- a/Makefile
+++ b/Makefile
@@ -25,20 +25,21 @@ PLATFORM_PATH := platform/$(if $(PLATFORM),$(PLATFORM),$(CONFIGURED_PLATFORM))
 PLATFORM_CHECKOUT := platform/checkout
 PLATFORM_CHECKOUT_FILE := $(PLATFORM_CHECKOUT)/$(PLATFORM).ini
 PLATFORM_CHECKOUT_CMD := $(shell if [ -f $(PLATFORM_CHECKOUT_FILE) ]; then PLATFORM_PATH=$(PLATFORM_PATH) j2 $(PLATFORM_CHECKOUT)/template.j2 $(PLATFORM_CHECKOUT_FILE); fi)
+MAKE_WITH_RETRY := ./scripts/run_with_retry $(MAKE)
 
 %::
 	@echo "+++ --- Making $@ --- +++"
 ifeq ($(NOJESSIE), 0)
-	EXTRA_DOCKER_TARGETS=$(notdir $@) make -f Makefile.work jessie
+	$(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) -f Makefile.work jessie
 endif
 ifeq ($(NOSTRETCH), 0)
-	EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch make -f Makefile.work stretch
+	$(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch -f Makefile.work stretch
 endif
 ifeq ($(NOBUSTER), 0)
-	EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=buster make -f Makefile.work buster
+	$(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=buster -f Makefile.work buster
 endif
 ifeq ($(NOBULLSEYE), 0)
-	BLDENV=bullseye make -f Makefile.work $@
+	$(MAKE_WITH_RETRY) BLDENV=bullseye -f Makefile.work $@
 endif
 
 jessie:
diff --git a/scripts/run_with_retry b/scripts/run_with_retry
new file mode 100755
index 000000000000..9e709bbf1bda
--- /dev/null
+++ b/scripts/run_with_retry
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+run_with_retry(){
+    [ "$SONIC_BUILD_RETRY_COUNT" -gt 0 ] || SONIC_BUILD_RETRY_COUNT=0
+    [[ "$*" == "" ]] && { echo "run_with_retry: input command can't be empty." 1>&2;exit 1; }
+    for ((i=0; i<=$SONIC_BUILD_RETRY_COUNT; i++))
+    do
+        if [[ $i != 0 ]];then
+            echo "==============================================================================" 1>&2
+            echo "Waiting $SONIC_BUILD_RETRY_INTERVAL to run again, $i/$SONIC_BUILD_RETRY_COUNT" 1>&2
+            echo "==============================================================================" 1>&2
+            sleep $SONIC_BUILD_RETRY_INTERVAL
+        fi
+        "$@" && break
+    done
+}
+run_with_retry "$@"