Merge branch 'sonic-net:master' into reset_factory

Author: Mohammedz93

.azure-pipelines/azure-pipelines-build.yml

@@ -94,6 +94,15 @@ jobs:
           variables:
             PLATFORM_ARCH: armhf
 
+        - name: marvell-arm64
+          ${{ if not(parameters.qemuOrCrossBuild) }}:
+            pool: sonicbld-arm64
+          timeoutInMinutes: 2880
+          variables:
+            PLATFORM_ARCH: arm64
+
+        - name: marvell
+
         - name: mellanox
           variables:
             dbg_image: yes
@@ -125,7 +134,7 @@ jobs:
             make $BUILD_OPTIONS target/docker-ptf-sai.gz
             if [ $(Build.Reason) != 'PullRequest' ];then
               gzip -kd target/sonic-vs.img.gz
-              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make sonic-slave-run
+              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make $BUILD_OPTIONS sonic-slave-run
               rm target/sonic-vs.img
             fi
           else

.azure-pipelines/azure-pipelines-image-template.yml

@@ -28,11 +28,12 @@ jobs:
       - template: .azure-pipelines/cleanup.yml@buildimage
       - ${{ parameters.preSteps }}
       - script: |
+          BUILD_OPTIONS="$(BUILD_OPTIONS)"
           if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf)$"; then
             CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM_AZP)"
             BUILD_OPTIONS="$(BUILD_OPTIONS) $CACHE_OPTIONS"
-            echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
           fi
+          echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
         displayName: "Set cache options"
       - checkout: self
         submodules: recursive

.azure-pipelines/azure-pipelines-repd-build-variables.yml

@@ -1,5 +1,2 @@
 variables:
-  ${{ if eq(variables['Build.Reason'],'PullRequest') }}:
-    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=$([[ "$(System.PullRequest.TargetBranch)" =~ ^20[2-9][0-9]{3}$ ]] && echo deb,py2,py3,web,git,docker)'
-  ${{ else }}:
-    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
+  VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'

.azure-pipelines/run-test-scheduler-template.yml

@@ -48,13 +48,13 @@ parameters:
 
 steps:
   - script: |
-      set -ex
+      set -e
       wget -O ./.azure-pipelines/test_plan.py  https://raw.githubusercontent.com/sonic-net/sonic-mgmt/master/.azure-pipelines/test_plan.py
       wget -O ./.azure-pipelines/pr_test_scripts.yaml  https://raw.githubusercontent.com/sonic-net/sonic-mgmt/master/.azure-pipelines/pr_test_scripts.yaml
     displayName: Download TestbedV2 scripts
 
   - script: |
-      set -ex
+      set -e
       pip install PyYAML
       rm -f new_test_plan_id.txt
       python ./.azure-pipelines/test_plan.py create -t ${{ parameters.TOPOLOGY }} -o new_test_plan_id.txt \
@@ -76,45 +76,45 @@ steps:
     displayName: Trigger test
 
   - script: |
-      set -ex
+      set -e
       echo "Lock testbed"
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to [email protected]"
-      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID "
       # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED"
       python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state LOCK_TESTBED
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Lock testbed
 
   - script: |
-      set -ex
+      set -e
       echo "Prepare testbed"
       echo "Preparing the testbed(add-topo, deploy-mg) may take 15-30 minutes. Before the testbed is ready, the progress of the test plan keeps displayed as 0, please be patient(We will improve the indication in a short time)"
       echo "If the progress keeps as 0 for more than 1 hour, please cancel and retry this pipeline"
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to [email protected]"
-      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID "
       # When "PREPARE_TESTBED" finish, it changes into "EXECUTING"
       python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state PREPARE_TESTBED
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Prepare testbed
 
   - script: |
-      set -ex
+      set -e
       echo "Run test"
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to [email protected]"
-      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID "
       # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED"
       python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state EXECUTING
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Run test
 
   - script: |
-      set -ex
+      set -e
       echo "KVM dump"
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to [email protected]"
-      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID "
       # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
       python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state KVMDUMP
     condition: succeededOrFailed()
@@ -123,7 +123,7 @@ steps:
     displayName: KVM dump
 
   - script: |
-      set -ex
+      set -e
       echo "Try to cancel test plan $TEST_PLAN_ID, cancelling finished test plan has no effect."
       python ./.azure-pipelines/test_plan.py cancel -i "$(TEST_PLAN_ID)"
     condition: always()

.github/pull_request_template.md

@@ -12,10 +12,17 @@
 
 #### Why I did it
 
+##### Work item tracking
+- Microsoft ADO **(number only)**:
+
 #### How I did it
 
 #### How to verify it
 
+<!--
+If PR needs to be backported, then the PR must be tested against the base branch and the earliest backport release branch and provide tested image version on these two branches. For example, if the PR is requested for master, 202211 and 202012, then the requester needs to provide test results on master and 202012.
+-->
+
 #### Which release branch to backport (provide reason below if selected)
 
 <!--
@@ -34,13 +41,26 @@
 - [ ] 202205
 - [ ] 202211
 
+#### Tested branch (Please provide the tested image version)
+
+<!--
+- Please provide tested image version
+- e.g.
+- [x] 20201231.100
+-->
+
+- [ ] <!-- image version 1 -->
+- [ ] <!-- image version 2 -->
+
 #### Description for the changelog
 <!--
 Write a short (one line) summary that describes the changes in this
 pull request for inclusion in the changelog:
 -->
 
-#### Ensure to add label/tag for the feature raised. example - PR#2174 under sonic-utilities repo. where, Generic Config and Update feature has been labelled as GCU.
+<!--
+ Ensure to add label/tag for the feature raised. example - PR#2174 under sonic-utilities repo. where, Generic Config and Update feature has been labelled as GCU.
+-->
 
 #### Link to config_db schema for YANG module changes
 <!--

.github/workflows/automerge_scan.yml

@@ -30,6 +30,9 @@ jobs:
           echo PR: $(($i+1))/$count, URL: $url, createdAt: $created_at, now: $(date -u +"%FT%TZ")
           [[ "$url" == "" ]] && continue
           [[ $created_at > $(date --date "1 hour ago" -u +"%FT%TZ") ]] && continue
+          # only check automerge PR.
+          cat prs.log | jq -r ".[$i].labels[].name" | grep automerge || continue
+
           checks=$(cat prs.log | jq ".[$i].statusCheckRollup")
           checks_count=$(echo $checks | jq 'length')
           echo Checks count: $checks_count
@@ -38,9 +41,14 @@ jobs:
             check=$(echo $checks | jq ".[$j]")
             state=$(echo $check | jq -r '.state')
             conclusion=$(echo $check | jq -r '.conclusion')
+            name=$(echo $check | jq -r '.name')
 
             # EasyCLA success flag:  state=SUCCESS
             # Others success flag:   conclusion in SUCCESS,NEUTRAL
+            # Ignore Azure.sonic-buildimage stage check result. It may be set continueOnError
+            echo "$name" | grep "Azure.sonic-buildimage (" && continue
+            # Ignore Semgrep, it has issues.
+            [[ "$name" == "Semgrep" ]] && continue
             if [[ "$state" == "SUCCESS" ]];then
               # check pass
               continue

.gitignore

@@ -63,6 +63,12 @@ platform/**/*.egg-info
 platform/**/*-none-any.whl
 platform/**/.pybuild
 platform/**/debian/*
+!platform/**/debian/control
+!platform/**/debian/rules
+!platform/**/debian/changelog
+!platform/**/debian/compat
+!platform/**/debian/*.postinst
+!platform/**/debian/*.install
 platform/**/build
 platform/**/*.ko
 platform/**/*.mod.c

.gitmodules

@@ -112,3 +112,6 @@
 [submodule "src/sonic-genl-packet"]
 	path = src/sonic-genl-packet
 	url = https://github.com/sonic-net/sonic-genl-packet
+[submodule "src/dhcpmon"]
+	path = src/dhcpmon
+	url = https://github.com/sonic-net/sonic-dhcpmon.git

Makefile.work

@@ -224,6 +224,7 @@ PREPARE_DOCKER=BUILD_SLAVE=y \
 	DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
 	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
 	DBGOPT='$(DBGOPT)' \
+	MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
 	scripts/prepare_docker_buildinfo.sh \
 		$(SLAVE_BASE_IMAGE) \
 		$(SLAVE_DIR)/Dockerfile \
@@ -245,6 +246,7 @@ SLAVE_BASE_TAG = $(shell \
 SLAVE_TAG = $(shell \
 	(cat $(SLAVE_DIR)/Dockerfile.user \
 		 $(SLAVE_DIR)/Dockerfile \
+		 $(SLAVE_DIR)/sources.list.* \
 		 $(SLAVE_DIR)/buildinfo/versions/versions-* \
 	&& echo $(USER)/$(PWD)/$(CONFIGURED_PLATFORM)) \
 	| sha1sum \
@@ -309,6 +311,19 @@ ifdef SONIC_BUILD_QUIETER
 	DOCKER_RUN += -e "SONIC_BUILD_QUIETER=$(SONIC_BUILD_QUIETER)"
 endif
 
+# Mount the Signing key and Certificate in the slave container
+ifneq ($(SECURE_UPGRADE_DEV_SIGNING_KEY),)
+    DOCKER_RUN += -v $(SECURE_UPGRADE_DEV_SIGNING_KEY):$(SECURE_UPGRADE_DEV_SIGNING_KEY):ro
+endif
+ifneq ($(SECURE_UPGRADE_SIGNING_CERT),)
+    DOCKER_RUN += -v $(SECURE_UPGRADE_SIGNING_CERT):$(SECURE_UPGRADE_SIGNING_CERT):ro
+endif
+# Mount the Signing prod tool in the slave container
+$(info "SECURE_UPGRADE_PROD_SIGNING_TOOL": "$(SECURE_UPGRADE_PROD_SIGNING_TOOL)")
+ifneq ($(SECURE_UPGRADE_PROD_SIGNING_TOOL),)
+	DOCKER_RUN += -v $(SECURE_UPGRADE_PROD_SIGNING_TOOL):/sonic/scripts/$(shell basename -- $(SECURE_UPGRADE_PROD_SIGNING_TOOL)):ro
+endif
+
 ifneq ($(SONIC_DPKG_CACHE_SOURCE),)
 	DOCKER_RUN += -v "$(SONIC_DPKG_CACHE_SOURCE):/dpkg_cache:rw"
 endif
@@ -399,7 +414,7 @@ endif
 
 endif
 
-SPLIT_LOG = | tee
+SPLIT_LOG = 2>&1 | tee
 
 DOCKER_BASE_LOG = $(SLAVE_DIR)/$(SLAVE_BASE_IMAGE)_$(SLAVE_BASE_TAG).log
 DOCKER_LOG = $(SLAVE_DIR)/$(SLAVE_IMAGE)_$(SLAVE_TAG).log
@@ -416,7 +431,7 @@ DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \
 		    $(SPLIT_LOG) $(DOCKER_BASE_LOG)
 
 DOCKER_BASE_PULL = docker pull \
-			$(REGISTRY_SERVER):$(REGISTRY_PORT)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)
+			$(REGISTRY_SERVER):$(REGISTRY_PORT)$(REGISTRY_SERVER_PATH)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)
 
 DOCKER_USER_BUILD = docker build --no-cache \
 	       --build-arg user=$(USER) \
@@ -442,7 +457,7 @@ DOCKER_SLAVE_BASE_PULL_REGISTRY = \
 		$(DOCKER_BASE_PULL); \
 	} && \
 	{ \
-		docker tag $(REGISTRY_SERVER):$(REGISTRY_PORT)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) && \
+		docker tag $(REGISTRY_SERVER):$(REGISTRY_PORT)$(REGISTRY_SERVER_PATH)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) && \
 		$(COLLECT_DOCKER); \
 	}\
 
@@ -490,6 +505,7 @@ SONIC_BUILD_INSTRUCTION :=  $(MAKE) \
                            SHUTDOWN_BGP_ON_START=$(SHUTDOWN_BGP_ON_START) \
                            INCLUDE_KUBERNETES=$(INCLUDE_KUBERNETES) \
                            KUBERNETES_VERSION=$(KUBERNETES_VERSION) \
+                           KUBERNETES_CNI_VERSION=$(KUBERNETES_CNI_VERSION) \
                            K8s_GCR_IO_PAUSE_VERSION=$(K8s_GCR_IO_PAUSE_VERSION) \
                            INCLUDE_KUBERNETES_MASTER=$(INCLUDE_KUBERNETES_MASTER) \
                            SONIC_ENABLE_PFCWD_ON_START=$(ENABLE_PFCWD_ON_START) \
@@ -525,6 +541,11 @@ SONIC_BUILD_INSTRUCTION :=  $(MAKE) \
                            BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \
                            SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
                            SONIC_ENABLE_SECUREBOOT_SIGNATURE=$(SONIC_ENABLE_SECUREBOOT_SIGNATURE) \
+                           SECURE_UPGRADE_MODE=$(SECURE_UPGRADE_MODE) \
+                           SECURE_UPGRADE_DEV_SIGNING_KEY=$(SECURE_UPGRADE_DEV_SIGNING_KEY) \
+                           SECURE_UPGRADE_SIGNING_CERT=$(SECURE_UPGRADE_SIGNING_CERT) \
+                           SECURE_UPGRADE_PROD_SIGNING_TOOL=$(SECURE_UPGRADE_PROD_SIGNING_TOOL) \
+                           SECURE_UPGRADE_PROD_TOOL_ARGS=$(SECURE_UPGRADE_PROD_TOOL_ARGS) \
                            SONIC_DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
                            ENABLE_HOST_SERVICE_ON_START=$(ENABLE_HOST_SERVICE_ON_START) \
                            SLAVE_DIR=$(SLAVE_DIR) \
@@ -540,6 +561,8 @@ SONIC_BUILD_INSTRUCTION :=  $(MAKE) \
                            MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \
                            GZ_COMPRESS_PROGRAM=$(GZ_COMPRESS_PROGRAM) \
                            MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
+                           SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
+                           SONIC_OS_VERSION=$(SONIC_OS_VERSION) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset
@@ -560,6 +583,7 @@ endif
 
 export MIRROR_URLS
 export MIRROR_SECURITY_URLS
+export MIRROR_SNAPSHOT
 export SONIC_VERSION_CONTROL_COMPONENTS
 
 %:: | sonic-build-hooks

README.md

@@ -227,7 +227,7 @@ make configure PLATFORM=marvell-arm64 PLATFORM_ARCH=arm64
     `/usr/bin/tar: /path/to/sonic-buildimage/<some_file>:
      Cannot write: No space left on device`
 
-    The solution is to [move the directory](https://linuxconfig.org/how-to-move-docker-s-default-var-lib-docker-to-another-directory-on-ubuntu-debian-linux)
+    The solution is to [move the directory](https://www.ibm.com/docs/en/z-logdata-analytics/5.1.0?topic=compose-relocating-docker-root-directory)
     to a partition with more free space.
 * Use
   `http_proxy=[your_proxy] https_proxy=[your_proxy] no_proxy=[your_no_proxy] make`