Merge pull request sonic-net#207 from Yakiv-Huryk/sonic-net-merge

Merge remote-tracking branch 'sonic-net/master' into bluefield

Author: oleksandrivantsiv

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -42,12 +42,32 @@ parameters:
   - mellanox
 
 stages:
+- stage: Prepare
+  jobs:
+  - job: Prepare
+    steps:
+    - script: |
+        DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
+        DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp)
+        DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp)
+        echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_TIMESTAMP;isOutput=true]$DEBIAN_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_SECURITY_TIMESTAMP;isOutput=true]$DEBIAN_SECURITY_TIMESTAMP"
+      name: SetVersions
+      displayName: 'Set snapshot versions'
 - stage: Build
+  dependsOn: Prepare
   variables:
     - name: CACHE_MODE
       value: none
     - name: VERSION_CONTROL_OPTIONS
       value: 'SONIC_VERSION_CONTROL_COMPONENTS='
+    - name: SKIP_CHECKOUT
+      value: true
+    - name: DEBIAN_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_TIMESTAMP'] ]
+    - name: DEBIAN_SECURITY_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_SECURITY_TIMESTAMP'] ]
     - template: .azure-pipelines/template-variables.yml@buildimage
   jobs:
   - template: azure-pipelines-build.yml
@@ -56,6 +76,21 @@ stages:
       buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} ENABLE_DOCKER_BASE_PULL=n SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
       preSteps:
       - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
+      - checkout: self
+        submodules: recursive
+        fetchDepth: 0
+        path: s
+        displayName: 'Checkout code'
+      - script: |
+          echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+          if [ "$MIRROR_SNAPSHOT" == y ]; then
+            mkdir -p target/versions/default/
+            echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror
+            echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror
+            cat target/versions/default/versions-mirror
+          fi
+        displayName: 'Set snapshot versions'
+
 - stage: UpgradeVersions
   jobs:
   - job: UpgradeVersions

.azure-pipelines/azure-pipelines-build.yml

@@ -104,6 +104,15 @@ jobs:
           variables:
             PLATFORM_ARCH: armhf
 
+        - name: marvell-arm64
+          ${{ if not(parameters.qemuOrCrossBuild) }}:
+            pool: sonicbld-arm64
+          timeoutInMinutes: 2880
+          variables:
+            PLATFORM_ARCH: arm64
+
+        - name: marvell
+
         - name: mellanox
           variables:
             dbg_image: yes
@@ -118,7 +127,8 @@ jobs:
             platform_rpc: nephos
 
     buildSteps:
-      - template: template-skipvstest.yml
+      - template: .azure-pipelines/template-skipvstest.yml@buildimage
+      - template: .azure-pipelines/template-daemon.yml@buildimage
       - bash: |
           set -ex
           if [ $(GROUP_NAME) == vs ]; then
@@ -137,7 +147,7 @@ jobs:
             fi
             if [ $(Build.Reason) != 'PullRequest' ];then
               gzip -kd target/sonic-vs.img.gz
-              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make sonic-slave-run
+              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make $BUILD_OPTIONS sonic-slave-run
               rm target/sonic-vs.img
             fi
           else
@@ -162,16 +172,6 @@ jobs:
                 pushd ./src/sonic-sairedis/SAI
                 git stash
                 popd 
-                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
-                pushd ./src/sonic-sairedis/SAI
-                git stash
-                popd
-              fi
-              if [ $(GROUP_NAME) == barefoot ]; then
-                make $BUILD_OPTIONS SAITHRIFT_V2=y ENABLE_SYNCD_RPC=y target/docker-saiserverv2-bfn.gz
-                pushd ./src/sonic-sairedis/SAI
-                git stash
-                popd
               fi
             fi
             if [ $(syncd_rpc_image) == yes ]; then
@@ -187,4 +187,4 @@ jobs:
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
         displayName: "Build sonic image"
-      - template: check-dirty-version.yml
+      - template: .azure-pipelines/check-dirty-version.yml@buildimage

.azure-pipelines/azure-pipelines-image-template.yml

@@ -25,14 +25,17 @@ jobs:
     jobFilters: ${{ parameters.jobFilters }}
     jobVariables: ${{ parameters.jobVariables }}
     preSteps:
-      - template: cleanup.yml
+      - template: .azure-pipelines/cleanup.yml@buildimage
       - ${{ parameters.preSteps }}
       - script: |
+          [ -n "$OVERRIDE_BUILD_OPTIONS" ] && OVERRIDE_BUILD_OPTIONS=$(OVERRIDE_BUILD_OPTIONS)
+          BUILD_OPTIONS="$(BUILD_OPTIONS) $OVERRIDE_BUILD_OPTIONS"
           if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf)$"; then
             CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM_AZP)"
-            BUILD_OPTIONS="$(BUILD_OPTIONS) $CACHE_OPTIONS"
-            echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
+            BUILD_OPTIONS="$BUILD_OPTIONS $CACHE_OPTIONS"
           fi
+          echo $BUILD_OPTIONS
+          echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
         displayName: "Set cache options"
       - checkout: self
         submodules: recursive
@@ -67,6 +70,6 @@ jobs:
             artifactName: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
             publishPrefix: '$(Build.DefinitionName)/$(Build.SourceBranchName)/$(GROUP_NAME)'
       - ${{ parameters.postSteps }}
-      - template: cleanup.yml
+      - template: .azure-pipelines/cleanup.yml@buildimage
     jobGroups: ${{ parameters.jobGroups }}
     buildSteps: ${{ parameters.buildSteps }}

.azure-pipelines/azure-pipelines-repd-build-variables.yml

@@ -1,2 +1,2 @@
 variables:
-  VERSION_CONTROL_OPTIONS: ''
+  VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'

.azure-pipelines/cleanup.yml

@@ -1,5 +1,11 @@
 steps:
 - script: |
+    set -x
+    # kill daemon process
+    ps $(cat /tmp/azp_daemon_kill_docker_pid)
+    sudo kill $(cat /tmp/azp_daemon_kill_docker_pid)
+    rm /tmp/azp_daemon_kill_docker_pid
+
     if sudo [ -f /var/run/march/docker.pid ] ; then
         pid=`sudo cat /var/run/march/docker.pid` ; sudo kill $pid
     fi
@@ -11,4 +17,5 @@ steps:
         pid=`sudo cat dockerfs/var/run/docker.pid` ; sudo kill $pid
     fi
     sudo rm -rf $(ls -A1)
+  condition: always()
   displayName: "Clean Workspace"

.azure-pipelines/official-build-cache.yml

@@ -11,6 +11,14 @@ schedules:
     - master
     - 202012
 
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: sonic-net/sonic-buildimage
+    ref: master
+    endpoint: sonic-net
+
 trigger: none
 pr: none
 
@@ -20,7 +28,8 @@ stages:
   variables:
   - name: CACHE_MODE
     value: cache
-  - template: azure-pipelines-repd-build-variables.yml
+  - template: .azure-pipelines/azure-pipelines-repd-build-variables.yml@buildimage
+  - template: .azure-pipelines/template-variables.yml@buildimage
   jobs:
   - template: azure-pipelines-build.yml
     parameters:

.azure-pipelines/run-test-scheduler-template.yml

@@ -48,13 +48,13 @@ parameters:
 
 steps:
   - script: |
-      set -ex
+      set -e
       wget -O ./.azure-pipelines/test_plan.py  https://raw.githubusercontent.com/sonic-net/sonic-mgmt/master/.azure-pipelines/test_plan.py
       wget -O ./.azure-pipelines/pr_test_scripts.yaml  https://raw.githubusercontent.com/sonic-net/sonic-mgmt/master/.azure-pipelines/pr_test_scripts.yaml
-    displayName: Download TestbedV2 scripts
+    displayName: Download test plan scripts
 
   - script: |
-      set -ex
+      set -e
       pip install PyYAML
       rm -f new_test_plan_id.txt
       python ./.azure-pipelines/test_plan.py create -t ${{ parameters.TOPOLOGY }} -o new_test_plan_id.txt \
@@ -66,7 +66,7 @@ steps:
       TEST_PLAN_ID=`cat new_test_plan_id.txt`
 
       echo "Created test plan $TEST_PLAN_ID"
-      echo "Check https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID for test plan status"
+      echo "Check $(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID for test plan status"
       echo "##vso[task.setvariable variable=TEST_PLAN_ID]$TEST_PLAN_ID"
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
@@ -76,45 +76,44 @@ steps:
     displayName: Trigger test
 
   - script: |
-      set -ex
+      set -e
       echo "Lock testbed"
-      echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
-      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      echo "SONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com"
+      echo "Runtime detailed progress at $(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
       # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED"
       python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state LOCK_TESTBED
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Lock testbed
 
   - script: |
-      set -ex
+      set -e
       echo "Prepare testbed"
-      echo "Preparing the testbed(add-topo, deploy-mg) may take 15-30 minutes. Before the testbed is ready, the progress of the test plan keeps displayed as 0, please be patient(We will improve the indication in a short time)"
-      echo "If the progress keeps as 0 for more than 1 hour, please cancel and retry this pipeline"
-      echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to [email protected]"
-      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      echo "Preparing the testbed(add-topo, deploy-mg) may take 15-30 minutes. Before the testbed is ready, the progress of the test plan keeps displayed as 0, please be patient"
+      echo "SONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to [email protected]"
+      echo "Runtime detailed progress at $(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
       # When "PREPARE_TESTBED" finish, it changes into "EXECUTING"
       python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state PREPARE_TESTBED
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Prepare testbed
 
   - script: |
-      set -ex
+      set -e
       echo "Run test"
-      echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
-      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      echo "SONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com"
+      echo "Runtime detailed progress at $(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
       # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED"
       python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state EXECUTING
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Run test
 
   - script: |
-      set -ex
+      set -e
       echo "KVM dump"
-      echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
-      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      echo "SONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com"
+      echo "Runtime detailed progress at $(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
       # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
       python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state KVMDUMP
     condition: succeededOrFailed()
@@ -123,7 +122,7 @@ steps:
     displayName: KVM dump
 
   - script: |
-      set -ex
+      set -e
       echo "Try to cancel test plan $TEST_PLAN_ID, cancelling finished test plan has no effect."
       python ./.azure-pipelines/test_plan.py cancel -i "$(TEST_PLAN_ID)"
     condition: always()

.azure-pipelines/template-daemon.yml

@@ -0,0 +1,24 @@
+steps:
+- bash: |
+    (
+      while true
+      do
+        sleep 120
+        now=$(date +%s)
+        pids=$(ps -C docker -o pid,etime,args | grep "docker build" | cut -d" " -f2)
+        for pid in $pids
+        do
+          start=$(date --date="$(ls -dl /proc/$pid --time-style full-iso | awk '{print$6,$7}')" +%s)
+          time_s=$(($now-$start))
+          if [[ $time_s -gt $(DOCKER_BUILD_TIMEOUT) ]]; then
+            echo =========== $(date +%F%T) $time_s &>> target/daemon.log
+            ps $pid &>> target/daemon.log
+            sudo kill $pid
+          fi
+        done
+      done
+    ) &
+    daemon_pid=$!
+    ps $daemon_pid
+    echo $daemon_pid >> /tmp/azp_daemon_kill_docker_pid
+  displayName: start daemon to kill hang docker

.azure-pipelines/template-variables.yml

@@ -5,3 +5,5 @@ variables:
   SONIC_BUILD_RETRY_COUNT: 3
   SONIC_BUILD_RETRY_INTERVAL: 600
   DOCKER_BUILDKIT: 0
+  DOCKER_BUILD_TIMEOUT: 3600
+  MIRROR_SNAPSHOT: y

.azure-pipelines/vstest-exclude

@@ -1,4 +1,3 @@
 ^platform
 ^.azure-pipelines
-^files/build/versions
 ^sonic-slave-

.github/pull_request_template.md

@@ -12,10 +12,17 @@
 
 #### Why I did it
 
+##### Work item tracking
+- Microsoft ADO **(number only)**:
+
 #### How I did it
 
 #### How to verify it
 
+<!--
+If PR needs to be backported, then the PR must be tested against the base branch and the earliest backport release branch and provide tested image version on these two branches. For example, if the PR is requested for master, 202211 and 202012, then the requester needs to provide test results on master and 202012.
+-->
+
 #### Which release branch to backport (provide reason below if selected)
 
 <!--
@@ -33,14 +40,28 @@
 - [ ] 202111
 - [ ] 202205
 - [ ] 202211
+- [ ] 202305
+
+#### Tested branch (Please provide the tested image version)
+
+<!--
+- Please provide tested image version
+- e.g.
+- [x] 20201231.100
+-->
+
+- [ ] <!-- image version 1 -->
+- [ ] <!-- image version 2 -->
 
 #### Description for the changelog
 <!--
 Write a short (one line) summary that describes the changes in this
 pull request for inclusion in the changelog:
 -->
 
-#### Ensure to add label/tag for the feature raised. example - PR#2174 under sonic-utilities repo. where, Generic Config and Update feature has been labelled as GCU.
+<!--
+ Ensure to add label/tag for the feature raised. example - PR#2174 under sonic-utilities repo. where, Generic Config and Update feature has been labelled as GCU.
+-->
 
 #### Link to config_db schema for YANG module changes
 <!--