Merge branch 'as4630-54pe_20210721_api2_202012br' of https://github.com/jostar-yang/sonic-buildimage into as4630-54pe_20210721_api2_202012br

Author: Jostar Yang

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -14,6 +14,14 @@ schedules:
     - 202012
   always: true
 
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
 pool: sonicbld
 
 parameters:
@@ -28,6 +36,8 @@ parameters:
     - innovium
     - mellanox
     - nephos
+    - centec-arm64
+    - marvell-armhf
 
 stages:
 - stage: Build
@@ -37,20 +47,10 @@ stages:
   jobs:
   - template: azure-pipelines-build.yml
     parameters:
-      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
       jobFilters: ${{ parameters.jobFilters }}
+      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} ENABLE_DOCKER_BASE_PULL=n SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
       preSteps:
-      - script: |
-          containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
-          if [ ! -z "$containers" ]; then
-            docker container kill $containers || true
-            sleep 5
-          fi
-          images=$(docker images 'sonic-slave-*' -a -q)
-          if [ ! -z "$images" ]; then
-            docker rmi -f $images
-          fi
-        displayName: 'Cleanup sonic slave'
+      - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
 - stage: UpgradeVersions
   jobs:
   - job: UpgradeVersions

.azure-pipelines/azure-pipelines-build.yml

@@ -64,7 +64,7 @@ jobs:
             platform_rpc: centec
 
         - name: centec-arm64
-          pool: sonicbld_8c
+          pool: sonicbld-arm64
           timeoutInMinutes: 2880
           variables:
             PLATFORM_ARCH: arm64
@@ -78,7 +78,7 @@ jobs:
             dbg_image: yes
 
         - name: marvell-armhf
-          pool: sonicbld_8c
+          pool: sonicbld-armhf
           timeoutInMinutes: 2880
           variables:
             PLATFORM_ARCH: armhf
@@ -101,13 +101,19 @@ jobs:
           set -ex
           if [ $(GROUP_NAME) == vs ]; then
             if [ $(dbg_image) == yes ]; then
-              make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz && mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
+              make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz
+              mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
             fi
             make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+            if [ $(Build.Reason) != 'PullRequest' ];then
+              gzip -kd target/sonic-vs.img.gz
+              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make sonic-slave-run
+              rm target/sonic-vs.img
+            fi
           else
             if [ $(dbg_image) == yes ]; then
-              make  $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-$(GROUP_NAME).bin && \
-                mv target/sonic-$(GROUP_NAME).bin target/sonic-$(GROUP_NAME)-dbg.bin
+              make  $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-$(GROUP_NAME).bin
+              mv target/sonic-$(GROUP_NAME).bin target/sonic-$(GROUP_NAME)-dbg.bin
             fi
             if [ $(swi_image) == yes ]; then
               make $BUILD_OPTIONS ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-$(GROUP_NAME).swi
@@ -125,3 +131,4 @@ jobs:
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
         displayName: "Build sonic image"
+      - template: check-dirty-version.yml

.azure-pipelines/azure-pipelines-image-template.yml

@@ -26,16 +26,17 @@ jobs:
     jobVariables: ${{ parameters.jobVariables }}
     preSteps:
       - template: cleanup.yml
-      - ${{ parameters. preSteps }}
+      - ${{ parameters.preSteps }}
       - script: |
-          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox)$"; then
+          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf)$"; then
             CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM_AZP)"
             BUILD_OPTIONS="$(BUILD_OPTIONS) $CACHE_OPTIONS"
             echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
           fi
         displayName: "Set cache options"
       - checkout: self
         submodules: recursive
+        condition: eq(variables.SKIP_CHECKOUT, '') 
         displayName: 'Checkout code'
       - script: |
           BRANCH_NAME=$(Build.SourceBranchName)

.azure-pipelines/build-commonlib.yml

@@ -0,0 +1,19 @@
+pr: none
+trigger: none
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily build
+  branches:
+    include:
+      - master
+      - 202???
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
+jobs:
+- template: .azure-pipelines/template-commonlib.yml@buildimage

.azure-pipelines/check-dirty-version.yml

@@ -0,0 +1,16 @@
+steps:
+- script: |
+    . functions.sh
+    SONIC_VERSION=$(sonic_get_version)
+    echo "SONIC_VERSION=$SONIC_VERSION"
+    if [[ "$SONIC_VERSION" == *dirty* ]]; then
+      # Print the detail dirty info
+      git status --untracked-files=no -s --ignore-submodules
+
+      # Exit with error, if it is a PR build
+      if [ "$(Build.Reason)" == "PullRequest" ]; then
+        echo "Build failed for the dirty version: $SONIC_VERSION" 1>&2
+        exit 1
+      fi
+    fi
+  displayName: "Check the dirty version"

.azure-pipelines/docker-sonic-slave-template.yml

@@ -0,0 +1,106 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+# Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
+parameters:
+- name: arch
+  type: string
+  values:
+  - amd64
+  - armhf
+  - arm64
+- name: dist
+  type: string
+  values:
+  - bullseye
+  - buster
+  - stretch
+  - jessie
+- name: registry_url
+  type: string
+  default: sonicdev-microsoft.azurecr.io
+- name: registry_conn
+  type: string
+  default: sonicdev
+- name: pool
+  type: string
+  default: sonicbld
+  values:
+  - sonicbld
+  - sonicbld-arm64
+  - sonicbld-armhf
+
+jobs:
+- job: Build_${{ parameters.dist }}_${{ parameters.arch }}
+  timeoutInMinutes: 360
+  pool: ${{ parameters.pool }}
+  steps:
+  - template: cleanup.yml
+  - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
+  - checkout: self
+    clean: true
+    submodules: recursive
+  - bash: |
+      set -ex
+
+      SLAVE_DIR=sonic-slave-${{ parameters.dist }}
+      if [ x${{ parameters.pool }} == x"sonicbld" ]; then
+        if [ x${{ parameters.arch }} == x"amd64" ]; then
+          SLAVE_BASE_IMAGE=${SLAVE_DIR}
+          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}
+        elif [ x${{ parameters.pool }} == x"sonicbld" ]; then
+          SLAVE_BASE_IMAGE=${SLAVE_DIR}-march-${{ parameters.arch }}
+          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-march-${{ parameters.arch }}
+        fi
+      elif [[ x${{ parameters.pool }} == x"sonicbld-armhf" && x${{ parameters.arch }} == x"armhf" ]]; then
+        SLAVE_BASE_IMAGE=${SLAVE_DIR}
+        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-armhf
+      elif [[ x${{ parameters.pool }} == x"sonicbld-arm64" && x${{ parameters.arch }} == x"arm64" ]]; then
+        SLAVE_BASE_IMAGE=${SLAVE_DIR}
+        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-arm64
+      else
+        echo "do not support build ${{ parameters.arch }} on ${{ parameters.pool }}"
+        exit 1
+      fi
+
+      if [ x"$(Build.SourceBranchName)" == x"202012" ]; then
+        BUILD_OPTIONS = 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
+      fi
+
+      tmpfile=$(mktemp)
+
+      echo ${{ parameters.arch }} > .arch
+
+      DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker BLDENV=${{ parameters.dist }} $(BUILD_OPTIONS) make -f Makefile.work sonic-slave-build | tee $tmpfile
+      SLAVE_BASE_TAG=$(grep "^Checking sonic-slave-base image:" $tmpfile | awk -F ':' '{print $3}')
+      SLAVE_TAG=$(grep "^Checking sonic-slave image:" $tmpfile | awk -F ':' '{print $3}')
+
+      mkdir -p target
+
+      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:latest
+      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:$SLAVE_BASE_TAG
+      set +x
+      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_IMAGE]$SLAVE_BASE_IMAGE_UPLOAD"
+      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_TAG]$SLAVE_BASE_TAG"
+    env:
+      REGISTRY_SERVER: ${{ parameters.registry_url }}
+    displayName: Build sonic-slave-${{ parameters.dist }}-${{ parameters.arch }}
+
+  - task: Docker@2
+    displayName: Upload image
+    inputs:
+      containerRegistry: ${{ parameters.registry_conn }}
+      repository: $(VARIABLE_SLAVE_BASE_IMAGE)
+      command: push
+      tags: |
+        $(VARIABLE_SLAVE_BASE_TAG)
+        latest

.azure-pipelines/docker-sonic-slave.yml

@@ -21,6 +21,7 @@ pr:
     - sonic-slave-jessie
     - sonic-slave-stretch
     - sonic-slave-buster
+    - src/sonic-build-hooks
 
 parameters:
 - name: 'arches'
@@ -43,9 +44,9 @@ parameters:
   default: sonicdev
 
 variables:
-- ${{ if and(startsWith(variables['Build.SourceBranchName'], '202'), eq(length(variables['Build.SourceBranchName']), 6)) }}:
+- ${{ if eq(variables['Build.SourceBranchName'], '202012') }}:
   - name: BUILD_OPTIONS
-    value: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web'
+    value: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
 
 stages:
 - stage: Build

.azure-pipelines/official-build-cisco-8000.yml

@@ -0,0 +1,94 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+schedules:
+- cron: "0 8 * * *"
+  displayName: Daily midnight build
+  branches:
+    include:
+    - master
+    - 202012
+  always: true
+
+trigger: none
+pr: none
+
+resources:
+  repositories:
+  - repository: cisco-8000
+    type: github
+    name: Cisco-8000-sonic/platform-cisco-8000
+    endpoint: cisco-connection
+
+variables:
+- group: SONIC-AKV-STROAGE-1
+- name: StorageSASKey
+  value: $(sonicstorage-SasToken)
+
+stages:
+- stage: Build
+  pool: sonic
+  variables:
+    CACHE_MODE: wcache
+    SKIP_CHECKOUT: true
+    TERM: ''
+    PACKAGE_URL: "https://sonicstorage.blob.core.windows.net/packages"
+
+  jobs:
+  - template: azure-pipelines-build.yml
+    parameters:
+      buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) ${{ variables.VERSION_CONTROL_OPTIONS }}'
+      preSteps:
+      - checkout: self
+        submodules: recursive
+        path: s
+        displayName: 'Checkout code'
+      - checkout: cisco-8000
+        submodules: recursive
+        path: s/platform/cisco-8000
+        displayName: 'Checkout cisco-8000'
+      - script: |
+          cisco_ref=$(cat platform/checkout/cisco-8000.ini | grep '^ref=' | cut -d= -f2)
+          echo "The cisco 8000 release version is $cisco_ref"
+          echo "##vso[task.setvariable variable=cisco.ref]$cisco_ref"
+        displayName: 'Export cisco release version'
+      - task: DownloadGitHubRelease@0
+        inputs:
+          connection: cisco-connection
+          userRepository: Cisco-8000-sonic/platform-cisco-8000
+          defaultVersionType: specificTag
+          version: $(cisco.ref)
+          itemPattern: 'artifactory-*.tar.gz'
+          downloadPath: '$(System.ArtifactsDirectory)'
+      - script: |
+          make PLATFORM=cisco-8000 platform/cisco-8000
+          tar xfz $(System.ArtifactsDirectory)/artifactory-*.tar.gz -C platform/cisco-8000
+        displayName: 'Setup cisco artifacts'
+      - script: |
+          set -ex
+          filename=$(find platform/cisco-8000/artifactory/sonic -name cisco-* -type f | head -n 1)
+          if [ -z "$filename" ]; then
+            echo "Cisco sai package not found" 1>&2
+            exit 1
+          fi
+          cd $(dirname $filename)
+          echo "PWD=$(pwd)"
+          ls -l *.deb
+          while read -r package; do
+            # Cisco version format: <VERSION>-sai-<sai-ver>-<distribution>-<COMMIT HASH>
+            # The <sai-ver> may contain several values in one build, the part is skipped when publishing to storage
+            # See https://github.com/Cisco-8000-sonic/sdk/blob/master/azure-pipelines.yml
+            # The $PACKAGE_URL is only accessible for AZP
+            version=$(echo $package | awk -F_ '{print $(NF-1)}' | cut -d- -f1,2,4,5)
+            package_url="$PACKAGE_URL/sai/ciscosai/master/$version/$package"
+            echo "Override package $package from $package_url"
+            wget "$package_url$StorageSASKey" -O "$package"
+          done < <(ls *.deb)
+        env:
+          StorageSASKey: $(StorageSASKey)
+        condition: ne(variables['Build.Reason'], 'PullRequest')
+        displayName: "Override cisco sai packages"
+      jobGroups:
+      - name: cisco-8000

.azure-pipelines/official-build.yml

@@ -21,7 +21,7 @@ stages:
   variables:
     CACHE_MODE: wcache
     ${{ if eq(variables['Build.SourceBranchName'], '202012') }}:
-      VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web'
+      VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
   jobs:
   - template: azure-pipelines-build.yml
     parameters:

.azure-pipelines/template-clean-sonic-slave.yml

@@ -0,0 +1,8 @@
+steps:
+- script: |
+    containers=$(docker container ls -a | grep "sonic-slave" | awk '{ print $1 }')
+    [ -n "$containers" ] && docker container rm -f $containers
+    docker images | grep "^<none>" | awk '{print$3}' | xargs -i docker rmi {}
+    images=$(docker images 'sonic-slave-*' -a -q)
+    [ -n "$images" ] && docker rmi -f $images
+  displayName: 'Cleanup sonic slave'