[ci][asan] add DVS tests run with ASAN (sonic-net#2441)

This adds 3 new stages:

BuildAsan: same as 'Build' but with ENABLE_ASAN=y
BuildDockerAsan: same as 'BuildDocker' but uses asan-enabled swss and
adds swss-dbg package. It also uses asan-enabled docker-sonic-vs from
the sonic-buildimage ci.
TestAsan: same as 'Test' but uses asan-enabled docker and publishes
the asan reports if any. It also shows a warning if asan report dir is not empty
(which means that there is some memory usage issue in swss)

- What I did
Added new CI stages to run the DVS tests with ASAN and check the ASAN reports.

- Why I did it
To utilize ASAN for catching memory usage issues on PR.

- How I verified it
Run the CI on my fork.

Signed-off-by: Yakiv Huryk <[email protected]>

Author: Yakiv-Huryk

.azure-pipelines/build-docker-sonic-vs-template.yml

@@ -23,6 +23,10 @@ parameters:
 - name: artifact_name
   type: string
 
+- name: asan
+  type: boolean
+  default: false
+
 jobs:
 - job:
   displayName: ${{ parameters.arch }}
@@ -66,27 +70,32 @@ jobs:
       runVersion: 'latestFromBranch'
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
       path: $(Build.ArtifactStagingDirectory)/download
-      patterns: '**/target/docker-sonic-vs.gz'
-    displayName: "Download sonic-buildimage docker-sonic-vs"
+      patterns: '**/target/${{ parameters.artifact_name }}.gz'
+    displayName: "Download sonic-buildimage ${{ parameters.artifact_name }}"
   - script: |
       set -ex
       echo $(Build.DefinitionName).$(Build.BuildNumber)
 
-      docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz
+      docker load < $(Build.ArtifactStagingDirectory)/download/target/${{ parameters.artifact_name }}.gz
 
       mkdir -p .azure-pipelines/docker-sonic-vs/debs
 
       cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
 
       pushd .azure-pipelines
 
-      docker build --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) docker-sonic-vs
+      build_args=""
+      if [ '${{ parameters.asan }}' == True ]; then
+        build_args="--build-arg need_dbg=y"
+      fi
+
+      docker build $build_args --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} docker-sonic-vs
 
       popd
 
-      docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
+      docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
       rm -rf $(Build.ArtifactStagingDirectory)/download
-    displayName: "Build docker-sonic-vs"
+    displayName: "Build ${{ parameters.artifact_name }}"
   - publish: $(Build.ArtifactStagingDirectory)/
     artifact: ${{ parameters.artifact_name }}
     displayName: "Archive sonic docker vs image"

.azure-pipelines/build-template.yml

@@ -43,6 +43,10 @@ parameters:
 - name: common_lib_artifact_name
   type: string
 
+- name: asan
+  type: boolean
+  default: false
+
 jobs:
 - job:
   displayName: ${{ parameters.arch }}
@@ -139,6 +143,9 @@ jobs:
       if [ '${{ parameters.archive_gcov }}' == True ]; then
          export ENABLE_GCOV=y
       fi
+      if [ '${{ parameters.asan }}' == True ]; then
+         export ENABLE_ASAN=y
+      fi
       ./autogen.sh
       dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb .
     displayName: "Compile sonic swss"

.azure-pipelines/docker-sonic-vs/Dockerfile

@@ -1,6 +1,7 @@
 FROM docker-sonic-vs
 
 ARG docker_container_name
+ARG need_dbg
 
 ADD ["debs", "/debs"]
 
@@ -18,6 +19,7 @@ RUN dpkg -i /debs/syncd-vs_1.0.0_amd64.deb
 
 RUN dpkg --purge swss
 RUN dpkg -i /debs/swss_1.0.0_amd64.deb
+RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/swss-dbg_1.0.0_amd64.deb ; fi
 
 RUN apt-get update
 

.azure-pipelines/test-docker-sonic-vs-template.yml

@@ -16,6 +16,14 @@ parameters:
   type: boolean
   default: false
 
+- name: docker_sonic_vs_name
+  type: string
+  default: docker-sonic-vs
+
+- name: asan
+  type: boolean
+  default: false
+
 jobs:
 - job:
   displayName: vstest
@@ -30,9 +38,9 @@ jobs:
   - checkout: self
   - task: DownloadPipelineArtifact@2
     inputs:
-      artifact: docker-sonic-vs
+      artifact: ${{ parameters.docker_sonic_vs_name }}
       path: $(Build.ArtifactStagingDirectory)/download
-    displayName: "Download pre-stage built docker-sonic-vs"
+    displayName: "Download pre-stage built ${{ parameters.docker_sonic_vs_name }}"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
@@ -68,40 +76,46 @@ jobs:
       sudo /sbin/ip link del Vrf1 type vrf table 1001
       pushd tests
 
+      params=''
       if [ '${{ parameters.archive_gcov }}' == True ]; then
-        all_tests=$(ls test_*.py)
-        all_tests="${all_tests} p4rt"
-        test_set=()
-        # Run 20 tests as a set.
-        for test in ${all_tests}; do
-          test_set+=("${test}")
-          if [ ${#test_set[@]} -ge 20 ]; then
-            test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
-            echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
-            container_count=$(docker ps -q -a | wc -l)
-            if [ ${container_count} -gt 0 ]; then
-              ./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
-              docker stop $(docker ps -q -a)
-              docker rm $(docker ps -q -a)
-            fi
-            test_set=()
-          fi
-        done
-        if [ ${#test_set[@]} -gt 0 ]; then
+        params='--keeptb'
+      fi
+      if [ '${{ parameters.asan }}' == True ]; then
+        params='--graceful-stop'
+      fi
+
+      all_tests=$(ls test_*.py)
+      all_tests="${all_tests} p4rt"
+      test_set=()
+      # Run 20 tests as a set.
+      for test in ${all_tests}; do
+        test_set+=("${test}")
+        if [ ${#test_set[@]} -ge 20 ]; then
           test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
-          echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
+          echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" $params --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
           container_count=$(docker ps -q -a | wc -l)
-          if [ ${container_count} -gt 0 ]; then
+          if [ '${{ parameters.archive_gcov }}' == True ] && [ ${container_count} -gt 0 ]; then
             ./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
             docker stop $(docker ps -q -a)
             docker rm $(docker ps -q -a)
           fi
+          test_set=()
+        fi
+      done
+      if [ ${#test_set[@]} -gt 0 ]; then
+        test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
+        echo "${test_set[*]}" | xargs sudo py.test -v $params --force-flaky --junitxml="${test_name}_tr.xml" $params --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
+        container_count=$(docker ps -q -a | wc -l)
+        if [ '${{ parameters.archive_gcov }}' == True ] && [ ${container_count} -gt 0 ]; then
+          ./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
+          docker stop $(docker ps -q -a)
+          docker rm $(docker ps -q -a)
         fi
-      else
-        sudo py.test -v --force-flaky --junitxml=tests_tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
       fi
+
       rm -rf $(Build.ArtifactStagingDirectory)/download
     displayName: "Run vs tests"
+    continueOnError: ${{ parameters.asan }}
 
   - task: PublishTestResults@2
     inputs:
@@ -112,6 +126,10 @@ jobs:
   - script: |
       cp -r tests/log $(Build.ArtifactStagingDirectory)/
 
+      if [ '${{ parameters.asan }}' == True ]; then
+        cp -vr tests/log/*/log/asan $(Build.ArtifactStagingDirectory)/
+      fi
+
       if [ '${{ parameters.archive_gcov }}' == True ]; then
         sudo apt-get install -y lcov
         cd $(Build.ArtifactStagingDirectory)/gcov_tmp/
@@ -130,3 +148,19 @@ jobs:
     artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt)
     displayName: "Publish logs"
     condition: always()
+
+  - publish: $(Build.ArtifactStagingDirectory)/asan
+    artifact: asan-reports
+    displayName: "Publish ASAN reports"
+    condition: eq('${{ parameters.asan }}', true)
+
+  - script: |
+      if [ "$(ls -A $(Build.ArtifactStagingDirectory)/asan)" ]; then
+        echo "There are issues reported by ASAN"
+        exit 1
+      else
+        echo "No issues reported by ASAN"
+      fi
+    displayName: "Check ASAN reports"
+    condition: eq('${{ parameters.asan }}', true)
+    continueOnError: true

azure-pipelines.yml

@@ -49,6 +49,19 @@ stages:
       archive_pytests: true
       archive_gcov: true
 
+- stage: BuildAsan
+  dependsOn: []
+  jobs:
+  - template: .azure-pipelines/build-template.yml
+    parameters:
+      arch: amd64
+      sonic_slave: sonic-slave-buster
+      common_lib_artifact_name: common-lib
+      swss_common_artifact_name: sonic-swss-common
+      sairedis_artifact_name: sonic-sairedis
+      artifact_name: sonic-swss-asan
+      asan: true
+
 - stage: BuildArm
   dependsOn: Build
   condition: succeeded('Build')
@@ -88,6 +101,18 @@ stages:
       swss_artifact_name: sonic-swss
       artifact_name: docker-sonic-vs
 
+- stage: BuildDockerAsan
+  dependsOn: BuildAsan
+  condition: succeeded('BuildAsan')
+  jobs:
+  - template: .azure-pipelines/build-docker-sonic-vs-template.yml
+    parameters:
+      swss_common_artifact_name: sonic-swss-common
+      sairedis_artifact_name: sonic-sairedis
+      swss_artifact_name: sonic-swss-asan
+      artifact_name: docker-sonic-vs-asan
+      asan: true
+
 - stage: Test
   dependsOn: BuildDocker
   condition: succeeded('BuildDocker')
@@ -99,6 +124,18 @@ stages:
       sonic_slave: sonic-slave-buster
       archive_gcov: true
 
+- stage: TestAsan
+  dependsOn: BuildDockerAsan
+  condition: succeeded('BuildDockerAsan')
+  jobs:
+  - template: .azure-pipelines/test-docker-sonic-vs-template.yml
+    parameters:
+      log_artifact_name: log-asan
+      gcov_artifact_name: sonic-gcov
+      sonic_slave: sonic-slave-buster
+      docker_sonic_vs_name: docker-sonic-vs-asan
+      asan: true
+
 - stage: Gcov
   dependsOn: Test
   condition: in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues')