Skip to content

Commit ca9edca

Browse files
Yakiv-HurykYakiv-Huryk
authored
[ci][asan] add DVS tests run with ASAN (sonic-net#2363)
This adds 3 new stages: 1. BuildAsan: same as 'Build' but with ENABLE_ASAN=y 2. BuildDockerAsan: same as 'BuildDocker' but uses asan-enabled swss and adds swss-dbg package. It also uses asan-enabled docker-sonic-vs from the sonic-buildimage ci. 3. TestAsan: same as 'Test' but uses asan-enabled docker and publishes the asan reports if any. - What I did Added new CI stages to run the DVS tests with ASAN and check the ASAN reports. - Why I did it To utilize ASAN for catching memory usage issues on PR. - How I verified it Run the CI on my fork Signed-off-by: Yakiv Huryk <[email protected]>
1 parent dec4570 commit ca9edca

File tree

5 files changed

+97
-11
lines changed

5 files changed

+97
-11
lines changed

‎.azure-pipelines/build-docker-sonic-vs-template.yml

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ parameters:
2323
- name: artifact_name
2424
type: string
2525

26+
- name: asan
27+
type: boolean
28+
default: false
29+
2630
jobs:
2731
- job:
2832
displayName: ${{ parameters.arch }}
@@ -66,27 +70,32 @@ jobs:
6670
runVersion: 'latestFromBranch'
6771
runBranch: 'refs/heads/$(BUILD_BRANCH)'
6872
path: $(Build.ArtifactStagingDirectory)/download
69-
patterns: '**/target/docker-sonic-vs.gz'
70-
displayName: "Download sonic-buildimage docker-sonic-vs"
73+
patterns: '**/target/${{ parameters.artifact_name }}.gz'
74+
displayName: "Download sonic-buildimage ${{ parameters.artifact_name }}"
7175
- script: |
7276
set -ex
7377
echo $(Build.DefinitionName).$(Build.BuildNumber)
7478
75-
docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz
79+
docker load < $(Build.ArtifactStagingDirectory)/download/target/${{ parameters.artifact_name }}.gz
7680
7781
mkdir -p .azure-pipelines/docker-sonic-vs/debs
7882
7983
cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
8084
8185
pushd .azure-pipelines
8286
83-
docker build --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) docker-sonic-vs
87+
build_args=""
88+
if [ '${{ parameters.asan }}' == True ]; then
89+
build_args="--build-arg need_dbg=y"
90+
fi
91+
92+
docker build $build_args --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} docker-sonic-vs
8493
8594
popd
8695
87-
docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
96+
docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
8897
rm -rf $(Build.ArtifactStagingDirectory)/download
89-
displayName: "Build docker-sonic-vs"
98+
displayName: "Build ${{ parameters.artifact_name }}"
9099
- publish: $(Build.ArtifactStagingDirectory)/
91100
artifact: ${{ parameters.artifact_name }}
92101
displayName: "Archive sonic docker vs image"

‎.azure-pipelines/build-template.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ parameters:
4343
- name: common_lib_artifact_name
4444
type: string
4545

46+
- name: asan
47+
type: boolean
48+
default: false
49+
4650
jobs:
4751
- job:
4852
displayName: ${{ parameters.arch }}
@@ -138,6 +142,9 @@ jobs:
138142
if [ '${{ parameters.archive_gcov }}' == True ]; then
139143
export ENABLE_GCOV=y
140144
fi
145+
if [ '${{ parameters.asan }}' == True ]; then
146+
export ENABLE_ASAN=y
147+
fi
141148
./autogen.sh
142149
dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb .
143150
displayName: "Compile sonic swss"

‎.azure-pipelines/docker-sonic-vs/Dockerfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
FROM docker-sonic-vs
22

33
ARG docker_container_name
4+
ARG need_dbg
45

56
ADD ["debs", "/debs"]
67

@@ -18,6 +19,7 @@ RUN dpkg -i /debs/syncd-vs_1.0.0_amd64.deb
1819

1920
RUN dpkg --purge swss
2021
RUN dpkg -i /debs/swss_1.0.0_amd64.deb
22+
RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/swss-dbg_1.0.0_amd64.deb ; fi
2123

2224
RUN apt-get update
2325

‎.azure-pipelines/test-docker-sonic-vs-template.yml

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,14 @@ parameters:
1616
type: boolean
1717
default: false
1818

19+
- name: docker_sonic_vs_name
20+
type: string
21+
default: docker-sonic-vs
22+
23+
- name: asan
24+
type: boolean
25+
default: false
26+
1927
jobs:
2028
- job:
2129
displayName: vstest
@@ -30,9 +38,9 @@ jobs:
3038
- checkout: self
3139
- task: DownloadPipelineArtifact@2
3240
inputs:
33-
artifact: docker-sonic-vs
41+
artifact: ${{ parameters.docker_sonic_vs_name }}
3442
path: $(Build.ArtifactStagingDirectory)/download
35-
displayName: "Download pre-stage built docker-sonic-vs"
43+
displayName: "Download pre-stage built ${{ parameters.docker_sonic_vs_name }}"
3644
- task: DownloadPipelineArtifact@2
3745
inputs:
3846
source: specific
@@ -77,7 +85,7 @@ jobs:
7785
test_set+=("${test}")
7886
if [ ${#test_set[@]} -ge 20 ]; then
7987
test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
80-
echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
88+
echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
8189
container_count=$(docker ps -q -a | wc -l)
8290
if [ ${container_count} -gt 0 ]; then
8391
./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
@@ -89,7 +97,7 @@ jobs:
8997
done
9098
if [ ${#test_set[@]} -gt 0 ]; then
9199
test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
92-
echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
100+
echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
93101
container_count=$(docker ps -q -a | wc -l)
94102
if [ ${container_count} -gt 0 ]; then
95103
./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
@@ -98,8 +106,13 @@ jobs:
98106
fi
99107
fi
100108
else
101-
sudo py.test -v --force-flaky --junitxml=tests_tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
109+
test_args=""
110+
if [ '${{ parameters.asan }}' == True ]; then
111+
test_args="--graceful-stop"
112+
fi
113+
sudo py.test $test_args -v --force-flaky --junitxml=tests_tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
102114
fi
115+
103116
rm -rf $(Build.ArtifactStagingDirectory)/download
104117
displayName: "Run vs tests"
105118
@@ -112,6 +125,10 @@ jobs:
112125
- script: |
113126
cp -r tests/log $(Build.ArtifactStagingDirectory)/
114127
128+
if [ '${{ parameters.asan }}' == True ]; then
129+
cp -r tests/log/*/log/asan $(Build.ArtifactStagingDirectory)/
130+
fi
131+
115132
if [ '${{ parameters.archive_gcov }}' == True ]; then
116133
sudo apt-get install -y lcov
117134
cd $(Build.ArtifactStagingDirectory)/gcov_tmp/
@@ -130,3 +147,17 @@ jobs:
130147
artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt)
131148
displayName: "Publish logs"
132149
condition: always()
150+
151+
- publish: $(Build.ArtifactStagingDirectory)/asan
152+
artifact: asan-reports
153+
displayName: "Publish ASAN reports"
154+
condition: eq('${{ parameters.asan }}', true)
155+
156+
- script: |
157+
if [ "$(ls -A $(Build.ArtifactStagingDirectory)/asan)" ]; then
158+
echo "There are issues reported by ASAN"
159+
else
160+
echo "No issues reported by ASAN"
161+
fi
162+
displayName: "Check ASAN reports"
163+
condition: eq('${{ parameters.asan }}', true)

‎azure-pipelines.yml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,19 @@ stages:
4949
archive_pytests: true
5050
archive_gcov: true
5151

52+
- stage: BuildAsan
53+
54+
jobs:
55+
- template: .azure-pipelines/build-template.yml
56+
parameters:
57+
arch: amd64
58+
sonic_slave: sonic-slave-buster
59+
common_lib_artifact_name: common-lib
60+
swss_common_artifact_name: sonic-swss-common
61+
sairedis_artifact_name: sonic-sairedis
62+
artifact_name: sonic-swss-asan
63+
asan: true
64+
5265
- stage: BuildArm
5366
dependsOn: Build
5467
condition: succeeded('Build')
@@ -88,6 +101,18 @@ stages:
88101
swss_artifact_name: sonic-swss
89102
artifact_name: docker-sonic-vs
90103

104+
- stage: BuildDockerAsan
105+
dependsOn: BuildAsan
106+
condition: succeeded('BuildAsan')
107+
jobs:
108+
- template: .azure-pipelines/build-docker-sonic-vs-template.yml
109+
parameters:
110+
swss_common_artifact_name: sonic-swss-common
111+
sairedis_artifact_name: sonic-sairedis
112+
swss_artifact_name: sonic-swss-asan
113+
artifact_name: docker-sonic-vs-asan
114+
asan: true
115+
91116
- stage: Test
92117
dependsOn: BuildDocker
93118
condition: succeeded('BuildDocker')
@@ -99,6 +124,18 @@ stages:
99124
sonic_slave: sonic-slave-buster
100125
archive_gcov: true
101126

127+
- stage: TestAsan
128+
dependsOn: BuildDockerAsan
129+
condition: succeeded('BuildDockerAsan')
130+
jobs:
131+
- template: .azure-pipelines/test-docker-sonic-vs-template.yml
132+
parameters:
133+
log_artifact_name: log-asan
134+
gcov_artifact_name: sonic-gcov
135+
sonic_slave: sonic-slave-buster
136+
docker_sonic_vs_name: docker-sonic-vs-asan
137+
asan: true
138+
102139
- stage: Gcov
103140
dependsOn: Test
104141
condition: always()

0 commit comments

Comments
 (0)