Skip to content

Commit ed7e419

Browse files
liushilongbuaaliushilongbuaa
authored
[ci] Parameterize pipeline and improve azure pipeline (sonic-net#602)
1. Parameterize pipeline. When checkout new release branch, we don't need manually setup azp. 2. Use common lib pipeline instead of buildimage pipeline to get higher success rate. 3. Clean workspace when downloading external packages. This will ensure package version matching.
1 parent a1abaa8 commit ed7e419

6 files changed

+153
-73
lines changed

‎.azure-pipelines/build-docker-sonic-vs-template.yml

+17-8
Original file line numberDiff line numberDiff line change
@@ -32,35 +32,43 @@ jobs:
3232
vmImage: 'ubuntu-20.04'
3333

3434
steps:
35+
- checkout: self
36+
clean: true
3537
- task: DownloadPipelineArtifact@2
3638
inputs:
3739
artifact: ${{ parameters.swss_common_artifact_name }}
38-
displayName: "Download sonic swss common deb packages"
40+
path: $(Build.ArtifactStagingDirectory)/download
41+
displayName: "Download pre-stage built ${{ parameters.swss_common_artifact_name }}"
3942
- task: DownloadPipelineArtifact@2
4043
inputs:
4144
artifact: ${{ parameters.sairedis_artifact_name }}
42-
displayName: "Download sonic sairedis deb packages"
45+
path: $(Build.ArtifactStagingDirectory)/download
46+
displayName: "Download pre-stage built ${{ parameters.sairedis_artifact_name }}"
4347
- task: DownloadPipelineArtifact@2
4448
inputs:
4549
artifact: ${{ parameters.swss_artifact_name }}
46-
displayName: "Download sonic swss artifact"
50+
path: $(Build.ArtifactStagingDirectory)/download
51+
displayName: "Download pre-stage built ${{ parameters.swss_artifact_name }}"
4752
- task: DownloadPipelineArtifact@2
4853
inputs:
4954
source: specific
5055
project: build
51-
pipeline: 1
56+
pipeline: Azure.sonic-buildimage.official.vs
5257
artifact: sonic-buildimage.vs
5358
runVersion: 'latestFromBranch'
54-
runBranch: 'refs/heads/master'
55-
displayName: "Download sonic buildimage"
59+
runBranch: 'refs/heads/$(BUILD_BRANCH)'
60+
path: $(Build.ArtifactStagingDirectory)/download
61+
patterns: '**/target/docker-sonic-vs.gz'
62+
displayName: "Download sonic-buildimage docker-sonic-vs"
5663
- script: |
64+
set -ex
5765
echo $(Build.DefinitionName).$(Build.BuildNumber)
5866
59-
docker load < ../target/docker-sonic-vs.gz
67+
docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz
6068
6169
mkdir -p .azure-pipelines/docker-sonic-vs/debs
6270
63-
cp -v ../*.deb .azure-pipelines/docker-sonic-vs/debs
71+
cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
6472
6573
pushd .azure-pipelines
6674
@@ -69,6 +77,7 @@ jobs:
6977
popd
7078
7179
docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
80+
rm -rf $(Build.ArtifactStagingDirectory)/download
7281
displayName: "Build docker-sonic-vs"
7382
7483
- publish: $(Build.ArtifactStagingDirectory)/

‎.azure-pipelines/build-sairedis-template.yml

+21-9
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,22 @@ jobs:
4141
pool:
4242
${{ if ne(parameters.pool, 'default') }}:
4343
name: ${{ parameters.pool }}
44-
${{ if eq(parameters.pool, 'default') }}:
44+
${{ else }}:
4545
vmImage: 'ubuntu-20.04'
4646

4747
container:
4848
image: sonicdev-microsoft.azurecr.io:443/${{ parameters.sonic_slave }}:latest
4949

5050
steps:
51+
- checkout: sonic-sairedis
52+
submodules: true
53+
clean: true
54+
- script: |
55+
set -ex
56+
git checkout $(BUILD_BRANCH)
57+
git submodule update
58+
git status
59+
displayName: Set up sonic-sairedis branch
5160
- script: |
5261
sudo apt-get install -qq -y \
5362
qtbase5-dev \
@@ -81,18 +90,21 @@ jobs:
8190
- task: DownloadPipelineArtifact@2
8291
inputs:
8392
artifact: ${{ parameters.swss_common_artifact_name }}
84-
displayName: "Download sonic swss common deb packages"
93+
path: $(Build.ArtifactStagingDirectory)/download
94+
displayName: "Download pre-stage built ${{ parameters.swss_common_artifact_name }}"
8595
- script: |
86-
sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb
87-
sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
88-
workingDirectory: $(Pipeline.Workspace)
96+
set -ex
97+
sudo dpkg -i download/libswsscommon_1.0.0_${{ parameters.arch }}.deb
98+
sudo dpkg -i download/libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
99+
rm -rf download || true
100+
workingDirectory: $(Build.ArtifactStagingDirectory)
89101
displayName: "Install sonic swss Common"
90-
- checkout: sonic-sairedis
91-
path: s
92-
submodules: true
93102
- script: |
103+
set -ex
104+
rm ../*.deb || true
94105
./autogen.sh
95-
fakeroot dpkg-buildpackage -b -us -uc -Tbinary-syncd-vs -j$(nproc) && cp ../*.deb .
106+
fakeroot dpkg-buildpackage -b -us -uc -Tbinary-syncd-vs -j$(nproc)
107+
mv ../*.deb .
96108
displayName: "Compile sonic sairedis"
97109
- script: |
98110
sudo cp azsyslog.conf /etc/rsyslog.conf

‎.azure-pipelines/build-swss-template.yml

+48-30
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ parameters:
2121
- name: sonic_slave
2222
type: string
2323

24+
- name: debian_version
25+
type: string
26+
2427
- name: sairedis_artifact_name
2528
type: string
2629

@@ -38,13 +41,22 @@ jobs:
3841
pool:
3942
${{ if ne(parameters.pool, 'default') }}:
4043
name: ${{ parameters.pool }}
41-
${{ if eq(parameters.pool, 'default') }}:
44+
${{ else }}:
4245
vmImage: 'ubuntu-20.04'
4346

4447
container:
4548
image: sonicdev-microsoft.azurecr.io:443/${{ parameters.sonic_slave }}:latest
4649

4750
steps:
51+
- checkout: sonic-swss
52+
submodules: true
53+
clean: true
54+
- script: |
55+
set -ex
56+
git checkout $(BUILD_BRANCH)
57+
git submodule update
58+
git status
59+
displayName: Set up sonic-swss branch
4860
- script: |
4961
sudo apt-get install -y libhiredis0.14 libhiredis-dev
5062
sudo apt-get install -y libzmq5 libzmq3-dev
@@ -59,47 +71,53 @@ jobs:
5971
- task: DownloadPipelineArtifact@2
6072
inputs:
6173
artifact: ${{ parameters.swss_common_artifact_name }}
62-
displayName: "Download sonic swss common deb packages"
74+
path: $(Build.ArtifactStagingDirectory)/download
75+
patterns: |
76+
libswsscommon_1.0.0_*.deb
77+
libswsscommon-dev_1.0.0*.deb
78+
displayName: "Download pre-stage built ${{ parameters.swss_common_artifact_name }}"
6379
- task: DownloadPipelineArtifact@2
6480
inputs:
6581
artifact: ${{ parameters.sairedis_artifact_name }}
66-
displayName: "Download sonic sairedis deb packages"
82+
path: $(Build.ArtifactStagingDirectory)/download
83+
patterns: |
84+
libsaivs_*.deb
85+
libsaivs-dev_*.deb
86+
libsairedis_*.deb
87+
libsairedis-dev_*.deb
88+
libsaimetadata_*.deb
89+
libsaimetadata-dev_*.deb
90+
syncd-vs_*.deb
91+
displayName: "Download pre-stage built ${{ parameters.sairedis_artifact_name }}"
6792
- task: DownloadPipelineArtifact@2
6893
inputs:
6994
source: specific
7095
project: build
71-
pipeline: 1
72-
artifact: sonic-buildimage.vs
96+
pipeline: Azure.sonic-buildimage.common_libs
7397
runVersion: 'latestFromBranch'
74-
runBranch: 'refs/heads/master'
75-
displayName: "Download sonic buildimage"
98+
runBranch: 'refs/heads/$(BUILD_BRANCH)'
99+
path: $(Build.ArtifactStagingDirectory)/download
100+
artifact: common-lib
101+
patterns: |
102+
target/debs/${{ parameters.debian_version }}/libnl-3*.deb
103+
target/debs/${{ parameters.debian_version }}/libnl-genl*.deb
104+
target/debs/${{ parameters.debian_version }}/libnl-route*.deb
105+
target/debs/${{ parameters.debian_version }}/libnl-nf*.deb
106+
displayName: "Download common libs"
107+
76108
- script: |
77-
sudo dpkg -i target/debs/buster/libnl-3-200_*.deb
78-
sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb
79-
sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb
80-
sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb
81-
sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb
82-
sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb
83-
sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb
84-
sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb
85-
sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb
86-
sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
87-
sudo dpkg -i libsaivs_*.deb
88-
sudo dpkg -i libsaivs-dev_*.deb
89-
sudo dpkg -i libsairedis_*.deb
90-
sudo dpkg -i libsairedis-dev_*.deb
91-
sudo dpkg -i libsaimetadata_*.deb
92-
sudo dpkg -i libsaimetadata-dev_*.deb
93-
sudo dpkg -i syncd-vs_*.deb
94-
workingDirectory: $(Pipeline.Workspace)
109+
set -ex
110+
sudo dpkg -i $(find ./download -name *.deb)
111+
rm -rf download || true
112+
workingDirectory: $(Build.ArtifactStagingDirectory)
95113
displayName: "Install libnl3, sonic swss common, and sairedis"
96-
- checkout: sonic-swss
97-
path: s
98-
submodules: true
99114
- script: |
115+
set -ex
116+
rm ../*.deb || true
100117
./autogen.sh
101-
dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb .
118+
dpkg-buildpackage -us -uc -b -j$(nproc)
119+
mv ../*.deb $(Build.ArtifactStagingDirectory)
102120
displayName: "Compile sonic swss"
103-
- publish: $(System.DefaultWorkingDirectory)/
121+
- publish: $(Build.ArtifactStagingDirectory)
104122
artifact: ${{ parameters.artifact_name }}
105123
displayName: "Archive swss debian packages"

‎.azure-pipelines/build-template.yml

+7-3
Original file line numberDiff line numberDiff line change
@@ -38,20 +38,22 @@ jobs:
3838
displayName: ${{ parameters.arch }}
3939
timeoutInMinutes: ${{ parameters.timeout }}
4040
variables:
41-
DIFF_COVER_CHECK_THRESHOLD: 0
41+
DIFF_COVER_CHECK_THRESHOLD: 50
4242
${{ if eq(parameters.run_unit_test, true) }}:
4343
DIFF_COVER_ENABLE: 'true'
4444

4545
pool:
4646
${{ if ne(parameters.pool, 'default') }}:
4747
name: ${{ parameters.pool }}
48-
${{ if eq(parameters.pool, 'default') }}:
48+
${{ else }}:
4949
vmImage: 'ubuntu-20.04'
5050

5151
container:
5252
image: sonicdev-microsoft.azurecr.io:443/${{ parameters.sonic_slave }}:latest
5353

5454
steps:
55+
- checkout: self
56+
clean: true
5557
- script: |
5658
sudo apt-get install -qq -y \
5759
libhiredis-dev \
@@ -63,8 +65,10 @@ jobs:
6365
displayName: "Install dependencies"
6466
- script: |
6567
set -ex
68+
rm ../*.deb || true
6669
./autogen.sh
67-
fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="--coverage -fprofile-abs-path" LDFLAGS="--coverage -fprofile-abs-path" binary && cp ../*.deb .
70+
fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="--coverage -fprofile-abs-path" LDFLAGS="--coverage -fprofile-abs-path" binary
71+
mv ../*.deb .
6872
displayName: "Compile sonic swss common with coverage enabled"
6973
- ${{ if eq(parameters.run_unit_test, true) }}:
7074
- script: |

‎.azure-pipelines/test-docker-sonic-vs-template.yml

+23-16
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
parameters:
22
- name: timeout
33
type: number
4-
default: 240
4+
default: 360
55

66
- name: log_artifact_name
77
type: string
@@ -11,33 +11,39 @@ jobs:
1111
displayName: vstest
1212
timeoutInMinutes: ${{ parameters.timeout }}
1313

14-
pool:
15-
vmImage: 'ubuntu-20.04'
14+
pool: sonic-common
1615

1716
steps:
17+
- checkout: self
18+
clean: true
19+
displayName: "Checkout sonic-swss-common"
20+
- checkout: sonic-swss
21+
clean: true
22+
displayName: "Checkout sonic-swss"
23+
- script: |
24+
set -ex
25+
cd sonic-swss
26+
git checkout $(BUILD_BRANCH)
27+
displayName: Set up sonic-swss branch
1828
- task: DownloadPipelineArtifact@2
1929
inputs:
2030
artifact: docker-sonic-vs
21-
displayName: "Download docker sonic vs image"
22-
31+
path: $(Build.ArtifactStagingDirectory)/download
32+
displayName: "Download pre-stage built docker-sonic-vs"
2333
- task: DownloadPipelineArtifact@2
2434
inputs:
2535
artifact: sonic-swss-common.amd64.ubuntu20_04
26-
displayName: "Download sonic swss common deb packages"
27-
28-
- checkout: self
29-
displayName: "Checkout sonic-swss-common"
30-
- checkout: sonic-swss
31-
displayName: "Checkout sonic-swss"
36+
path: $(Build.ArtifactStagingDirectory)/download
37+
displayName: "Download pre-stage built sonic-swss-common.amd64.ubuntu20_04"
3238

3339
- script: |
34-
set -x
40+
set -ex
3541
ls -l
3642
sudo sonic-swss-common/.azure-pipelines/build_and_install_module.sh
3743
3844
sudo apt-get install -y libhiredis0.14
39-
sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f
40-
sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb
45+
sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
46+
sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
4147
4248
# install packages for vs test
4349
sudo apt-get install -y net-tools bridge-utils vlan
@@ -46,12 +52,13 @@ jobs:
4652
displayName: "Install dependencies"
4753
4854
- script: |
49-
set -x
50-
sudo docker load -i ../docker-sonic-vs.gz
55+
set -ex
56+
sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz
5157
docker ps
5258
ip netns list
5359
pushd sonic-swss/tests
5460
sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
61+
rm -rf $(Build.ArtifactStagingDirectory)/download
5562
displayName: "Run vs tests"
5663
5764
- task: PublishTestResults@2

0 commit comments

Comments
 (0)