Skip to content

Commit 9b057a1

Browse files
authored
Merge branch 'master' into sonic_docker_image_ctl_update
2 parents af82081 + fffd6e6 commit 9b057a1

File tree

1,735 files changed

+108979
-200462
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,735 files changed

+108979
-200462
lines changed

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

+4-3
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,14 @@ parameters:
3333
type: object
3434
default:
3535
- vs
36-
- barefoot
3736
- broadcom
3837
- centec
3938
- centec-arm64
4039
- generic
4140
- marvell-armhf
41+
- marvell-arm64
4242
- mellanox
43+
- pensando
4344

4445
stages:
4546
- stage: Prepare
@@ -161,8 +162,8 @@ stages:
161162
git branch -u remote/$BRANCH_NAME
162163
163164
echo $GIT_PASSWORD | gh auth login --with-token
164-
TITLE="Upgrade SONiC Versions"
165-
BODY="Upgrade SONiC Versions"
165+
TITLE="[${SOURCE_BRANCH#refs/heads/}] Upgrade SONiC package Versions"
166+
BODY=$TITLE
166167
RET=0
167168
if ! gh pr create -t "$TITLE" -b "$BODY" -B $(Build.SourceBranch) -R $(Build.Repository.Name) > pr.log 2>&1; then
168169
if ! grep -q "already exists" pr.log; then

.azure-pipelines/azure-pipelines-build-ubuntu-2004.yml

+14-2
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,16 @@ trigger:
99
branches:
1010
include:
1111
- master
12+
- 202???
13+
14+
schedules:
15+
- cron: "0 0 * * *"
16+
displayName: Daily build
17+
branches:
18+
include:
19+
- master
20+
- 202???
21+
always: true
1222

1323
stages:
1424
- stage: Build
@@ -28,13 +38,15 @@ stages:
2838
cmake pkg-config python3-pip python cmake libgtest-dev libgmock-dev libyang-dev \
2939
debhelper-compat dh-elpa dh-sequence-python3 python3-all \
3040
libpython3-all-dev python3-six xmlto unzip rake-compiler gem2deb pkg-php-tools \
31-
ant default-jdk maven-repo-helper libguava-java
41+
ant default-jdk maven-repo-helper libguava-java \
42+
libboost-all-dev libgtest-dev build-essential swig4.0 swig
43+
sudo pip3 install pytest click
3244
wget http://ftp.us.debian.org/debian/pool/main/libg/libgoogle-gson-java/libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
3345
sudo dpkg -i libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
3446
mkdir -p /tmp/artifacts
3547
displayName: "Install dependencies"
3648
- script: |
37-
SONIC_CONFIG_MAKE_JOBS=$(nproc) CONFIGURED_ARCH=amd64 DEST=/tmp/artifacts make -f ../rules/protobuf.mk -f protobuf/Makefile
49+
BLDENV=bullseye SONIC_CONFIG_MAKE_JOBS=$(nproc) CONFIGURED_ARCH=amd64 DEST=/tmp/artifacts make -f ../rules/protobuf.mk -f protobuf/Makefile
3850
workingDirectory: src
3951
displayName: "Build protobuf"
4052
- script: |

.azure-pipelines/azure-pipelines-build.yml

+9-7
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ jobs:
3535
dbg_image: no
3636
asan_image: no
3737
swi_image: no
38-
raw_image: no
3938
docker_syncd_rpc_image: no
4039
syncd_rpc_image: no
4140
platform_rpc: no
@@ -62,7 +61,6 @@ jobs:
6261
variables:
6362
dbg_image: yes
6463
swi_image: yes
65-
raw_image: yes
6664
docker_syncd_rpc_image: yes
6765
platform_rpc: brcm
6866

@@ -116,12 +114,19 @@ jobs:
116114
docker_syncd_rpc_image: yes
117115
platform_rpc: nephos
118116

117+
- name: pensando
118+
pool: sonicbld-arm64
119+
variables:
120+
PLATFORM_ARCH: arm64
121+
119122
buildSteps:
120123
- template: .azure-pipelines/template-skipvstest.yml@buildimage
121124
- template: .azure-pipelines/template-daemon.yml@buildimage
122125
- bash: |
123126
set -ex
124-
if [ $(GROUP_NAME) == vs ]; then
127+
if [ $(GROUP_NAME) == pensando ]; then
128+
make $BUILD_OPTIONS target/sonic-pensando.tar
129+
elif [ $(GROUP_NAME) == vs ]; then
125130
if [ $(dbg_image) == yes ]; then
126131
make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz
127132
mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
@@ -130,7 +135,7 @@ jobs:
130135
make $BUILD_OPTIONS ENABLE_ASAN=y target/docker-sonic-vs.gz
131136
mv target/docker-sonic-vs.gz target/docker-sonic-vs-asan.gz
132137
fi
133-
if [ $(K8S_OPTIONS) == 'INCLUDE_KUBERNETES_MASTER=y' ]; then
138+
if [ "$(K8S_OPTIONS)" == 'INCLUDE_KUBERNETES_MASTER=y' ]; then
134139
make $BUILD_OPTIONS $(K8S_OPTIONS) target/sonic-vs.img.gz
135140
mv target/sonic-vs.img.gz target/sonic-vs-k8s.img.gz
136141
fi
@@ -149,9 +154,6 @@ jobs:
149154
if [ $(swi_image) == yes ]; then
150155
make $BUILD_OPTIONS ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-$(GROUP_NAME).swi
151156
fi
152-
if [ $(raw_image) == yes ]; then
153-
make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
154-
fi
155157
if [ $(docker_syncd_rpc_image) == yes ]; then
156158
# workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache
157159
make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz

.azure-pipelines/azure-pipelines-image-template.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030
- script: |
3131
[ -n "$OVERRIDE_BUILD_OPTIONS" ] && OVERRIDE_BUILD_OPTIONS=$(OVERRIDE_BUILD_OPTIONS)
3232
BUILD_OPTIONS="$(BUILD_OPTIONS) $OVERRIDE_BUILD_OPTIONS"
33-
if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf)$"; then
33+
if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf|marvell-arm64)$"; then
3434
CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM_AZP)"
3535
BUILD_OPTIONS="$BUILD_OPTIONS $CACHE_OPTIONS"
3636
fi

.azure-pipelines/build-template.yml

-3
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,6 @@ jobs:
101101
if [ ${{ parameters.swi_image }} == true ]; then
102102
make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-${{ parameters.platform }}.swi
103103
fi
104-
if [ ${{ parameters.raw_image }} == true ]; then
105-
make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.raw
106-
fi
107104
if [ ${{ parameters.sync_rpc_image }} == true ]; then
108105
make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz
109106
# workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Starter pipeline
2+
# Start with a minimal pipeline that you can customize to build and deploy your code.
3+
# Add steps that build, run tests, deploy, and more:
4+
# https://aka.ms/yaml
5+
# Build and push sonic-mgmt image
6+
7+
schedules:
8+
- cron: "0 8 * * *"
9+
branches:
10+
include:
11+
- master
12+
always: true
13+
14+
trigger: none
15+
pr:
16+
branches:
17+
include:
18+
- master
19+
paths:
20+
include:
21+
- dockers/docker-sonic-mgmt
22+
23+
parameters:
24+
- name: registry_url
25+
type: string
26+
default: sonicdev-microsoft.azurecr.io
27+
- name: registry_conn
28+
type: string
29+
default: sonicdev
30+
31+
stages:
32+
- stage: Build
33+
jobs:
34+
- job: Build
35+
pool: sonicbld
36+
timeoutInMinutes: 360
37+
steps:
38+
- template: cleanup.yml
39+
- checkout: self
40+
clean: true
41+
- bash: |
42+
set -xe
43+
git submodule update --init --recursive -- src/sonic-platform-daemons src/sonic-genl-packet src/sonic-sairedis src/ptf src/sonic-device-data
44+
45+
make SONIC_BUILD_JOBS=$(nproc) DEFAULT_CONTAINER_REGISTRY=publicmirror.azurecr.io ENABLE_DOCKER_BASE_PULL=y configure PLATFORM=generic
46+
make -f Makefile.work BLDENV=bullseye SONIC_BUILD_JOBS=$(nproc) DEFAULT_CONTAINER_REGISTRY=publicmirror.azurecr.io ENABLE_DOCKER_BASE_PULL=y LEGACY_SONIC_MGMT_DOCKER=n target/docker-sonic-mgmt.gz
47+
cp target -r $(Build.ArtifactStagingDirectory)/target
48+
docker load -i target/docker-sonic-mgmt.gz
49+
docker tag docker-sonic-mgmt $REGISTRY_SERVER/docker-sonic-mgmt:py3only
50+
env:
51+
REGISTRY_SERVER: ${{ parameters.registry_url }}
52+
displayName: Build docker-sonic-mgmt.gz
53+
- task: Docker@2
54+
displayName: Upload image
55+
inputs:
56+
containerRegistry: ${{ parameters.registry_conn }}
57+
repository: docker-sonic-mgmt
58+
command: push
59+
tags: py3only
60+
- publish: $(Build.ArtifactStagingDirectory)
61+
artifact: 'docker-sonic-mgmt'
62+
displayName: "Archive docker image sonic-mgmt"
63+

.azure-pipelines/docker-sonic-mgmt.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ stages:
4343
git submodule update --init --recursive -- src/sonic-platform-daemons src/sonic-genl-packet src/sonic-sairedis src/ptf src/sonic-device-data
4444
4545
make SONIC_BUILD_JOBS=$(nproc) DEFAULT_CONTAINER_REGISTRY=publicmirror.azurecr.io ENABLE_DOCKER_BASE_PULL=y configure PLATFORM=generic
46-
make SONIC_BUILD_JOBS=$(nproc) DEFAULT_CONTAINER_REGISTRY=publicmirror.azurecr.io ENABLE_DOCKER_BASE_PULL=y target/docker-sonic-mgmt.gz
46+
make -f Makefile.work BLDENV=bullseye SONIC_BUILD_JOBS=$(nproc) DEFAULT_CONTAINER_REGISTRY=publicmirror.azurecr.io ENABLE_DOCKER_BASE_PULL=y LEGACY_SONIC_MGMT_DOCKER=y target/docker-sonic-mgmt.gz
4747
cp target -r $(Build.ArtifactStagingDirectory)/target
4848
docker load -i target/docker-sonic-mgmt.gz
4949
docker tag docker-sonic-mgmt $REGISTRY_SERVER/docker-sonic-mgmt:latest

.azure-pipelines/docker-sonic-slave-template.yml

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ parameters:
1616
- name: dist
1717
type: string
1818
values:
19+
- bookworm
1920
- bullseye
2021
- buster
2122
- stretch

.azure-pipelines/docker-sonic-slave.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ resources:
88
- repository: buildimage
99
type: github
1010
name: sonic-net/sonic-buildimage
11-
ref: master
11+
ref: bookworm
1212
endpoint: sonic-net
1313

1414
schedules:
@@ -44,6 +44,7 @@ parameters:
4444
- name: 'dists'
4545
type: object
4646
default:
47+
- bookworm
4748
- bullseye
4849
- buster
4950
- stretch

.azure-pipelines/dpkg-cache-cleanup.yml

+24-3
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,34 @@ trigger: none
1515
pr: none
1616

1717
jobs:
18-
- job: Build
18+
- job: Amd
1919
pool: sonicbld
2020
timeoutInMinutes: 5
2121
steps:
2222
- checkout: none
2323
- script: |
24-
set -xe
24+
df -h
2525
sudo find /nfs/dpkg_cache/ -name *.tgz -mtime +30 -type f -delete
26+
df -h
2627
displayName: clean dpkg cache
27-
28+
- job: Armhf
29+
pool: sonicbld-armhf
30+
timeoutInMinutes: 5
31+
steps:
32+
- checkout: none
33+
- script: |
34+
df -h
35+
sudo find /nfs/dpkg_cache/ -name *.tgz -mtime +30 -type f -delete
36+
df -h
37+
displayName: clean dpkg cache
38+
- job: Arm64
39+
pool: sonicbld-arm64
40+
timeoutInMinutes: 5
41+
steps:
42+
- checkout: none
43+
- script: |
44+
df -h
45+
sudo find /nfs/dpkg_cache/ -name *.tgz -mtime +30 -type f -delete
46+
df -h
47+
displayName: clean dpkg cache
48+

.azure-pipelines/official-build-cache.yml

-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ stages:
3939
- name: broadcom
4040
variables:
4141
swi_image: yes
42-
raw_image: yes
4342
docker_syncd_rpc_image: yes
4443
platform_rpc: brcm
4544
- name: mellanox

.github/workflows/automerge.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ on:
99

1010
jobs:
1111
automerge:
12-
if: github.event.check_suite.app.name == 'Azure Pipelines' && github.event.check_suite.conclusion == 'success'
12+
# Disable workflow. Use automerge_scan instead to have the same behaviour.
13+
if: github.event.check_suite.app.name == 'Azure Pipelines' && github.event.check_suite.conclusion == 'success' && false
1314
runs-on: ubuntu-latest
1415
steps:
1516
- name: automerge

.github/workflows/automerge_scan.yml

+43-26
Original file line numberDiff line numberDiff line change
@@ -16,58 +16,74 @@ jobs:
1616
set -e
1717
1818
echo ${TOKEN} | gh auth login --with-token
19-
gh pr list -R sonic-net/sonic-buildimage -A mssonicbld --json additions,assignees,author,baseRefName,body,changedFiles,closed,closedAt,comments,commits,createdAt,deletions,files,headRefName,headRepository,headRepositoryOwner,id,isCrossRepository,isDraft,labels,latestReviews,maintainerCanModify,mergeCommit,mergeStateStatus,mergeable,mergedAt,mergedBy,milestone,number,potentialMergeCommit,projectCards,reactionGroups,reviewDecision,reviewRequests,reviews,state,statusCheckRollup,title,updatedAt,url > prs.log
19+
gh pr list -R sonic-net/sonic-buildimage -A mssonicbld -L 100 -S "label:automerge" --json url,body,title,createdAt,labels,statusCheckRollup > prs.log
2020
cat prs.log | jq
2121
- name: Main
2222
run: |
2323
set -e
2424
25+
# PR merge run per 2 hours
26+
# Other operation run per day.
27+
# Cherry pick PR:
28+
# more than 3 days, comment @author to check
29+
# more than 10 days, stop comment.
30+
# more than 28 days, comment @author PR will be closed
31+
# more than 30 days, close PR
32+
date_3d_ago=$(date --date "3 day ago" -u +"%FT%TZ")
33+
date_10d_ago=$(date --date "10 day ago" -u +"%FT%TZ")
34+
date_28d_ago=$(date --date "28 day ago" -u +"%FT%TZ")
35+
date_30d_ago=$(date --date "30 day ago" -u +"%FT%TZ")
36+
date_now=$(date -u +"%T")
37+
operate=false
38+
[[ "$date_now" < "02:00:00" ]] && operate=true
39+
2540
count=$(cat prs.log | jq 'length')
2641
for ((i=0;i<$count;i++))
2742
do
2843
url=$(cat prs.log | jq -r ".[$i].url")
2944
body=$(cat prs.log | jq -r ".[$i].body")
3045
title=$(cat prs.log | jq -r ".[$i].title")
46+
origin_pr_id=$(echo $title | grep -Eo "\[action\] \[PR:[0-9]*\]" | grep -Eo [0-9]* || true)
3147
created_at=$(cat prs.log | jq -r ".[$i].createdAt")
32-
echo PR: $(($i+1))/$count, URL: $url, createdAt: $created_at, now: $(date -u +"%FT%TZ")
48+
echo PR: $(($i+1))/$count, URL: $url, origin PR: $origin_pr_id, createdAt: $created_at, operate: $operate
3349
[[ "$url" == "" ]] && continue
3450
[[ $created_at > $(date --date "1 hour ago" -u +"%FT%TZ") ]] && continue
35-
# only check automerge PR.
36-
cat prs.log | jq -r ".[$i].labels[].name" | grep automerge || continue
3751
3852
checks=$(cat prs.log | jq ".[$i].statusCheckRollup")
3953
checks_count=$(echo $checks | jq 'length')
40-
echo Checks count: $checks_count
54+
pr_success=true
4155
for ((j=0;j<$checks_count;j++))
4256
do
4357
check=$(echo $checks | jq ".[$j]")
44-
state=$(echo $check | jq -r '.state')
58+
status=$(echo $check | jq -r '.status')
4559
conclusion=$(echo $check | jq -r '.conclusion')
4660
name=$(echo $check | jq -r '.name')
4761
4862
# EasyCLA success flag: state=SUCCESS
4963
# Others success flag: conclusion in SUCCESS,NEUTRAL
50-
# Ignore Azure.sonic-buildimage stage check result. It may be set continueOnError
51-
echo "$name" | grep "Azure.sonic-buildimage (" && continue
52-
# rerun Azure.sonic-buildimage per day
53-
if [[ "$name" == "Azure.sonic-buildimage" ]] && [[ "$conclusion" == "FAILURE" ]];then
54-
completedAt=$(echo $check | jq -r '.completedAt')
55-
[[ "$completedAt" < $(date --date "2 hour ago" -u +"%FT%TZ") ]] && [[ $(date -u +"%T") < "02:00:00" ]] && gh pr comment $url --body "/azp run Azure.sonic-buildimage"
56-
fi
57-
# Ignore Semgrep, it has issues.
58-
[[ "$name" == "Semgrep" ]] && continue
59-
if [[ "$state" == "SUCCESS" ]];then
60-
# check pass
61-
continue
62-
elif [[ "$conclusion" == "SUCCESS" ]] || [[ "$conclusion" == "NEUTRAL" ]];then
63-
# check pass
64-
continue
65-
else
66-
echo "$url Check failed!!!"
67-
echo $check | jq
68-
continue 2
69-
fi
64+
# only check Azure.sonic-buildimage currently
65+
echo "$name" | grep -v "Azure.sonic-buildimage" > /dev/null && continue
66+
[[ "$status" != "COMPLETED" ]] && echo "$name: $status" && continue 2
67+
68+
success=true
69+
( [[ "$conclusion" == "FAILURE" ]] || [[ "$conclusion" == "CANCELLED" ]] ) && success=false && pr_success=false
70+
! $success && echo "FAIL: $name"
7071
done
72+
73+
# rerun Azure.sonic-buildimage per day
74+
! $pr_success && $operate && gh pr comment $url --body "/azp run Azure.sonic-buildimage"
75+
76+
# If auto cherry pick PRs failed, comment in original PR and close cherry pick PR
77+
if [ -n "$origin_pr_id" ] && [[ $created_at < $date_3d_ago ]] && ! $pr_success;then
78+
origin_pr_url=https://github.com/sonic-net/sonic-buildimage/pull/$origin_pr_id
79+
author=$(gh pr view $origin_pr_url --json author | jq .author.login -r)
80+
echo "Original author will check."
81+
$operate && [[ $created_at > $date_10d_ago ]] && gh pr comment $origin_pr_url --body "@$author cherry pick PR didn't pass PR checker. Please check!!!<br>$url"
82+
$operate && [[ $created_at < $date_28d_ago ]] && gh pr comment $origin_pr_url --body "@$author cherry pick PR didn't pass PR checker. Please check!!! Auto cherry pick PR will be closed in 2 days.<br>$url"
83+
$operate && [[ $created_at < $date_30d_ago ]] && echo "$url Closed" && gh pr close $url
84+
fi
85+
86+
! $pr_success && continue
7187
# merge the PR
7288
echo ========Merging PR========
7389
if echo $title | grep "^\[submodule\]";then
@@ -77,3 +93,4 @@ jobs:
7793
fi
7894
echo ========Finished PR========
7995
done
96+

0 commit comments

Comments
 (0)