Skip to content

Commit d6a6d25

Browse files
xumiaxumia
committed
[Build] Support j2 template for debian sources (sonic-net#12557)
Why I did it Unify the Debian mirror sources Make easy to upgrade to the next Debian release, not source url code change required. Support to customize the Debian mirror sources during the build Relative issue: sonic-net#12523
1 parent 7e6a25e commit d6a6d25

20 files changed

+72
-191
lines changed

‎.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -82,3 +82,7 @@ files/build/tmp
8282
dockers/**/buildinfo
8383
platform/**/buildinfo
8484
sonic-slave*/**/buildinfo
85+
86+
# Debian mirror Sources
87+
sources.list.*
88+
!sources.list*.j2

‎Makefile.work

+11-3
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER_LC)
134134
# Generate the version control build info
135135
$(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
136136
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
137+
MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
137138
scripts/generate_buildinfo_config.sh)
138139

139140
# Generate the slave Dockerfile, and prepare build info for it
@@ -282,13 +283,20 @@ SONIC_BUILD_INSTRUCTION := make \
282283
SONIC_DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
283284
ENABLE_HOST_SERVICE_ON_START=$(ENABLE_HOST_SERVICE_ON_START) \
284285
SLAVE_DIR=$(SLAVE_DIR) \
286+
MIRROR_URLS=$(MIRROR_URLS) \
287+
MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \
288+
MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
285289
$(SONIC_OVERRIDE_BUILD_VARS)
286290

287291
.PHONY: sonic-slave-build sonic-slave-bash init reset
288292

289293
.DEFAULT_GOAL := all
290294

291-
%::
295+
export MIRROR_URLS
296+
export MIRROR_SECURITY_URLS
297+
export SONIC_VERSION_CONTROL_COMPONENTS
298+
299+
%:: | sonic-build-hooks
292300
ifeq ($(MULTIARCH_QEMU_ENVIRON), y)
293301
@$(DOCKER_MULTIARCH_CHECK)
294302
ifneq ($(BLDENV), )
@@ -298,8 +306,6 @@ endif
298306
endif
299307
@$(OVERLAY_MODULE_CHECK)
300308

301-
@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
302-
@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
303309
@docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \
304310
{ [ $(ENABLE_DOCKER_BASE_PULL) == y ] && { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Pulling...; } && \
305311
$(DOCKER_BASE_PULL) && \
@@ -324,6 +330,8 @@ endif
324330
sonic-build-hooks:
325331
@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
326332
@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
333+
@[ "$(MULTIARCH_QEMU_ENVIRON)" == y ] && scripts/build_mirror_config.sh $(SLAVE_DIR) amd64 $(BLDENV)
334+
@scripts/build_mirror_config.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV)
327335

328336
sonic-slave-base-build : sonic-build-hooks
329337
@$(OVERLAY_MODULE_CHECK)

‎build_debian.sh

+1
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ sudo LANG=C chroot $FILESYSTEM_ROOT mount
104104
[ -d $TRUSTED_GPG_DIR ] && [ ! -z "$(ls $TRUSTED_GPG_DIR)" ] && sudo cp $TRUSTED_GPG_DIR/* ${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/
105105

106106
## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates
107+
scripts/build_mirror_config.sh files/apt $CONFIGURED_ARCH $IMAGE_DISTRO
107108
sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list
108109
sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
109110

‎dockers/docker-base-buster/Dockerfile.j2

+1-7
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive
2727

2828
# Configure data sources for apt/dpkg
2929
COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
30-
{% if CONFIGURED_ARCH == "armhf" %}
31-
COPY ["sources.list.armhf", "/etc/apt/sources.list"]
32-
{% elif CONFIGURED_ARCH == "arm64" %}
33-
COPY ["sources.list.arm64", "/etc/apt/sources.list"]
34-
{% else %}
35-
COPY ["sources.list", "/etc/apt/sources.list"]
36-
{% endif %}
30+
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
3731
COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
3832
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
3933

‎dockers/docker-base-buster/sources.list

-13
This file was deleted.

‎dockers/docker-base-buster/sources.list.arm64

-11
This file was deleted.

‎dockers/docker-base-buster/sources.list.armhf

-11
This file was deleted.

‎dockers/docker-base-stretch/Dockerfile.j2

+1-7
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive
2727

2828
# Configure data sources for apt/dpkg
2929
COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
30-
{% if CONFIGURED_ARCH == "armhf" %}
31-
COPY ["sources.list.armhf", "/etc/apt/sources.list"]
32-
{% elif CONFIGURED_ARCH == "arm64" %}
33-
COPY ["sources.list.arm64", "/etc/apt/sources.list"]
34-
{% else %}
35-
COPY ["sources.list", "/etc/apt/sources.list"]
36-
{% endif %}
30+
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
3731
COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
3832
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
3933

‎dockers/docker-base-stretch/sources.list

-11
This file was deleted.

‎dockers/docker-base-stretch/sources.list.arm64

-11
This file was deleted.

‎dockers/docker-base-stretch/sources.list.armhf

-11
This file was deleted.

‎files/apt/sources.list.amd64

-13
This file was deleted.

‎files/apt/sources.list.arm64

-13
This file was deleted.

‎files/apt/sources.list.armhf

-13
This file was deleted.

‎files/apt/sources.list.j2

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# The configuration is generated by template
2+
# Please add additional sources in /etc/apt/sources.list.d
3+
4+
{% for mirror_url in MIRROR_URLS.split(',') %}
5+
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free
6+
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free
7+
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free
8+
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free
9+
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-backports main contrib non-free
10+
{% endfor %}
11+
{% for mirror_url in MIRROR_SECURITY_URLS.split(',') %}
12+
{% set dist_separator='/' %}{% if 'packages.trafficmanager.net/debian' in mirror_url %}{% set dist_separator='_' %}{% endif %}
13+
{% if DISTRIBUTION == 'stretch' or DISTRIBUTION == 'buster' %}
14+
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free
15+
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free
16+
{% else %}
17+
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free
18+
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free
19+
{% endif %}
20+
{% endfor %}

‎scripts/build_mirror_config.sh

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
# Generate the sources.list.<arch> in the config path
4+
CONFIG_PATH=$1
5+
export ARCHITECTURE=$2
6+
export DISTRIBUTION=$3
7+
8+
# The default mirror urls
9+
DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/
10+
DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
11+
12+
# The debian-archive.trafficmanager.net does not support armhf, use debian.org instead
13+
if [ "$ARCHITECTURE" == "armhf" ]; then
14+
DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/
15+
DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
16+
fi
17+
18+
[ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS
19+
[ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS
20+
21+
TEMPLATE=files/apt/sources.list.j2
22+
[ -f files/apt/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=files/apt/sources.list.$ARCHITECTURE.j2
23+
[ -f $CONFIG_PATH/sources.list.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.j2
24+
[ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2
25+
26+
MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE

‎scripts/prepare_docker_buildinfo.sh

+4
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ if [ -z "$DISTRO" ]; then
2222
[ -z "$DISTRO" ] && DISTRO=jessie
2323
fi
2424

25+
if [[ "$IMAGENAME" == docker-base-* ]]; then
26+
scripts/build_mirror_config.sh ${DOCKERFILE_PATH} $ARCH $DISTRO
27+
fi
28+
2529
# add script for reproducible build. using sha256 instead of tag for docker base image.
2630
scripts/docker_version_control.sh $@
2731

‎slave.mk

+2
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,8 @@ export TRUSTED_GPG_URLS
121121
export SONIC_VERSION_CONTROL_COMPONENTS
122122
DEFAULT_CONTAINER_REGISTRY := $(SONIC_DEFAULT_CONTAINER_REGISTRY)
123123
export DEFAULT_CONTAINER_REGISTRY
124+
export MIRROR_URLS
125+
export MIRROR_SECURITY_URLS
124126

125127
ifeq ($(SONIC_ENABLE_PFCWD_ON_START),y)
126128
ENABLE_PFCWD_ON_START = y

‎sonic-slave-buster/Dockerfile.j2

+1-33
Original file line numberDiff line numberDiff line change
@@ -12,39 +12,7 @@ FROM {{ prefix }}debian:buster
1212
MAINTAINER [email protected]
1313

1414
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
15-
16-
RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \
17-
echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \
18-
echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \
19-
echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \
20-
echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list && \
21-
echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
22-
echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
23-
echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list
24-
25-
{%- if CONFIGURED_ARCH == "armhf" %}
26-
RUN echo "deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \
27-
echo "deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \
28-
echo "deb [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
29-
echo "deb-src [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
30-
echo "deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
31-
echo "deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
32-
echo 'deb [arch=armhf] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \
33-
echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
34-
echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
35-
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list
36-
{%- elif CONFIGURED_ARCH == "arm64" %}
37-
RUN echo "deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \
38-
echo "deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \
39-
echo "deb [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
40-
echo "deb-src [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
41-
echo "deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
42-
echo "deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
43-
echo 'deb [arch=arm64] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \
44-
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
45-
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
46-
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list
47-
{%- endif %}
15+
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
4816

4917
## Make apt-get non-interactive
5018
ENV DEBIAN_FRONTEND=noninteractive

0 commit comments

Comments
 (0)