Skip to content

Commit 5b31291

Browse files
xumiaxumia
committed
[Build] Support j2 template for debian sources (sonic-net#12557)
Why I did it Unify the Debian mirror sources Make easy to upgrade to the next Debian release, not source url code change required. Support to customize the Debian mirror sources during the build Relative issue: sonic-net#12523
1 parent bb2cec5 commit 5b31291

20 files changed

+72
-191
lines changed

‎.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -77,3 +77,7 @@ files/build/tmp
7777
dockers/**/buildinfo
7878
platform/**/buildinfo
7979
sonic-slave*/**/buildinfo
80+
81+
# Debian mirror Sources
82+
sources.list.*
83+
!sources.list*.j2

‎Makefile.work

+11-3
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER_LC)
135135
# Generate the version control build info
136136
$(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
137137
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
138+
MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
138139
scripts/generate_buildinfo_config.sh)
139140

140141
# Generate the slave Dockerfile, and prepare build info for it
@@ -280,13 +281,20 @@ SONIC_BUILD_INSTRUCTION := make \
280281
SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
281282
SONIC_DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
282283
SLAVE_DIR=$(SLAVE_DIR) \
284+
MIRROR_URLS=$(MIRROR_URLS) \
285+
MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \
286+
MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
283287
$(SONIC_OVERRIDE_BUILD_VARS)
284288

285289
.PHONY: sonic-slave-build sonic-slave-bash init reset
286290

287291
.DEFAULT_GOAL := all
288292

289-
%::
293+
export MIRROR_URLS
294+
export MIRROR_SECURITY_URLS
295+
export SONIC_VERSION_CONTROL_COMPONENTS
296+
297+
%:: | sonic-build-hooks
290298
ifeq ($(MULTIARCH_QEMU_ENVIRON), y)
291299
@$(DOCKER_MULTIARCH_CHECK)
292300
ifneq ($(BLDENV), )
@@ -296,8 +304,6 @@ endif
296304
endif
297305
@$(OVERLAY_MODULE_CHECK)
298306

299-
@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
300-
@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
301307
@docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \
302308
{ [ $(ENABLE_DOCKER_BASE_PULL) == y ] && { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Pulling...; } && \
303309
$(DOCKER_BASE_PULL) && \
@@ -322,6 +328,8 @@ endif
322328
sonic-build-hooks:
323329
@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
324330
@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
331+
@[ "$(MULTIARCH_QEMU_ENVIRON)" == y ] && scripts/build_mirror_config.sh $(SLAVE_DIR) amd64 $(BLDENV)
332+
@scripts/build_mirror_config.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV)
325333

326334
sonic-slave-base-build : sonic-build-hooks
327335
@$(OVERLAY_MODULE_CHECK)

‎build_debian.sh

+1
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ sudo LANG=C chroot $FILESYSTEM_ROOT mount
107107
[ -d $TRUSTED_GPG_DIR ] && [ ! -z "$(ls $TRUSTED_GPG_DIR)" ] && sudo cp $TRUSTED_GPG_DIR/* ${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/
108108

109109
## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates
110+
scripts/build_mirror_config.sh files/apt $CONFIGURED_ARCH $IMAGE_DISTRO
110111
sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list
111112
sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
112113

‎dockers/docker-base-buster/Dockerfile.j2

+1-7
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive
2727

2828
# Configure data sources for apt/dpkg
2929
COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
30-
{% if CONFIGURED_ARCH == "armhf" %}
31-
COPY ["sources.list.armhf", "/etc/apt/sources.list"]
32-
{% elif CONFIGURED_ARCH == "arm64" %}
33-
COPY ["sources.list.arm64", "/etc/apt/sources.list"]
34-
{% else %}
35-
COPY ["sources.list", "/etc/apt/sources.list"]
36-
{% endif %}
30+
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
3731
COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
3832
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
3933

‎dockers/docker-base-buster/sources.list

-13
This file was deleted.

‎dockers/docker-base-buster/sources.list.arm64

-11
This file was deleted.

‎dockers/docker-base-buster/sources.list.armhf

-11
This file was deleted.

‎dockers/docker-base-stretch/Dockerfile.j2

+1-7
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive
2727

2828
# Configure data sources for apt/dpkg
2929
COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
30-
{% if CONFIGURED_ARCH == "armhf" %}
31-
COPY ["sources.list.armhf", "/etc/apt/sources.list"]
32-
{% elif CONFIGURED_ARCH == "arm64" %}
33-
COPY ["sources.list.arm64", "/etc/apt/sources.list"]
34-
{% else %}
35-
COPY ["sources.list", "/etc/apt/sources.list"]
36-
{% endif %}
30+
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
3731
COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
3832
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
3933

‎dockers/docker-base-stretch/sources.list

-11
This file was deleted.

‎dockers/docker-base-stretch/sources.list.arm64

-11
This file was deleted.

‎dockers/docker-base-stretch/sources.list.armhf

-11
This file was deleted.

‎files/apt/sources.list.amd64

-13
This file was deleted.

‎files/apt/sources.list.arm64

-13
This file was deleted.

‎files/apt/sources.list.armhf

-13
This file was deleted.

‎files/apt/sources.list.j2

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# The configuration is generated by template
2+
# Please add additional sources in /etc/apt/sources.list.d
3+
4+
{% for mirror_url in MIRROR_URLS.split(',') %}
5+
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free
6+
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free
7+
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free
8+
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free
9+
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-backports main contrib non-free
10+
{% endfor %}
11+
{% for mirror_url in MIRROR_SECURITY_URLS.split(',') %}
12+
{% set dist_separator='/' %}{% if 'packages.trafficmanager.net/debian' in mirror_url %}{% set dist_separator='_' %}{% endif %}
13+
{% if DISTRIBUTION == 'stretch' or DISTRIBUTION == 'buster' %}
14+
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free
15+
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free
16+
{% else %}
17+
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free
18+
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free
19+
{% endif %}
20+
{% endfor %}

‎scripts/build_mirror_config.sh

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
# Generate the sources.list.<arch> in the config path
4+
CONFIG_PATH=$1
5+
export ARCHITECTURE=$2
6+
export DISTRIBUTION=$3
7+
8+
# The default mirror urls
9+
DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/
10+
DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
11+
12+
# The debian-archive.trafficmanager.net does not support armhf, use debian.org instead
13+
if [ "$ARCHITECTURE" == "armhf" ]; then
14+
DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/
15+
DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
16+
fi
17+
18+
[ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS
19+
[ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS
20+
21+
TEMPLATE=files/apt/sources.list.j2
22+
[ -f files/apt/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=files/apt/sources.list.$ARCHITECTURE.j2
23+
[ -f $CONFIG_PATH/sources.list.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.j2
24+
[ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2
25+
26+
MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE

‎scripts/prepare_docker_buildinfo.sh

+4
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ if [ -z "$DISTRO" ]; then
2222
[ -z "$DISTRO" ] && DISTRO=jessie
2323
fi
2424

25+
if [[ "$IMAGENAME" == docker-base-* ]]; then
26+
scripts/build_mirror_config.sh ${DOCKERFILE_PATH} $ARCH $DISTRO
27+
fi
28+
2529
# add script for reproducible build. using sha256 instead of tag for docker base image.
2630
scripts/docker_version_control.sh $@
2731

‎slave.mk

+2
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,8 @@ export TRUSTED_GPG_URLS
125125
export SONIC_VERSION_CONTROL_COMPONENTS
126126
DEFAULT_CONTAINER_REGISTRY := $(SONIC_DEFAULT_CONTAINER_REGISTRY)
127127
export DEFAULT_CONTAINER_REGISTRY
128+
export MIRROR_URLS
129+
export MIRROR_SECURITY_URLS
128130

129131
ifeq ($(SONIC_ENABLE_PFCWD_ON_START),y)
130132
ENABLE_PFCWD_ON_START = y

‎sonic-slave-buster/Dockerfile.j2

+1-33
Original file line numberDiff line numberDiff line change
@@ -12,39 +12,7 @@ FROM {{ prefix }}debian:buster
1212
MAINTAINER [email protected]
1313

1414
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
15-
16-
RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \
17-
echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \
18-
echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \
19-
echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \
20-
echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list && \
21-
echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
22-
echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
23-
echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list
24-
25-
{%- if CONFIGURED_ARCH == "armhf" %}
26-
RUN echo "deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \
27-
echo "deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \
28-
echo "deb [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
29-
echo "deb-src [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
30-
echo "deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
31-
echo "deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
32-
echo 'deb [arch=armhf] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \
33-
echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
34-
echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
35-
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list
36-
{%- elif CONFIGURED_ARCH == "arm64" %}
37-
RUN echo "deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \
38-
echo "deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \
39-
echo "deb [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
40-
echo "deb-src [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
41-
echo "deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
42-
echo "deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
43-
echo 'deb [arch=arm64] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \
44-
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
45-
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
46-
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list
47-
{%- endif %}
15+
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
4816

4917
## Make apt-get non-interactive
5018
ENV DEBIAN_FRONTEND=noninteractive

0 commit comments

Comments
 (0)