Skip to content

Commit 6cb8893

Browse files
Yakiv-HurykYakiv-Huryk
authored
[build] add support for 2 stage rootfs build (#15924)
This adds optimization for the SONiC image build by splitting the final build step into two stages. It allows running the first stage in parallel, improving build time. The optimization is enabled via new rules/config flag ENABLE_RFS_SPLIT_BUILD (disabled by default) - Why I did it To improve a build time. - How I did it Added a logic to run build_debian.sh in two stages, transferring the progress via a new build artifact. - How to verify it make ENABLE_RFS_SPLIT_BUILD=y SONIC_BUILD_JOBS=32 target/<IMAGE_NAME>.bin Signed-off-by: Yakiv Huryk <[email protected]>
1 parent 7059f42 commit 6cb8893

File tree

4 files changed

+177
-23
lines changed

4 files changed

+177
-23
lines changed

‎Makefile.cache

+25-1
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,21 @@ define SAVE_CACHE
357357
$(if $(call CHECK_WCACHE_ENABLED,$(1)), $(call SAVE_INTO_CACHE,$(1),$(2)))
358358
endef
359359

360+
RFS_DEP_FILES := $(wildcard \
361+
$(addprefix scripts/, build_debian_base_system.sh prepare_debian_image_buildinfo.sh build_mirror_config.sh) \
362+
$(shell git ls-files files/initramfs-tools) \
363+
$(shell git ls-files files/image_config) \
364+
$(shell git ls-files files/apparmor) \
365+
$(shell git ls-files files/apt) \
366+
$(shell git ls-files files/sshd) \
367+
$(shell git ls-files files/dhcp) \
368+
src/sonic-build-hooks/buildinfo/trusted.gpg.d \
369+
platform/$(CONFIGURED_PLATFORM)/modules \
370+
files/docker/docker.service.conf \
371+
files/build_templates/default_users.json.j2 \
372+
files/build_scripts/generate_asic_config_checksum.py \
373+
files/scripts/core_cleanup.py \
374+
build_debian.sh onie-image.conf)
360375

361376

362377
# Set the target path for each target.
@@ -384,11 +399,17 @@ $(foreach pkg, $(SONIC_INSTALL_PKGS), \
384399
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(FSROOT_PATH))) \
385400
$(eval $(FSROOT_PATH)/$(pkg)_TARGET := $(pkg)) )
386401

402+
$(foreach pkg, $(SONIC_RFS_TARGETS), \
403+
$(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(TARGET_PATH))) \
404+
$(eval $(pkg)_CACHE_MODE := GIT_CONTENT_SHA) \
405+
$(eval $(pkg)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)) \
406+
$(eval $(pkg)_DEP_FILES := $(SONIC_COMMON_BASE_FILES_LIST) $(RFS_DEP_FILES)) \
407+
$(eval $(TARGET_PATH)/$(pkg)_TARGET := $(pkg)) )
387408

388409
# define the DEP files(.dep and .smdep) and SHA files (.sha and smsha) for each target
389410
$(foreach pkg, $(SONIC_MAKE_DEBS) $(SONIC_DPKG_DEBS) $(SONIC_ONLINE_DEBS) $(SONIC_COPY_DEBS) \
390411
$(SONIC_MAKE_FILES) $(SONIC_PYTHON_STDEB_DEBS) $(SONIC_PYTHON_WHEELS) \
391-
$(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES) $(SONIC_INSTALL_PKGS), \
412+
$(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES) $(SONIC_INSTALL_PKGS) $(SONIC_RFS_TARGETS), \
392413
$(eval $(pkg)_MOD_SRC_PATH:=$(if $($(pkg)_SRC_PATH),$($(pkg)_SRC_PATH),$($(pkg)_PATH))) \
393414
$(eval $(pkg)_BASE_PATH:=$(if $($(pkg)_BASE_PATH),$($(pkg)_BASE_PATH),$(CURDIR))) \
394415
$(eval $(pkg)_DEP_FLAGS_FILE:=$($(pkg)_DST_PATH)/$(pkg).flags) \
@@ -489,6 +510,7 @@ $(eval $(call FLAGS_DEP_RULES, $(SONIC_PYTHON_STDEB_DEBS), $(PYTHON_DEBS_PATH),f
489510
$(eval $(call FLAGS_DEP_RULES, $(SONIC_PYTHON_WHEELS), $(PYTHON_WHEELS_PATH),flags))
490511
$(eval $(call FLAGS_DEP_RULES, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), $(TARGET_PATH),flags))
491512
$(eval $(call FLAGS_DEP_RULES, $(SONIC_INSTALL_PKGS), $(FSROOT_PATH),flags))
513+
$(eval $(call FLAGS_DEP_RULES, $(SONIC_RFS_TARGETS), $(TARGET_PATH),flags))
492514

493515

494516

@@ -585,6 +607,7 @@ $(eval $(call SHA_DEP_RULES, $(SONIC_PYTHON_STDEB_DEBS), $(PYTHON_DEBS_PATH),dep
585607
$(eval $(call SHA_DEP_RULES, $(SONIC_PYTHON_WHEELS), $(PYTHON_WHEELS_PATH),dep))
586608
$(eval $(call SHA_DEP_RULES, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), $(TARGET_PATH),dep))
587609
$(eval $(call SHA_DEP_RULES, $(SONIC_INSTALL_PKGS), $(FSROOT_PATH),dep))
610+
$(eval $(call SHA_DEP_RULES, $(SONIC_RFS_TARGETS), $(TARGET_PATH),dep))
588611

589612

590613

@@ -618,6 +641,7 @@ SONIC_CACHE_CLEAN_TARGETS = $(addsuffix -clean,$(addprefix $(TARGET_PATH)/, \
618641
$(SONIC_DOCKER_IMAGES) \
619642
$(SONIC_DOCKER_DBG_IMAGES) \
620643
$(SONIC_SIMPLE_DOCKER_IMAGES) \
644+
$(SONIC_RFS_TARGETS) \
621645
$(SONIC_INSTALLERS)))
622646
$(SONIC_CACHE_CLEAN_TARGETS) :: $(TARGET_PATH)/%-clean : .platform
623647
@rm -f $($*_DEP_FLAGS_FILE) $($*_MOD_HASH_FILE) $($*_SMOD_HASH_FILE) \

‎build_debian.sh

+58-19
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ TRUSTED_GPG_DIR=$BUILD_TOOL_PATH/trusted.gpg.d
5959
exit 1
6060
}
6161

62+
## Check if not a last stage of RFS build
63+
if [[ $RFS_SPLIT_LAST_STAGE != y ]]; then
64+
6265
## Prepare the file system directory
6366
if [[ -d $FILESYSTEM_ROOT ]]; then
6467
sudo rm -rf $FILESYSTEM_ROOT || die "Failed to clean chroot directory"
@@ -71,11 +74,6 @@ touch $FILESYSTEM_ROOT/$PLATFORM_DIR/firsttime
7174
## ensure proc is mounted
7275
sudo mount proc /proc -t proc || true
7376

74-
## make / as a mountpoint in chroot env, needed by dockerd
75-
pushd $FILESYSTEM_ROOT
76-
sudo mount --bind . .
77-
popd
78-
7977
## Build the host debian base system
8078
echo '[INFO] Build host debian base system...'
8179
TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT
@@ -576,24 +574,11 @@ if [ -f files/image_config/ntp/ntp-systemd-wrapper ]; then
576574
sudo cp ./files/image_config/ntp/ntp-systemd-wrapper $FILESYSTEM_ROOT/usr/lib/ntp/
577575
fi
578576

579-
## Version file
577+
## Version file part 1
580578
sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic
581579
if [ -f files/image_config/sonic_release ]; then
582580
sudo cp files/image_config/sonic_release $FILESYSTEM_ROOT/etc/sonic/
583581
fi
584-
export build_version="${SONIC_IMAGE_VERSION}"
585-
export debian_version="$(cat $FILESYSTEM_ROOT/etc/debian_version)"
586-
export kernel_version="${kversion}"
587-
export asic_type="${sonic_asic_platform}"
588-
export asic_subtype="${TARGET_MACHINE}"
589-
export commit_id="$(git rev-parse --short HEAD)"
590-
export branch="$(git rev-parse --abbrev-ref HEAD)"
591-
export release="$(if [ -f $FILESYSTEM_ROOT/etc/sonic/sonic_release ]; then cat $FILESYSTEM_ROOT/etc/sonic/sonic_release; fi)"
592-
export build_date="$(date -u)"
593-
export build_number="${BUILD_NUMBER:-0}"
594-
export built_by="$USER@$BUILD_HOSTNAME"
595-
export sonic_os_version="${SONIC_OS_VERSION}"
596-
j2 files/build_templates/sonic_version.yml.j2 | sudo tee $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml
597582

598583
# Default users info
599584
export password_expire="$( [[ "$CHANGE_DEFAULT_PASSWORD" == "y" ]] && echo true || echo false )"
@@ -615,6 +600,60 @@ if [[ ! -f './asic_config_checksum' ]]; then
615600
fi
616601
sudo cp ./asic_config_checksum $FILESYSTEM_ROOT/etc/sonic/asic_config_checksum
617602

603+
## Check if not a last stage of RFS build
604+
fi
605+
606+
if [[ $RFS_SPLIT_FIRST_STAGE == y ]]; then
607+
echo '[INFO] Finished with RFS first stage'
608+
echo '[INFO] Umount all'
609+
610+
## Display all process details access /proc
611+
sudo LANG=C chroot $FILESYSTEM_ROOT fuser -vm /proc
612+
## Kill the processes
613+
sudo LANG=C chroot $FILESYSTEM_ROOT fuser -km /proc || true
614+
## Wait fuser fully kill the processes
615+
sudo timeout 15s bash -c 'until LANG=C chroot $0 umount /proc; do sleep 1; done' $FILESYSTEM_ROOT || true
616+
617+
sudo rm -f $TARGET_PATH/$RFS_SQUASHFS_NAME
618+
sudo mksquashfs $FILESYSTEM_ROOT $TARGET_PATH/$RFS_SQUASHFS_NAME -Xcompression-level 1
619+
620+
exit 0
621+
fi
622+
623+
if [[ $RFS_SPLIT_LAST_STAGE == y ]]; then
624+
echo '[INFO] RFS build: second stage'
625+
626+
## ensure proc is mounted
627+
sudo mount proc /proc -t proc || true
628+
629+
sudo fuser -vm $FILESYSTEM_ROOT || true
630+
sudo rm -rf $FILESYSTEM_ROOT
631+
sudo unsquashfs -d $FILESYSTEM_ROOT $TARGET_PATH/$RFS_SQUASHFS_NAME
632+
633+
## make / as a mountpoint in chroot env, needed by dockerd
634+
pushd $FILESYSTEM_ROOT
635+
sudo mount --bind . .
636+
popd
637+
638+
trap_push 'sudo LANG=C chroot $FILESYSTEM_ROOT umount /proc || true'
639+
sudo LANG=C chroot $FILESYSTEM_ROOT mount proc /proc -t proc
640+
fi
641+
642+
## Version file part 2
643+
export build_version="${SONIC_IMAGE_VERSION}"
644+
export debian_version="$(cat $FILESYSTEM_ROOT/etc/debian_version)"
645+
export kernel_version="${kversion}"
646+
export asic_type="${sonic_asic_platform}"
647+
export asic_subtype="${TARGET_MACHINE}"
648+
export commit_id="$(git rev-parse --short HEAD)"
649+
export branch="$(git rev-parse --abbrev-ref HEAD)"
650+
export release="$(if [ -f $FILESYSTEM_ROOT/etc/sonic/sonic_release ]; then cat $FILESYSTEM_ROOT/etc/sonic/sonic_release; fi)"
651+
export build_date="$(date -u)"
652+
export build_number="${BUILD_NUMBER:-0}"
653+
export built_by="$USER@$BUILD_HOSTNAME"
654+
export sonic_os_version="${SONIC_OS_VERSION}"
655+
j2 files/build_templates/sonic_version.yml.j2 | sudo tee $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml
656+
618657
if [ -f sonic_debian_extension.sh ]; then
619658
./sonic_debian_extension.sh $FILESYSTEM_ROOT $PLATFORM_DIR $IMAGE_DISTRO
620659
fi

‎rules/functions

-1
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,6 @@ SONIC_DBG_DOCKERS += $(2)
143143
endef
144144

145145

146-
147146
###############################################################################
148147
## Utility functions
149148
###############################################################################

‎slave.mk

+94-2
Original file line numberDiff line numberDiff line change
@@ -448,6 +448,34 @@ else
448448
$(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH))
449449
endif
450450

451+
# Definition of SONIC_RFS_TARGETS
452+
define rfs_get_installer_dependencies
453+
$(call rfs_build_target_name,$(1),$($(1)_MACHINE))
454+
endef
455+
456+
define rfs_build_target_name
457+
$(1)__$(2)__rfs.squashfs
458+
endef
459+
460+
define rfs_define_target
461+
$(eval rfs_target=$(call rfs_build_target_name,$(1),$($(1)_MACHINE)))
462+
$(eval $(rfs_target)_INSTALLER=$(1))
463+
$(eval $(rfs_target)_MACHINE=$($(1)_MACHINE))
464+
$(eval SONIC_RFS_TARGETS+=$(rfs_target))
465+
466+
$(if $($(1)_DEPENDENT_MACHINE),\
467+
$(eval dependent_rfs_target=$(call rfs_build_target_name,$(1),$($(1)_DEPENDENT_MACHINE)))
468+
$(eval $(dependent_rfs_target)_INSTALLER=$(1))
469+
$(eval $(dependent_rfs_target)_MACHINE=$($(1)_DEPENDENT_MACHINE))
470+
$(eval SONIC_RFS_TARGETS+=$(dependent_rfs_target))
471+
$(eval $(rfs_target)_DEPENDENT_RFS=$(dependent_rfs_target)))
472+
endef
473+
474+
$(foreach installer,$(SONIC_INSTALLERS),$(eval $(call rfs_define_target,$(installer))))
475+
$(foreach installer, $(SONIC_INSTALLERS), $(eval $(installer)_RFS_DEPENDS=$(call rfs_get_installer_dependencies,$(installer))))
476+
477+
SONIC_TARGET_LIST += $(addprefix $(TARGET_PATH)/, $(SONIC_RFS_TARGETS))
478+
451479
# Overwrite the buildinfo in slave container
452480
ifeq ($(filter clean,$(MAKECMDGOALS)),)
453481
$(shell DBGOPT='$(DBGOPT)' scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV))
@@ -1210,6 +1238,62 @@ $(DOCKER_LOAD_TARGETS) : $(TARGET_PATH)/%.gz-load : .platform docker-start $$(TA
12101238
## Installers
12111239
###############################################################################
12121240

1241+
$(addprefix $(TARGET_PATH)/, $(SONIC_RFS_TARGETS)) : $(TARGET_PATH)/% : \
1242+
.platform \
1243+
build_debian.sh \
1244+
$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(INITRAMFS_TOOLS) $(LINUX_KERNEL)) \
1245+
$(addsuffix -install,$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(DEBOOTSTRAP))) \
1246+
$$(addprefix $(TARGET_PATH)/,$$($$*_DEPENDENT_RFS)) \
1247+
$(call dpkg_depend,$(TARGET_PATH)/%.dep)
1248+
$(HEADER)
1249+
1250+
$(call LOAD_CACHE,$*,$@)
1251+
1252+
# Skip building the target if it is already loaded from cache
1253+
if [ -z '$($*_CACHE_LOADED)' ] ; then
1254+
1255+
$(eval installer=$($*_INSTALLER))
1256+
$(eval machine=$($*_MACHINE))
1257+
1258+
export debs_path="$(IMAGE_DISTRO_DEBS_PATH)"
1259+
export initramfs_tools="$(IMAGE_DISTRO_DEBS_PATH)/$(INITRAMFS_TOOLS)"
1260+
export linux_kernel="$(IMAGE_DISTRO_DEBS_PATH)/$(LINUX_KERNEL)"
1261+
export kversion="$(KVERSION)"
1262+
export image_type="$($(installer)_IMAGE_TYPE)"
1263+
export sonicadmin_user="$(USERNAME)"
1264+
export sonic_asic_platform="$(patsubst %-$(CONFIGURED_ARCH),%,$(CONFIGURED_PLATFORM))"
1265+
export RFS_SPLIT_FIRST_STAGE=y
1266+
export RFS_SPLIT_LAST_STAGE=n
1267+
1268+
j2 -f env files/initramfs-tools/union-mount.j2 onie-image.conf > files/initramfs-tools/union-mount
1269+
j2 -f env files/initramfs-tools/arista-convertfs.j2 onie-image.conf > files/initramfs-tools/arista-convertfs
1270+
1271+
RFS_SQUASHFS_NAME=$* \
1272+
USERNAME="$(USERNAME)" \
1273+
PASSWORD="$(PASSWORD)" \
1274+
CHANGE_DEFAULT_PASSWORD="$(CHANGE_DEFAULT_PASSWORD)" \
1275+
TARGET_MACHINE=$(machine) \
1276+
IMAGE_TYPE=$($(installer)_IMAGE_TYPE) \
1277+
TARGET_PATH=$(TARGET_PATH) \
1278+
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
1279+
SONIC_ENABLE_SECUREBOOT_SIGNATURE="$(SONIC_ENABLE_SECUREBOOT_SIGNATURE)" \
1280+
SIGNING_KEY="$(SIGNING_KEY)" \
1281+
SIGNING_CERT="$(SIGNING_CERT)" \
1282+
PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
1283+
DBGOPT='$(DBGOPT)' \
1284+
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
1285+
MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \
1286+
CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) \
1287+
MASTER_KUBERNETES_VERSION=$(MASTER_KUBERNETES_VERSION) \
1288+
MASTER_CRI_DOCKERD=$(MASTER_CRI_DOCKERD) \
1289+
./build_debian.sh $(LOG)
1290+
1291+
$(call SAVE_CACHE,$*,$@)
1292+
1293+
fi
1294+
1295+
$(FOOTER)
1296+
12131297
# targets for building installers with base image
12141298
$(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
12151299
.platform \
@@ -1264,7 +1348,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
12641348
$(addprefix $(FILES_PATH)/,$($(SONIC_CTRMGRD)_FILES)) \
12651349
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3)) \
12661350
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \
1267-
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))
1351+
$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3)) \
1352+
$$(addprefix $(TARGET_PATH)/,$$($$*_RFS_DEPENDS))
1353+
12681354
$(HEADER)
12691355
# Pass initramfs and linux kernel explicitly. They are used for all platforms
12701356
export debs_path="$(IMAGE_DISTRO_DEBS_PATH)"
@@ -1420,13 +1506,17 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
14201506
chmod +x sonic_debian_extension.sh,
14211507
)
14221508

1509+
export RFS_SPLIT_FIRST_STAGE=n
1510+
export RFS_SPLIT_LAST_STAGE=y
1511+
14231512
# Build images for the MACHINE, DEPENDENT_MACHINE defined.
14241513
$(foreach dep_machine, $($*_MACHINE) $($*_DEPENDENT_MACHINE), \
14251514
DEBUG_IMG="$(INSTALL_DEBUG_TOOLS)" \
14261515
DEBUG_SRC_ARCHIVE_DIRS="$(DBG_SRC_ARCHIVE)" \
14271516
DEBUG_SRC_ARCHIVE_FILE="$(DBG_SRC_ARCHIVE_FILE)" \
14281517
scripts/dbg_files.sh
14291518

1519+
RFS_SQUASHFS_NAME=$*__$(dep_machine)__rfs.squashfs \
14301520
DEBUG_IMG="$(INSTALL_DEBUG_TOOLS)" \
14311521
DEBUG_SRC_ARCHIVE_FILE="$(DBG_SRC_ARCHIVE_FILE)" \
14321522
USERNAME="$(USERNAME)" \
@@ -1521,7 +1611,9 @@ SONIC_CLEAN_TARGETS += $(addsuffix -clean,$(addprefix $(TARGET_PATH)/, \
15211611
$(SONIC_DOCKER_IMAGES) \
15221612
$(SONIC_DOCKER_DBG_IMAGES) \
15231613
$(SONIC_SIMPLE_DOCKER_IMAGES) \
1524-
$(SONIC_INSTALLERS)))
1614+
$(SONIC_INSTALLERS) \
1615+
$(SONIC_RFS_TARGETS)))
1616+
15251617
$(SONIC_CLEAN_TARGETS) :: $(TARGET_PATH)/%-clean : .platform
15261618
$(Q)rm -rf $(TARGET_PATH)/$* target/versions/dockers/$(subst .gz,,$*)
15271619

0 commit comments

Comments
 (0)