Skip to content

Commit 8dae992

Browse files
danzattdanzatt
authored
Merge pull request #2798 from flatcar/danzatt/nvidia-drivers-signing
Add prebuilt NVIDIA drivers in a sysext - Add capability to specify per-sysext USE flags and compile different versions of upstream portage nvidia-drivers (including open and non-open variants). - Allow architecture-specific OS-dependent sysexts - Pull `nvidia-drivers` from portage and build sysexts from the package Related PRs: NVIDIA tests using sysext: [mantle #598](flatcar/mantle#598) NVIDIA runtime modifications to remove `nvidia-smi` symlink: [sysext-bakery #153](flatcar/sysext-bakery#153)
2 parents df5484c + 0d1dcb5 commit 8dae992

File tree

56 files changed

+6991
-29
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+6991
-29
lines changed

‎.github/workflows/portage-stable-packages-list

+1
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ acct-user/named
5151
acct-user/netperf
5252
acct-user/nobody
5353
acct-user/ntp
54+
acct-user/nvpd
5455
acct-user/pcap
5556
acct-user/pcscd
5657
acct-user/polkitd

‎build_image

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ DEFINE_string base_pkg "coreos-base/coreos" \
3333
"The base portage package to base the build off of (only applies to prod images)"
3434
DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \
3535
"The base portage package to base the build off of (only applies to dev containers)"
36-
DEFINE_string base_sysexts "containerd-flatcar:app-containers/containerd,docker-flatcar:app-containers/docker&app-containers/docker-cli&app-containers/docker-buildx" \
36+
DEFINE_string base_sysexts "containerd-flatcar|app-containers/containerd,docker-flatcar|app-containers/docker&app-containers/docker-cli&app-containers/docker-buildx" \
3737
"Comma-separated list of name:package[&package[&package]] - build 'package' (a single package or a list of packages separated by '&') into sysext 'name', and include with OS image and update payload. Must be in order of dependencies, base sysexts come first."
3838
DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
3939
"Directory in which to place image result directories (named by version)"

‎build_library/extra_sysexts.sh

+25-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,27 @@
11
EXTRA_SYSEXTS=(
2-
zfs:sys-fs/zfs
3-
podman:app-containers/podman,net-misc/passt
4-
python:dev-lang/python,dev-python/pip
2+
"zfs|sys-fs/zfs"
3+
"podman|app-containers/podman,net-misc/passt"
4+
"python|dev-lang/python,dev-python/pip"
5+
"nvidia-drivers-535|x11-drivers/nvidia-drivers:0/535|-kernel-open persistenced|amd64"
6+
"nvidia-drivers-535-open|x11-drivers/nvidia-drivers:0/535|kernel-open persistenced|amd64"
7+
"nvidia-drivers-550|x11-drivers/nvidia-drivers:0/550|-kernel-open persistenced|amd64"
8+
"nvidia-drivers-550-open|x11-drivers/nvidia-drivers:0/550|kernel-open persistenced|amd64"
9+
"nvidia-drivers-570|x11-drivers/nvidia-drivers:0/570|-kernel-open persistenced|amd64"
10+
"nvidia-drivers-570-open|x11-drivers/nvidia-drivers:0/570|kernel-open persistenced|amd64"
511
)
12+
13+
_get_unversioned_sysext_packages_unsorted() {
14+
for sysext in "${EXTRA_SYSEXTS[@]}"; do
15+
IFS="|" read -r _ PACKAGE_ATOMS _ <<< "$sysext"
16+
17+
IFS=,
18+
for atom in $PACKAGE_ATOMS; do
19+
qatom "$atom" -F "%{CATEGORY}/%{PN}"
20+
done
21+
unset IFS
22+
done
23+
}
24+
25+
get_unversioned_sysext_packages() {
26+
_get_unversioned_sysext_packages_unsorted | sort | uniq
27+
}

‎build_library/prod_image_util.sh

+22-4
Original file line numberDiff line numberDiff line change
@@ -213,18 +213,36 @@ create_prod_sysexts() {
213213
local image_name="$1"
214214
local image_sysext_base="${image_name%.bin}_sysext.squashfs"
215215
for sysext in "${EXTRA_SYSEXTS[@]}"; do
216-
local name="flatcar-${sysext%:*}"
217-
local pkgs="${sysext#*:}"
216+
local name pkgs useflags arches
217+
IFS="|" read -r name pkgs useflags arches <<< "$sysext"
218+
name="flatcar-$name"
218219
local pkg_array=(${pkgs//,/ })
220+
local arch_array=(${arches//,/ })
221+
local useflags_array=(${useflags//,/ })
222+
219223
local mangle_script="${BUILD_LIBRARY_DIR}/sysext_mangle_${name}"
220224
if [[ ! -x "${mangle_script}" ]]; then
221225
mangle_script=
222226
fi
227+
228+
if [[ -n "$arches" ]]; then
229+
should_skip=1
230+
for arch in "${arch_array[@]}"; do
231+
if [[ $arch == "$ARCH" ]]; then
232+
should_skip=0
233+
fi
234+
done
235+
if [[ $should_skip -eq 1 ]]; then
236+
continue
237+
fi
238+
fi
239+
223240
sudo rm -f "${BUILD_DIR}/${name}.raw" \
224241
"${BUILD_DIR}/flatcar-test-update-${name}.gz" \
225242
"${BUILD_DIR}/${name}_*"
226-
sudo "${SCRIPT_ROOT}/build_sysext" --board="${BOARD}" \
227-
--squashfs_base="${BUILD_DIR}/${image_sysext_base}" \
243+
# we use -E to pass the USE flags, but also MODULES_SIGN variables
244+
USE="${useflags_array[*]}" sudo -E "${SCRIPT_ROOT}/build_sysext" --board="${BOARD}" \
245+
--squashfs_base="${BUILD_DIR}/${image_sysext_base}" \
228246
--image_builddir="${BUILD_DIR}" \
229247
${mangle_script:+--manglefs_script=${mangle_script}} \
230248
"${name}" "${pkg_array[@]}"

‎build_library/sysext_prod_builder

+2-2
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,8 @@ prev_pkginfo=""
113113
sysext_lowerdirs="${sysext_mountdir}/rootfs-lower"
114114
for sysext in ${sysexts_list//,/ }; do
115115
# format is "<name>:<group>/<package>"
116-
name="${sysext%:*}"
117-
grp_pkg="${sysext#*:}"
116+
name="${sysext%|*}"
117+
grp_pkg="${sysext#*|}"
118118
create_prod_sysext "${BOARD}" \
119119
"${sysext_output_dir}" \
120120
"${sysext_workdir}" \

‎build_packages

+53-1
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ fi
117117
. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
118118
. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
119119
. "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
120+
. "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1
120121

121122
# Setup all the emerge command/flags.
122123
EMERGE_FLAGS=( --update --deep --newuse --verbose --backtrack=30 --select )
@@ -285,6 +286,48 @@ export KBUILD_BUILD_HOST="${BUILD_HOST:-pony-truck.infra.kinvolk.io}"
285286
info "Merging board packages now"
286287
sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "$@"
287288

289+
info "Merging sysext packages now"
290+
for sysext in "${EXTRA_SYSEXTS[@]}"; do
291+
IFS="|" read -r SYSEXT_NAME PACKAGE_ATOMS USEFLAGS ARCHES <<< "$sysext"
292+
293+
arch_array=("${ARCHES//,/ }")
294+
if [[ -n $ARCHES ]]; then
295+
should_skip=1
296+
for arch in "${arch_array[@]}"; do
297+
if [[ $arch == "$ARCH" ]]; then
298+
should_skip=0
299+
fi
300+
done
301+
if [[ $should_skip -eq 1 ]]; then
302+
continue
303+
fi
304+
fi
305+
306+
307+
info "Building packages for $SYSEXT_NAME sysext with USE=$USEFLAGS"
308+
IFS=,
309+
for package in $PACKAGE_ATOMS; do
310+
# --buildpkgonly does not install dependencies, so we install them
311+
# separately before building the binary package
312+
sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
313+
env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
314+
"${EMERGE_FLAGS[@]}" \
315+
--quiet \
316+
--onlydeps \
317+
--binpkg-respect-use=y \
318+
"${package}"
319+
320+
sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
321+
env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
322+
"${EMERGE_FLAGS[@]}" \
323+
--quiet \
324+
--buildpkgonly \
325+
--binpkg-respect-use=y \
326+
"${package}"
327+
done
328+
unset IFS
329+
done
330+
288331
info "Removing obsolete packages"
289332
# The return value of emerge is not clearly reliable. It may fail with
290333
# an output like following:
@@ -319,7 +362,16 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
319362
fi
320363
fi
321364

322-
eclean-$BOARD -d packages
365+
exclusions_file=$(mktemp)
366+
if [ ! -f "$exclusions_file" ]; then
367+
die_notrace "Couldn't create temporary exclusions file $exclusions_file for eclean"
368+
fi
369+
get_unversioned_sysext_packages > "$exclusions_file"
370+
eclean-"$BOARD" -d --exclude-file="$exclusions_file" packages
371+
rm -f "$exclusions_file"
372+
# run eclean again, this time without the --deep option, to clean old versions
373+
# of sysext packages (those, for which .ebuild file no longer exists)
374+
eclean-"$BOARD" packages
323375

324376
info "Checking build root"
325377
test_image_content "${BOARD_ROOT}"

‎build_sysext

+2-1
Original file line numberDiff line numberDiff line change
@@ -221,11 +221,12 @@ info "Building '${SYSEXTNAME}' squashfs with (meta-)packages '${@}' in '${BUILD_
221221

222222
for package; do
223223
echo "Installing package into sysext image: $package"
224-
FEATURES="-ebuild-locks" emerge \
224+
FEATURES="-ebuild-locks binpkg-multi-instance" emerge \
225225
--root="${BUILD_DIR}/install-root" \
226226
--config-root="/build/${FLAGS_board}" \
227227
--sysroot="/build/${FLAGS_board}" \
228228
--usepkgonly \
229+
--binpkg-respect-use=y \
229230
--getbinpkg \
230231
--verbose \
231232
--jobs=${NUM_JOBS} \

‎changelog/changes/2025-05-13-nvidia-sysext.md

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
- Compile OS-dependent NVIDIA kernel module sysexts signed for secure boot. ([scripts#2798](https://github.com/flatcar/scripts/pull/2798/))
2+
- Allow per-sysext USE flags and architecture-specific sysexts. ([scripts#2798](https://github.com/flatcar/scripts/pull/2798/))

‎changelog/changes/2025-05-13-oot-module-signing.md

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
- Sign out-of-tree kernel modules using the ephemeral signing key so that ZFS and NVIDIA sysexts can work with secure boot. ([scripts#2636](https://github.com/flatcar/scripts/pull/2636/))

‎ci-automation/base_sysexts.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,6 @@ if [[ ${1:-} = 'local' ]]; then
66
fi
77

88
ciabs_base_sysexts=(
9-
'containerd-flatcar:app-containers/containerd'
10-
'docker-flatcar:app-containers/docker&app-containers/docker-cli&app-containers/docker-buildx'
9+
'containerd-flatcar|app-containers/containerd'
10+
'docker-flatcar|app-containers/docker&app-containers/docker-cli&app-containers/docker-buildx'
1111
)

‎ci-automation/image_changes.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,7 @@ function get_base_sysext_list() {
280280

281281
source "${scripts_repo}/ci-automation/base_sysexts.sh" 'local'
282282

283-
list_var_ref=( "${ciabs_base_sysexts[@]%%:*}" )
283+
list_var_ref=( "${ciabs_base_sysexts[@]%%|*}" )
284284
}
285285

286286
function get_extra_sysext_list() {
@@ -291,7 +291,7 @@ function get_extra_sysext_list() {
291291
local -a EXTRA_SYSEXTS
292292
source "${scripts_repo}/build_library/extra_sysexts.sh"
293293

294-
list_var_ref=( "${EXTRA_SYSEXTS[@]%%:*}" )
294+
list_var_ref=( "${EXTRA_SYSEXTS[@]%%|*}" )
295295
}
296296

297297
# Generates reports with passed parameters. The report is redirected

‎sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1.ebuild

+1-1
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ RDEPEND="${RDEPEND}
206206
sys-power/acpid
207207
sys-process/lsof
208208
sys-process/procps
209-
x11-drivers/nvidia-drivers
209+
x11-drivers/nvidia-drivers-service
210210
"
211211

212212
# OEM specific bits that need to go in USR

‎sdk_container/src/third_party/coreos-overlay/coreos/config/env/x11-drivers/nvidia-drivers

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
: ${MODULES_ROOT:=$(echo ${SYSROOT}/lib/modules/*)}
2+
KERNEL_DIR="${MODULES_ROOT}/build"
3+
4+
# This addresses an issue with the kernel version compatibility check
5+
# when installing zfs-kmod to /build/<arch> (e.g. via build_packages)
6+
# from its binpkg (i.e. not recompiling it).
7+
SKIP_KERNEL_BINPKG_ENV_RESET=1
8+
9+
# Necessary to prevent KV_FULL & KV_OUT_DIR from being unset
10+
# when building Kernel modules for sysext. See also eclass/linux-info.eclass.
11+
cros_pre_pkg_setup_kernel_version() {
12+
LINUX_INFO_BINARY_RESET=1
13+
get_version
14+
}

‎sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/make.defaults

+2-1
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,9 @@ USE="${USE} bindist"
5757
# linux-fw-redistributable - license for sys-kernel/coreos-firmware
5858
# freedist - license for sys-kernel/coreos-kernel
5959
# intel-ucode - license for sys-firmware/intel-microcode
60+
# NVIDIA-r2 - license for x11-drivers/nvidia-drivers
6061
ACCEPT_LICENSE="${ACCEPT_LICENSE} no-source-code
61-
linux-fw-redistributable freedist intel-ucode"
62+
linux-fw-redistributable freedist intel-ucode NVIDIA-r2"
6263

6364
# Favor our own mirrors over Gentoo's
6465
GENTOO_MIRRORS="

‎sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use.mask

+3
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,6 @@ dev-python/pillow jpeg
3131
# or not. We don't have gtk on Flatcar, so it is not an issue here,
3232
# but we need to mask X, so we won't try pulling gtk package.
3333
app-emulation/qemu X
34+
35+
# disable all tools for NVIDIA driver, keep just kmods
36+
x11-drivers/nvidia-drivers tools X static-libs

‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/install-nvidia renamed to ‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers-service/files/bin/install-nvidia

File renamed without changes.

‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/setup-nvidia renamed to ‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers-service/files/bin/setup-nvidia

+25-10
Original file line numberDiff line numberDiff line change
@@ -97,20 +97,30 @@ EOF
9797
}
9898

9999
function install_and_load() {
100-
# This creates symlinks to sonames
101-
mkdir -p /etc/ld.so.conf.d/
102-
echo "/opt/nvidia/${NVIDIA_CURRENT_INSTALLATION}/usr/lib64" > /etc/ld.so.conf.d/nvidia.conf
103-
ldconfig
104-
105100
modprobe -a i2c_core ipmi_msghandler ipmi_devintf
106101
# This is needed on amd64 due to CONFIG_ACPI_VIDEO=m
107102
modprobe -q video || true
108103

109-
pushd "/opt/nvidia/${NVIDIA_CURRENT_INSTALLATION}/usr/lib/modules/$(uname -r)/video/"
110-
insmod nvidia.ko
111-
insmod nvidia-modeset.ko
112-
insmod nvidia-uvm.ko
113-
popd
104+
if systemd-sysext list | grep -q flatcar-nvidia-drivers; then
105+
modprobe nvidia
106+
modprobe nvidia-modeset
107+
modprobe nvidia-uvm
108+
109+
systemctl daemon-reload
110+
# create the nvpd user
111+
systemd-sysusers
112+
else
113+
# This creates symlinks to sonames
114+
mkdir -p /etc/ld.so.conf.d/
115+
echo "/opt/nvidia/${NVIDIA_CURRENT_INSTALLATION}/usr/lib64" > /etc/ld.so.conf.d/nvidia.conf
116+
117+
pushd "/opt/nvidia/${NVIDIA_CURRENT_INSTALLATION}/usr/lib/modules/$(uname -r)/video/"
118+
insmod nvidia.ko
119+
insmod nvidia-modeset.ko
120+
insmod nvidia-uvm.ko
121+
popd
122+
fi
123+
ldconfig
114124

115125
# based on https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#runfile-verifications
116126
if [ ! -c /dev/nvidiactl ]
@@ -161,6 +171,11 @@ function is_nvidia_installation_required() {
161171
if [[ -d "/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}" ]]; then
162172
return 1
163173
fi
174+
175+
if systemd-sysext list | grep -q flatcar-nvidia-drivers; then
176+
echo "Pre-build NVIDIA drivers sysext is loaded, skipping drivers build."
177+
return 1
178+
fi
164179
}
165180

166181
function presetup() {

‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/nvidia-metadata renamed to ‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers-service/files/nvidia-metadata

File renamed without changes.

‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/units/nvidia.service renamed to ‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers-service/files/units/nvidia.service

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[Unit]
22
Description=NVIDIA Configure Service
33
Wants=network-online.target
4-
After=network-online.target
4+
After=network-online.target systemd-sysext.service
55
Before=containerd.target
66

77
[Service]

‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/nvidia-drivers-535.230.02.ebuild renamed to ‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers-service/nvidia-drivers-service-535.230.02.ebuild

File renamed without changes.

‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/nvidia-drivers-570.86.15.ebuild renamed to ‎sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers-service/nvidia-drivers-service-570.86.15.ebuild

File renamed without changes.

‎sdk_container/src/third_party/portage-stable/acct-user/nvpd/metadata.xml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!DOCTYPE pkgmetadata SYSTEM "https://www.gentoo.org/dtd/metadata.dtd">
3+
<pkgmetadata>
4+
<maintainer type="person">
5+
<email>[email protected]</email>
6+
<name>Ionen Wolkens</name>
7+
</maintainer>
8+
</pkgmetadata>

‎sdk_container/src/third_party/portage-stable/acct-user/nvpd/nvpd-0-r2.ebuild

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Copyright 1999-2024 Gentoo Authors
2+
# Distributed under the terms of the GNU General Public License v2
3+
4+
EAPI=8
5+
6+
inherit acct-user
7+
8+
DESCRIPTION="User for nvidia-persistenced"
9+
ACCT_USER_ID=458
10+
ACCT_USER_GROUPS=( video )
11+
12+
acct-user_add_deps

0 commit comments

Comments
 (0)