Copy over the osbuilder-tools image assets and merge with auroraboot

.github/workflows/image.yml

@@ -61,7 +61,7 @@ jobs:
         with:
           builder: ${{ steps.buildx.outputs.name }}
           build-args: |
-            BINARY_VERSION=${{ steps.prep.outputs.binary_version }}
+            VERSION=${{ steps.prep.outputs.binary_version }}
           context: ./
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm64

.gitignore

@@ -1,3 +1,4 @@
 examples/airgap/build
 examples/airgap/data
 dist/
+build/

Dockerfile

@@ -1,19 +1,24 @@
-ARG VERSION=v0.400.3
 ARG LUET_VERSION=0.35.5
+ARG LEAP_VERSION=15.5
 
 FROM quay.io/luet/base:$LUET_VERSION AS luet
 
 FROM golang AS builder
-ARG BINARY_VERSION=v0.0.0
+ARG VERSION=v0.0.0
 WORKDIR /work
 ADD go.mod .
 ADD go.sum .
 RUN go mod download
 ADD . .
-RUN CGO_ENABLED=0 go build -ldflags "-X main.version=${BINARY_VERSION}" -o auroraboot
-
-FROM quay.io/kairos/osbuilder-tools:$VERSION
+ENV CGO_ENABLED=0
+ENV VERSION=$VERSION
+RUN go build -buildvcs=false -ldflags "-X main.version=${VERSION}" -o auroraboot
 
+FROM opensuse/leap:$LEAP_VERSION AS default
+RUN zypper ref && zypper dup -y
+## ISO+ Arm image + Netboot + cloud images Build depedencies
+RUN zypper ref && zypper in -y bc qemu qemu-tools jq cdrtools docker git curl gptfdisk kpartx sudo xfsprogs parted binutils \
+    util-linux-systemd e2fsprogs curl util-linux udev rsync grub2 dosfstools grub2-x86_64-efi squashfs mtools xorriso lvm2 zstd
 COPY --from=luet /usr/bin/luet /usr/bin/luet
 ENV LUET_NOLOCK=true
 ENV TMPDIR=/tmp
@@ -27,6 +32,80 @@ RUN cp /tmp/luet-${TARGETARCH}.yaml /etc/luet/luet.yaml
 ## Uki artifacts, will be set under the /usr/kairos directory
 RUN luet install -y system/systemd-boot
 
-RUN zypper in -y qemu binutils
+## Live CD artifacts
+RUN luet install -y livecd/grub2 --system-target /grub2
+RUN luet install -y livecd/grub2-efi-image --system-target /efi
+
+## RPI64
+RUN luet install -y firmware/u-boot-rpi64 firmware/raspberrypi-firmware firmware/raspberrypi-firmware-config firmware/raspberrypi-firmware-dt --system-target /rpi/
+
+## PineBook64 Pro
+RUN luet install -y arm-vendor-blob/u-boot-rockchip --system-target /pinebookpro/u-boot
+
+## Odroid fw
+RUN luet install -y firmware/odroid-c2 --system-target /firmware/odroid-c2
+
+## RAW images for current arch
+RUN luet install -y static/grub-efi --system-target /raw/grub
+RUN luet install -y static/grub-config --system-target /raw/grubconfig
+RUN luet install -y static/grub-artifacts --system-target /raw/grubartifacts
+
+## RAW images for arm64
+# Luet will install this artifacts from the current arch repo, so in x86 it will
+# get them from the x86 repo and we want it to do it from the arm64 repo, even on x86
+# so we use the arm64 luet config and use that to install those on x86
+# This is being used by the prepare_arm_images.sh and build-arch-image.sh scripts
+RUN luet install --config /tmp/luet-arm64.yaml -y static/grub-efi --system-target /arm/raw/grubefi
+RUN luet install --config /tmp/luet-arm64.yaml -y static/grub-config --system-target /arm/raw/grubconfig
+RUN luet install --config /tmp/luet-arm64.yaml -y static/grub-artifacts --system-target /arm/raw/grubartifacts
+
+# kairos-agent so we can use the pull-image
+RUN luet install -y system/kairos-agent
+
+# remove luet tmp files. Side effect of setting the system-target is that it treats it as a root fs
+# so temporal files are stored in each dir
+RUN rm -Rf /grub2/var/tmp
+RUN rm -Rf /grub2/var/cache
+RUN rm -Rf /efi/var/tmp
+RUN rm -Rf /efi/var/cache
+RUN rm -Rf /rpi/var/tmp
+RUN rm -Rf /rpi/var/cache
+RUN rm -Rf /pinebookpro/u-boot/var/tmp
+RUN rm -Rf /pinebookpro/u-boot/var/cache
+RUN rm -Rf /firmware/odroid-c2/var/tmp
+RUN rm -Rf /firmware/odroid-c2/var/cache
+RUN rm -Rf /raw/grub/var/tmp
+RUN rm -Rf /raw/grub/var/cache
+RUN rm -Rf /raw/grubconfig/var/tmp
+RUN rm -Rf /raw/grubconfig/var/cache
+RUN rm -Rf /raw/grubartifacts/var/tmp
+RUN rm -Rf /raw/grubartifacts/var/cache
+RUN rm -Rf /arm/raw/grubefi/var/tmp
+RUN rm -Rf /arm/raw/grubefi/var/cache
+RUN rm -Rf /arm/raw/grubconfig/var/tmp
+RUN rm -Rf /arm/raw/grubconfig/var/cache
+RUN rm -Rf /arm/raw/grubartifacts/var/tmp
+RUN rm -Rf /arm/raw/grubartifacts/var/cache
+
+# ISO build config
+COPY ./image-assets/add-cloud-init.sh /add-cloud-init.sh
+COPY ./image-assets/kairos-release.tmpl /kairos-release.tmpl
+COPY ./image-assets/ipxe.tmpl /ipxe.tmpl
+COPY ./image-assets/update-os-release.sh /update-os-release.sh
+
+# ARM helpers
+COPY ./image-assets/build-arm-image.sh /build-arm-image.sh
+COPY ./image-assets/arm /arm
+COPY ./image-assets/prepare_arm_images.sh /prepare_arm_images.sh
+
+# RAW images helpers
+COPY ./image-assets/gce.sh /gce.sh
+COPY ./image-assets/raw-images.sh /raw-images.sh
+COPY ./image-assets/azure.sh /azure.sh
+COPY ./image-assets/netboot.sh /netboot.sh
+
+COPY ./image-assets/defaults.yaml /defaults.yaml
+
 COPY --from=builder /work/auroraboot /usr/bin/auroraboot
+
 ENTRYPOINT ["/usr/bin/auroraboot"]

Earthfile

@@ -1,14 +1,25 @@
 VERSION 0.7
-ARG --global OSBUILDER_VERSION=v0.9.0
 ARG --global GO_VERSION=1.23-bookworm
 
 # renovate: datasource=github-releases depName=kairos-io/kairos
 ARG IMAGE_VERSION=v3.2.1
 ARG --global BASE_IMAGE=quay.io/kairos/ubuntu:24.04-core-amd64-generic-${IMAGE_VERSION}-uki
 
+version:
+  FROM alpine
+  RUN apk update && apk add git
+
+  COPY . .
+  RUN --no-cache git describe --always --tags --dirty > VERSION
+  SAVE ARTIFACT VERSION VERSION
+
 image:
-    FROM DOCKERFILE --build-arg VERSION=$OSBUILDER_VERSION -f Dockerfile .
-    RUN zypper in -y qemu
+  FROM +version
+  ARG VERSION=$(cat VERSION)
+
+  FROM DOCKERFILE --build-arg VERSION=$VERSION -f Dockerfile .
+
+  SAVE IMAGE quay.io/kairos/auroraboot:$VERSION
 
 test-label:
     FROM alpine
@@ -71,3 +82,27 @@ test-bootable:
     ARG CREATE_VM=true
     RUN date
     RUN go run github.com/onsi/ginkgo/v2/ginkgo run --label-filter "bootable" -v --fail-fast -r ./e2e
+
+last-commit-packages:
+    FROM quay.io/skopeo/stable
+    RUN dnf install -y jq
+    WORKDIR build
+    ENV jqQuery='.Tags | map(select(. | contains("-repository.yaml"))) | sort_by(. | sub("v";"") | sub("-repository.yaml";"") | sub("-git.*";"") | .[0:12] | tonumber) | .[-1]'
+    RUN skopeo list-tags docker://quay.io/kairos/packages | jq -rc "${jqQuery}" >  REPO_AMD64
+    RUN skopeo list-tags docker://quay.io/kairos/packages-arm64 | jq -rc "${jqQuery}" > REPO_ARM64
+    SAVE ARTIFACT REPO_AMD64 REPO_AMD64
+    SAVE ARTIFACT REPO_ARM64 REPO_ARM64
+
+bump-repositories:
+    FROM mikefarah/yq
+    WORKDIR build
+    COPY +last-commit-packages/REPO_AMD64 REPO_AMD64
+    COPY +last-commit-packages/REPO_ARM64 REPO_ARM64
+    ARG REPO_AMD64=$(cat REPO_AMD64)
+    ARG REPO_ARM64=$(cat REPO_ARM64)
+    COPY image-assets/luet-amd64.yaml luet-amd64.yaml
+    COPY image-assets/luet-arm64.yaml luet-arm64.yaml
+    RUN yq eval ".repositories[0] |= . * { \"reference\": \"${REPO_AMD64}\" }" -i luet-amd64.yaml
+    RUN yq eval ".repositories[0] |= . * { \"reference\": \"${REPO_ARM64}\" }" -i luet-arm64.yaml
+    SAVE ARTIFACT luet-arm64.yaml AS LOCAL image-assets/luet-arm64.yaml
+    SAVE ARTIFACT luet-amd64.yaml AS LOCAL image-assets/luet-amd64.yaml

deployer/register.go

@@ -19,7 +19,7 @@ const (
 	opPrepareNetboot   = "prepare-netboot"
 	opStartNetboot     = "start-netboot"
 
-	opContainerPull  = "container-pull"
+	opDumpSource     = "dump-source"
 	opGenISO         = "gen-iso"
 	opPreparetmproot = "prepare-temp"
 	opExtractNetboot = "extract-netboot"
@@ -43,7 +43,7 @@ func RegisterAll(d *Deployer) error {
 		d.StepPrepNetbootDir,
 		d.StepPrepISODir,
 		d.StepCopyCloudConfig,
-		d.StepPullContainer,
+		d.StepDumpSource,
 		d.StepGenISO,
 		d.StepExtractNetboot,
 		//TODO: add Validate step

deployer/steps.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/kairos-io/AuroraBoot/pkg/ops"
 	"github.com/spectrocloud-labs/herd"
@@ -21,7 +20,7 @@ func (d *Deployer) StepPrepNetbootDir() error {
 func (d *Deployer) StepPrepTmpRootDir() error {
 	return d.Add(opPreparetmproot, herd.WithCallback(
 		func(ctx context.Context) error {
-			return os.MkdirAll(d.dstNetboot(), 0700)
+			return os.MkdirAll(d.tmpRootFs(), 0700)
 		},
 	))
 }
@@ -40,17 +39,17 @@ func (d *Deployer) StepCopyCloudConfig() error {
 		}))
 }
 
-func (d *Deployer) StepPullContainer() error {
+func (d *Deployer) StepDumpSource() error {
 	// Ops to generate from container image
-	return d.Add(opContainerPull,
+	return d.Add(opDumpSource,
 		herd.EnableIf(d.fromImage),
-		herd.WithDeps(opPreparetmproot), herd.WithCallback(ops.PullContainerImage(d.containerImage(), d.tmpRootFs())))
+		herd.WithDeps(opPreparetmproot), herd.WithCallback(ops.DumpSource(d.Artifact.ContainerImage, d.tmpRootFs())))
 }
 
 func (d *Deployer) StepGenISO() error {
 	return d.Add(opGenISO,
 		herd.EnableIf(func() bool { return d.fromImage() && !d.rawDiskIsSet() && d.Config.Disk.ARM == nil }),
-		herd.WithDeps(opContainerPull, opCopyCloudConfig), herd.WithCallback(ops.GenISO(d.tmpRootFs(), d.destination(), d.Config.ISO)))
+		herd.WithDeps(opDumpSource, opCopyCloudConfig), herd.WithCallback(ops.GenISO(d.tmpRootFs(), d.destination(), d.Config.ISO)))
 }
 
 func (d *Deployer) StepExtractNetboot() error {
@@ -178,16 +177,6 @@ func (d *Deployer) StepStartNetboot() error {
 	)
 }
 
-func (d *Deployer) containerImage() string {
-	// Pull local docker daemon if container image starts with docker://
-	containerImage := d.Artifact.ContainerImage
-	if strings.HasPrefix(containerImage, "docker://") {
-		containerImage = strings.ReplaceAll(containerImage, "docker://", "")
-	}
-
-	return containerImage
-}
-
 func (d *Deployer) fromImage() bool {
 	return d.Artifact.ContainerImage != ""
 }
@@ -233,7 +222,7 @@ func (d *Deployer) isoOption() bool {
 }
 
 func (d *Deployer) imageOrSquashFS() herd.OpOption {
-	return herd.IfElse(d.fromImage(), herd.WithDeps(opContainerPull), herd.WithDeps(opExtractSquashFS))
+	return herd.IfElse(d.fromImage(), herd.WithDeps(opDumpSource), herd.WithDeps(opExtractSquashFS))
 }
 
 func (d *Deployer) cloudConfigPath() string {

go.mod

@@ -9,7 +9,6 @@ godebug x509negativeserial=1
 require (
 	github.com/cavaliergopher/grab/v3 v3.0.1
 	github.com/containerd/containerd v1.7.23
-	github.com/distribution/reference v0.6.0
 	github.com/foxboron/go-uefi v0.0.0-20241017190036-fab4fdf2f2f3
 	github.com/foxboron/sbctl v0.0.0-20240526163235-64e649b31c8e
 	github.com/gofrs/uuid v4.4.0+incompatible
@@ -71,6 +70,7 @@ require (
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/denisbrodbeck/machineid v1.0.1 // indirect
 	github.com/diskfs/go-diskfs v1.4.2 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
 	github.com/djherbis/times v1.6.0 // indirect
 	github.com/docker/cli v27.1.1+incompatible // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect

image-assets/add-cloud-init.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# docker run --entrypoint /add-cloud-init.sh -v $PWD:/work -ti --rm test https://github.com/kairos-io/kairos/releases/download/v1.1.2/kairos-alpine-v1.1.2.iso /work/test.iso /work/config.yaml
+
+set -ex
+
+ISO=$1
+OUT=$2
+CONFIG=$3
+
+case ${ISO} in
+http*)
+    curl -L "${ISO}" -o in.iso
+    ISO=in.iso
+    ;;
+esac
+
+# Needs xorriso >=1.5.4
+xorriso -indev $ISO -outdev $OUT -map $CONFIG /config.yaml -boot_image any replay

image-assets/arm/boards/odroid_c2.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+image=$1
+
+if [ -z "$image" ]; then
+    echo "No image specified"
+    exit 1
+fi
+
+# conv=notrunc ?
+dd if=/firmware/odroid-c2/bl1.bin.hardkernel of=$image conv=fsync bs=1 count=442
+dd if=/firmware/odroid-c2/bl1.bin.hardkernel of=$image conv=fsync bs=512 skip=1 seek=1
+dd if=/firmware/odroid-c2/u-boot.odroidc2 of=$image conv=fsync bs=512 seek=97

image-assets/arm/boards/pinebookpro.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+image=$1
+
+if [ -z "$image" ]; then
+    echo "No image specified"
+    exit 1
+fi
+
+LOADER_OFFSET=${LOADER_OFFSET:-"64"}
+LOADER_IMAGE=${LOADER_IMAGE:-"idbloader.img"}
+UBOOT_IMAGE=${UBOOT_IMAGE:-"u-boot.itb"}
+UBOOT_OFFSET=${UBOOT_OFFSET:-"16384"}
+
+echo "Writing idbloader"
+dd conv=notrunc if=/pinebookpro/u-boot/usr/lib/u-boot/pinebook-pro-rk3399/${LOADER_IMAGE} of="$image" conv=fsync seek=${LOADER_OFFSET}
+echo "Writing u-boot image"
+dd conv=notrunc if=/pinebookpro/u-boot/usr/lib/u-boot/pinebook-pro-rk3399/${UBOOT_IMAGE} of="$image" conv=fsync seek=${UBOOT_OFFSET}
+sync $image

image-assets/arm/boards/rpi3.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+partprobe
+
+kpartx -va $DRIVE
+
+image=$1
+
+if [ -z "$image" ]; then
+    echo "No image specified"
+    exit 1
+fi
+
+set -ax
+TEMPDIR="$(mktemp -d)"
+echo $TEMPDIR
+mount "${device}p1" "${TEMPDIR}"
+
+# Copy all rpi files
+cp -rfv /rpi/* $TEMPDIR
+
+umount "${TEMPDIR}"

image-assets/arm/boards/rpi4.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+partprobe
+
+image=$1
+
+if [ -z "$image" ]; then
+    echo "No image specified"
+    exit 1
+fi
+
+set -ax
+TEMPDIR="$(mktemp -d)"
+echo $TEMPDIR
+mount "${device}p1" "${TEMPDIR}"
+
+# Copy all rpi files
+cp -rfv /rpi/* $TEMPDIR
+
+umount "${TEMPDIR}"