Merge branch 'lxc:main' into stable-test-final-pr

Author: NathanChase22

CONTRIBUTING.md

@@ -18,6 +18,22 @@ By default, any contribution to this project is made under the Apache
 The author of a change remains the copyright holder of their code
 (no copyright assignment).
 
+## No Large Language Models (LLMs) or similar AI tools
+
+All contributions to this project are expected to be done by human
+beings or through standard predictable tooling (e.g. scripts, formatters, ...).
+
+We expect all contributors to be able to reason about the code that they
+contribute and explain why they're taking a particular approach.
+
+LLMs and similar predictive tools have the annoying tendency of
+producing large amount of low quality code with subtle issues which end
+up taking the maintainers more time to debug than it would have taken to
+write the code by hand in the first place.
+
+Any attempt at hiding the use of LLMs or similar tools in Incus contributions
+will result in a revert of the affected changes and a ban from the project.
+
 ## Pull requests
 
 Changes to this project should be proposed as pull requests on GitHub

Makefile

@@ -95,6 +95,7 @@ ifneq "$(INCUS_OFFLINE)" ""
 endif
 	$(GO) get -t -v -u ./...
 	$(GO) get github.com/go-jose/go-jose/[email protected]
+	$(GO) get github.com/olekukonko/[email protected]
 	$(GO) mod tidy --go=1.23.7
 	$(GO) get toolchain@none
 
@@ -306,7 +307,6 @@ endif
 ifeq ($(shell command -v shellcheck),)
 	echo "Please install shellcheck"
 	exit 1
-else
 endif
 ifeq ($(shell command -v flake8),)
 	echo "Please install flake8"

client/incus_network_address_sets.go

@@ -44,8 +44,8 @@ func (r *ProtocolIncus) GetNetworkAddressSets() ([]api.NetworkAddressSet, error)
 
 // GetNetworkAddressSetsAllProjects returns a list of network address set structs across all projects.
 func (r *ProtocolIncus) GetNetworkAddressSetsAllProjects() ([]api.NetworkAddressSet, error) {
-	if !r.HasExtension("network_address_sets_all_projects") {
-		return nil, fmt.Errorf(`The server is missing the required "network_address_sets_all_projects" API extension`)
+	if !r.HasExtension("network_address_set") {
+		return nil, fmt.Errorf(`The server is missing the required "network_address_set" API extension`)
 	}
 
 	addressSets := []api.NetworkAddressSet{}

client/incus_server.go

@@ -537,6 +537,16 @@ func (r *ProtocolIncus) ApplyServerPreseed(config api.InitPreseed) error {
 		}
 	}
 
+	// Apply certificate configuration.
+	if len(config.Server.Certificates) > 0 {
+		for _, certificate := range config.Server.Certificates {
+			err := r.CreateCertificate(certificate)
+			if err != nil {
+				return fmt.Errorf("Failed to create certificate %q: %w", certificate.Name, err)
+			}
+		}
+	}
+
 	// Cluster configuration.
 	if config.Cluster != nil && config.Cluster.Enabled {
 		// Get the current cluster configuration

client/incus_storage_volumes.go

@@ -3,10 +3,13 @@ package incus
 import (
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"strings"
 
+	"github.com/pkg/sftp"
+
 	"github.com/lxc/incus/v6/shared/api"
 	"github.com/lxc/incus/v6/shared/cancel"
 	"github.com/lxc/incus/v6/shared/ioprogress"
@@ -1119,3 +1122,44 @@ func (r *ProtocolIncus) CreateStoragePoolVolumeFromBackup(pool string, args Stor
 
 	return &op, nil
 }
+
+// GetStoragePoolVolumeFileSFTPConn returns a connection to the volume's SFTP endpoint.
+func (r *ProtocolIncus) GetStoragePoolVolumeFileSFTPConn(pool string, volType string, volName string) (net.Conn, error) {
+	if !r.HasExtension("custom_volume_sftp") {
+		return nil, fmt.Errorf(`The server is missing the required "custom_volume_sftp" API extension`)
+	}
+
+	u := api.NewURL()
+	u.URL = r.httpBaseURL // Preload the URL with the client base URL.
+	u.Path("1.0", "storage-pools", pool, "volumes", volType, volName, "sftp")
+	r.setURLQueryAttributes(&u.URL)
+
+	return r.rawSFTPConn(&u.URL)
+}
+
+// GetStoragePoolVolumeFileSFTP returns an SFTP connection to the volume.
+func (r *ProtocolIncus) GetStoragePoolVolumeFileSFTP(pool string, volType string, volName string) (*sftp.Client, error) {
+	if !r.HasExtension("custom_volume_sftp") {
+		return nil, fmt.Errorf(`The server is missing the required "custom_volume_sftp" API extension`)
+	}
+
+	conn, err := r.GetStoragePoolVolumeFileSFTPConn(pool, volType, volName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get a SFTP client.
+	client, err := sftp.NewClientPipe(conn, conn, sftp.MaxPacketUnchecked(128*1024))
+	if err != nil {
+		_ = conn.Close()
+		return nil, err
+	}
+
+	go func() {
+		// Wait for the client to be done before closing the connection.
+		_ = client.Wait()
+		_ = conn.Close()
+	}()
+
+	return client, nil
+}

client/interfaces.go

@@ -388,6 +388,10 @@ type InstanceServer interface {
 	CreateStoragePoolVolumeFromISO(pool string, args StorageVolumeBackupArgs) (op Operation, err error)
 	CreateStoragePoolVolumeFromMigration(pool string, volume api.StorageVolumesPost) (op Operation, err error)
 
+	// Storage volume SFTP functions ("custom_volume_sftp" API extension)
+	GetStoragePoolVolumeFileSFTPConn(pool string, volType string, volName string) (net.Conn, error)
+	GetStoragePoolVolumeFileSFTP(pool string, volType string, volName string) (*sftp.Client, error)
+
 	// Cluster functions ("cluster" API extensions)
 	GetCluster() (cluster *api.Cluster, ETag string, err error)
 	UpdateCluster(cluster api.ClusterPut, ETag string) (op Operation, err error)

client/oci_images.go

@@ -155,11 +155,6 @@ func (r *ProtocolOCI) GetImageFile(fingerprint string, req ImageFileRequest) (*I
 		return nil, fmt.Errorf("OCI image export currently requires root access")
 	}
 
-	_, err = exec.LookPath("umoci")
-	if err != nil {
-		return nil, fmt.Errorf("OCI container handling requires \"umoci\" be present on the system")
-	}
-
 	// Get some temporary storage.
 	ociPath, err := os.MkdirTemp("", "incus-oci-")
 	if err != nil {
@@ -183,6 +178,8 @@ func (r *ProtocolOCI) GetImageFile(fingerprint string, req ImageFileRequest) (*I
 		req.ProgressHandler(ioprogress.ProgressData{Text: "Retrieving OCI image from registry"})
 	}
 
+	imageTag := "latest"
+
 	stdout, _, err := subprocess.RunCommandSplit(
 		ctx,
 		env,
@@ -192,7 +189,7 @@ func (r *ProtocolOCI) GetImageFile(fingerprint string, req ImageFileRequest) (*I
 		"copy",
 		"--remove-signatures",
 		fmt.Sprintf("%s/%s", strings.Replace(r.httpHost, "https://", "docker://", 1), info.Alias),
-		fmt.Sprintf("oci:%s:latest", filepath.Join(ociPath, "oci")))
+		fmt.Sprintf("oci:%s:%s", filepath.Join(ociPath, "oci"), imageTag))
 	if err != nil {
 		logger.Debug("Error copying remote image to local", logger.Ctx{"image": info.Alias, "stdout": stdout, "stderr": err})
 		return nil, err
@@ -203,14 +200,9 @@ func (r *ProtocolOCI) GetImageFile(fingerprint string, req ImageFileRequest) (*I
 		req.ProgressHandler(ioprogress.ProgressData{Text: "Unpacking the OCI image"})
 	}
 
-	stdout, err = subprocess.RunCommand(
-		"umoci",
-		"unpack",
-		"--keep-dirlinks",
-		"--image", filepath.Join(ociPath, "oci"),
-		filepath.Join(ociPath, "image"))
+	err = unpackOCIImage(filepath.Join(ociPath, "oci"), imageTag, filepath.Join(ociPath, "image"))
 	if err != nil {
-		logger.Debug("Error unpacking OCI image", logger.Ctx{"image": filepath.Join(ociPath, "oci"), "stdout": stdout, "stderr": err})
+		logger.Debug("Error unpacking OCI image", logger.Ctx{"image": filepath.Join(ociPath, "oci"), "err": err})
 		return nil, err
 	}
 

client/oci_util.go

@@ -0,0 +1,11 @@
+//go:build !linux
+
+package incus
+
+import (
+	"fmt"
+)
+
+func unpackOCIImage(imagePath string, imageTag string, bundlePath string) error {
+	return fmt.Errorf("Platform isn't supported")
+}

client/oci_util_linux.go

@@ -0,0 +1,60 @@
+//go:build linux
+
+package incus
+
+import (
+	"fmt"
+
+	"github.com/apex/log"
+	"github.com/opencontainers/umoci"
+	"github.com/opencontainers/umoci/oci/cas/dir"
+	"github.com/opencontainers/umoci/oci/casext"
+	"github.com/opencontainers/umoci/oci/layer"
+
+	"github.com/lxc/incus/v6/shared/logger"
+)
+
+// Custom handler to intercept logs.
+type umociLogHandler struct {
+	Message string
+}
+
+// HandleLog implements a proxy between apex/log and our logger.
+func (h *umociLogHandler) HandleLog(e *log.Entry) error {
+	switch e.Level {
+	case log.DebugLevel:
+		logger.Debug(h.Message, logger.Ctx{"log": e.Message})
+	case log.InfoLevel:
+		logger.Info(h.Message, logger.Ctx{"log": e.Message})
+	case log.WarnLevel:
+		logger.Warn(h.Message, logger.Ctx{"log": e.Message})
+	case log.ErrorLevel:
+		logger.Error(h.Message, logger.Ctx{"log": e.Message})
+	case log.FatalLevel:
+		logger.Panic(h.Message, logger.Ctx{"log": e.Message})
+	default:
+		logger.Error("Unknown umoci log level", logger.Ctx{"log": e.Message})
+	}
+
+	return nil
+}
+
+func unpackOCIImage(imagePath string, imageTag string, bundlePath string) error {
+	// Set the custom handler
+	log.SetHandler(&umociLogHandler{Message: "Unpacking OCI image"})
+	defer log.SetHandler(nil)
+
+	var unpackOptions layer.UnpackOptions
+	unpackOptions.KeepDirlinks = true
+
+	// Get a reference to the CAS.
+	engine, err := dir.Open(imagePath)
+	if err != nil {
+		return fmt.Errorf("Open CAS: %w", err)
+	}
+
+	engineExt := casext.NewEngine(engine)
+	defer func() { _ = engine.Close() }()
+
+	return umoci.Unpack(engineExt, imageTag, bundlePath, unpackOptions)
+}

cmd/generate-config/incus_doc.go

@@ -69,7 +69,7 @@ func getSortedKeysFromMap[K string, V IterableAny](m map[K]V) []K {
 		keys = append(keys, k)
 	}
 
-	sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
+	slices.Sort(keys)
 	return keys
 }
 
@@ -262,7 +262,7 @@ func writeDocFile(inputJSONPath, outputTxtPath string) error {
 	countMaxBackTicks := func(s string) int {
 		count, curr_count := 0, 0
 		n := len(s)
-		for i := 0; i < n; i++ {
+		for i := range n {
 			if s[i] == '`' {
 				curr_count++
 				continue

cmd/generate-database/db.go

@@ -132,8 +132,9 @@ func generate(pkgs []string, boilerplateFilename string) error {
 				// Lazy matching for prefix, does not consider Go syntax and therefore
 				// lines starting with prefix, that are part of e.g. multiline strings
 				// match as well. This is highly unlikely to cause false positives.
-				if strings.HasPrefix(line, prefix) {
-					line = strings.TrimPrefix(line, prefix)
+				after, ok := strings.CutPrefix(line, prefix)
+				if ok {
+					line = after
 
 					// Use csv parser to properly handle arguments surrounded by double quotes.
 					r := csv.NewReader(strings.NewReader(line))

cmd/generate-database/db/method.go

@@ -621,7 +621,7 @@ func (m *Method) getRefs(buf *file.Buffer, parentTable string, refMapping *Mappi
 
 	switch refMapping.Type {
 	case ReferenceTable:
-		buf.L("%s, err := Get%s(ctx, db, \"%s\", \"%s\", filters...)", refParentList, lex.Plural(refStruct), parentTable, m.entity)
+		buf.L("%s, err := Get%s(ctx, db, \"%s\", \"%s\", filters...)", refParentList, lex.Plural(refStruct), parentTable, lex.SnakeCase(m.entity))
 		m.ifErrNotNil(buf, true, "nil", "err")
 		buf.L("%s := map[string]%s{}", refList, refMapping.ImportType())
 		buf.L("for _, ref := range %s[%sID] {", refParentList, refParent)
@@ -634,7 +634,7 @@ func (m *Method) getRefs(buf *file.Buffer, parentTable string, refMapping *Mappi
 		buf.L("}")
 		buf.N()
 	case MapTable:
-		buf.L("%s, err := Get%s(ctx, db, \"%s\", \"%s\", filters...)", refParentList, lex.Plural(refStruct), parentTable, m.entity)
+		buf.L("%s, err := Get%s(ctx, db, \"%s\", \"%s\", filters...)", refParentList, lex.Plural(refStruct), parentTable, lex.SnakeCase(m.entity))
 		m.ifErrNotNil(buf, true, "nil", "err")
 		buf.L("%s, ok := %s[%sID]", refList, refParentList, refParent)
 		buf.L("if !ok {")
@@ -1023,7 +1023,7 @@ func (m *Method) createRefs(buf *file.Buffer, parentTable string, refMapping *Ma
 		buf.L("%s[key] = %s", lex.Plural(refVar), refVar)
 		buf.L("}")
 		buf.N()
-		buf.L("err := Create%s(ctx, db, \"%s\", \"%s\", %s)", lex.Plural(refStruct), parentTable, m.entity, lex.Plural(refVar))
+		buf.L("err := Create%s(ctx, db, \"%s\", \"%s\", %s)", lex.Plural(refStruct), parentTable, lex.SnakeCase(m.entity), lex.Plural(refVar))
 		m.ifErrNotNil(buf, false, fmt.Sprintf("fmt.Errorf(\"Insert %s failed for %s: %%w\", err)", refStruct, lex.PascalCase(m.entity)))
 	case MapTable:
 		buf.L("referenceID := int(%sID)", refParent)
@@ -1035,7 +1035,7 @@ func (m *Method) createRefs(buf *file.Buffer, parentTable string, refMapping *Ma
 
 		buf.L("}")
 		buf.N()
-		buf.L("err := Create%s(ctx, db, \"%s\", \"%s\", insert)", refStruct, parentTable, m.entity)
+		buf.L("err := Create%s(ctx, db, \"%s\", \"%s\", insert)", refStruct, parentTable, lex.SnakeCase(m.entity))
 		m.ifErrNotNil(buf, true, fmt.Sprintf("fmt.Errorf(\"Insert %s failed for %s: %%w\", err)", refStruct, lex.PascalCase(m.entity)))
 		buf.L("}")
 	}
@@ -1277,7 +1277,7 @@ func (m *Method) updateRefs(buf *file.Buffer, parentTable string, refMapping *Ma
 	refList := lex.Plural(refVar)
 	refParent := lex.CamelCase(m.entity)
 
-	buf.L("err := Update%s(ctx, db, \"%s\", \"%s\", int(%sID), %s)", lex.Plural(refStruct), parentTable, m.entity, refParent, refList)
+	buf.L("err := Update%s(ctx, db, \"%s\", \"%s\", int(%sID), %s)", lex.Plural(refStruct), parentTable, lex.SnakeCase(m.entity), refParent, refList)
 	m.ifErrNotNil(buf, true, fmt.Sprintf("fmt.Errorf(\"Replace %s for %s failed: %%w\", err)", refStruct, lex.PascalCase(m.entity)))
 	buf.L("return nil")
 

cmd/generate-database/db/parse.go

@@ -424,7 +424,7 @@ func findStruct(scope *types.Scope, name string) *types.Struct {
 func parseStruct(str *types.Struct, kind string, pkgName string) ([]*Field, error) {
 	fields := make([]*Field, 0)
 
-	for i := 0; i < str.NumFields(); i++ {
+	for i := range str.NumFields() {
 		f := str.Field(i)
 		if f.Embedded() {
 			// Check if this is a parent struct.

cmd/incus-agent/api_1.0.go

@@ -111,6 +111,10 @@ func api10Put(d *Daemon, r *http.Request) response.Response {
 }
 
 func startDevIncusServer(d *Daemon) error {
+	if !osGuestAPISupport {
+		return nil
+	}
+
 	d.DevIncusMu.Lock()
 	defer d.DevIncusMu.Unlock()
 
@@ -148,6 +152,10 @@ func startDevIncusServer(d *Daemon) error {
 }
 
 func stopDevIncusServer(d *Daemon) error {
+	if !osGuestAPISupport {
+		return nil
+	}
+
 	d.DevIncusMu.Lock()
 	d.DevIncusRunning = false
 	d.DevIncusMu.Unlock()

cmd/incus-agent/dev_incus.go

@@ -121,7 +121,7 @@ var DevIncusMetadataGet = devIncusHandler{"/1.0/meta-data", func(d *Daemon, w ht
 	var client incus.InstanceServer
 	var err error
 
-	for i := 0; i < 10; i++ {
+	for range 10 {
 		client, err = getVsockClient(d)
 		if err == nil {
 			break

cmd/incus-agent/events.go

@@ -148,7 +148,7 @@ func eventsProcess(event api.Event) {
 	// Attempt to perform the mount.
 	mntSource := fmt.Sprintf("incus_%s", e.Name)
 
-	for i := 0; i < 20; i++ {
+	for range 20 {
 		time.Sleep(500 * time.Millisecond)
 
 		err = osMountShared(mntSource, e.Config["path"], "virtiofs", nil)