Merge pull request #1621 from luissimas/linstor

Add `linstor` storage driver

Author: stgraber

.github/workflows/tests.yml

@@ -140,6 +140,7 @@ jobs:
           - lvm
           - zfs
           - ceph
+          - linstor
           - random
         os:
           - ubuntu-22.04
@@ -391,6 +392,48 @@ jobs:
           sudo microceph.ceph status
           sudo rm -f /snap/bin/rbd
 
+      - name: Setup LINSTOR
+        if: ${{ matrix.backend == 'linstor' }}
+        run: |
+          set -x
+
+          # As with Ceph, we hope for a spare disk.
+          if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then
+            echo "FAIL: rootfs and ephemeral part on the same disk, aborting"
+            exit 1
+          fi
+
+          sudo add-apt-repository ppa:linbit/linbit-drbd9-stack -y
+
+          # Install everything required to compile DRBD and run LINSTOR tools.
+          sudo apt-get install --no-install-recommends -y \
+            drbd-dkms \
+            linstor-client \
+            linstor-controller \
+            linstor-satellite \
+            linux-headers-generic \
+            python3-setuptools
+
+          # Enable DRBD.
+          sudo modprobe -r drbd
+          sudo modprobe drbd
+
+          # Get the runner IP.
+          runner_ip="$(hostname -I | cut -d' ' -f1)"
+
+          # Create a single local node.
+          sudo linstor node create local "${runner_ip}" --node-type combined
+
+          # Repurpose the ephemeral disk for LINSTOR physical storage.
+          sudo swapoff /mnt/swapfile
+          ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')"
+          sudo umount /mnt
+          sudo wipefs -a "${ephemeral_disk}"
+          sudo linstor physical-storage create-device-pool --storage-pool incus --pool-name linstor-incus lvmthin local "${ephemeral_disk}"
+
+          # Update the runner env.
+          echo "INCUS_LINSTOR_CLUSTER=${runner_ip}" >> "$GITHUB_ENV"
+
       - name: "Ensure offline mode (block image server)"
         run: |
           sudo nft add table inet filter
@@ -404,6 +447,7 @@ jobs:
           INCUS_CEPH_CLUSTER: "ceph"
           INCUS_CEPH_CEPHFS: "cephfs"
           INCUS_CEPH_CEPHOBJECT_RADOSGW: "http://127.0.0.1"
+          INCUS_LINSTOR_LOCAL_SATELLITE: "local"
           INCUS_CONCURRENT: "1"
           INCUS_VERBOSE: "1"
           INCUS_OFFLINE: "1"
@@ -413,7 +457,7 @@ jobs:
           chmod +x ~
           echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid
           cd test
-          sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,INCUS_VERBOSE,INCUS_BACKEND,INCUS_CEPH_CLUSTER,INCUS_CEPH_CEPHFS,INCUS_CEPH_CEPHOBJECT_RADOSGW,INCUS_OFFLINE,INCUS_SKIP_TESTS,INCUS_REQUIRED_TESTS, INCUS_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }}
+          sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,INCUS_VERBOSE,INCUS_BACKEND,INCUS_CEPH_CLUSTER,INCUS_CEPH_CEPHFS,INCUS_CEPH_CEPHOBJECT_RADOSGW,INCUS_LINSTOR_LOCAL_SATELLITE,INCUS_LINSTOR_CLUSTER,INCUS_OFFLINE,INCUS_SKIP_TESTS,INCUS_REQUIRED_TESTS, INCUS_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }}
 
   client:
     name: Client

cmd/incusd/api_1.0.go

@@ -792,6 +792,7 @@ func doApi10UpdateTriggers(d *Daemon, nodeChanged, clusterChanged map[string]str
 	oidcChanged := false
 	openFGAChanged := false
 	ovnChanged := false
+	linstorChanged := false
 	ovsChanged := false
 	syslogChanged := false
 
@@ -835,6 +836,9 @@ func doApi10UpdateTriggers(d *Daemon, nodeChanged, clusterChanged map[string]str
 
 		case "openfga.api.url", "openfga.api.token", "openfga.store.id":
 			openFGAChanged = true
+
+		case "storage.linstor.controller_connection", "storage.linstor.ca_cert", "storage.linstor.client_cert", "storage.linstor.client_key":
+			linstorChanged = true
 		}
 	}
 
@@ -1002,6 +1006,13 @@ func doApi10UpdateTriggers(d *Daemon, nodeChanged, clusterChanged map[string]str
 		}
 	}
 
+	if linstorChanged {
+		err := d.setupLinstor()
+		if err != nil {
+			return err
+		}
+	}
+
 	// Compile and load the instance placement scriptlet.
 	value, ok = clusterChanged["instances.placement.scriptlet"]
 	if ok {

cmd/incusd/daemon.go

@@ -63,6 +63,7 @@ import (
 	"github.com/lxc/incus/v6/internal/server/state"
 	storagePools "github.com/lxc/incus/v6/internal/server/storage"
 	storageDrivers "github.com/lxc/incus/v6/internal/server/storage/drivers"
+	"github.com/lxc/incus/v6/internal/server/storage/linstor"
 	"github.com/lxc/incus/v6/internal/server/storage/s3/miniod"
 	"github.com/lxc/incus/v6/internal/server/sys"
 	"github.com/lxc/incus/v6/internal/server/syslog"
@@ -172,6 +173,10 @@ type Daemon struct {
 
 	// API info.
 	apiExtensions int
+
+	// Linstor client.
+	linstor   *linstor.Client
+	linstorMu sync.Mutex
 }
 
 // DaemonConfig holds configuration values for Daemon.
@@ -604,6 +609,7 @@ func (d *Daemon) State() *state.State {
 		OS:                     d.os,
 		OVN:                    d.getOVN,
 		OVS:                    d.getOVS,
+		Linstor:                d.getLinstor,
 		Proxy:                  d.proxy,
 		ServerCert:             d.serverCert,
 		ServerClustered:        d.serverClustered,
@@ -2732,3 +2738,40 @@ func (d *Daemon) getOVS() (*ovs.VSwitch, error) {
 
 	return d.ovs, nil
 }
+
+func (d *Daemon) setupLinstor() error {
+	d.linstorMu.Lock()
+	defer d.linstorMu.Unlock()
+
+	// Clear any existing client.
+	d.linstor = nil
+
+	// Get the Linstor controller connection string.
+	controllerConnection := d.globalConfig.LinstorControllerConnection()
+
+	// Get the SSL certificates if needed.
+	sslCACert, sslClientCert, sslClientKey := d.globalConfig.LinstorSSL()
+
+	// Get Linstor client.
+	client, err := linstor.NewClient(controllerConnection, sslCACert, sslClientCert, sslClientKey)
+	if err != nil {
+		return fmt.Errorf("Failed to connect to Linstor: %w", err)
+	}
+
+	// Set the client.
+	d.linstor = client
+
+	return nil
+}
+
+func (d *Daemon) getLinstor() (*linstor.Client, error) {
+	// Setup the client if it does not exist.
+	if d.linstor == nil {
+		err := d.setupLinstor()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return d.linstor, nil
+}

doc/.wordlist.txt

@@ -64,11 +64,15 @@ DHCPv
 Diátaxis
 Diffie
 Distrobuilder
+diskful
+Diskless
+diskless
 DNS
 dnsmasq
 DNSSEC
 DoS
 DRM
+DRBD
 EB
 Ebit
 eBPF
@@ -104,8 +108,10 @@ GPUs
 Grafana
 HAProxy
 hardcoded
+HDDs
 Hellman
 Homebrew
+hostname
 hotplug
 hotplugged
 hotplugging
@@ -151,6 +157,9 @@ LXCFS
 LXC's
 LXD
 LXD's
+LINBIT
+LINSTOR
+LINSTOR's
 macOS
 macvlan
 Makefile
@@ -178,6 +187,7 @@ NIC
 NICs
 NixOS
 NUMA
+NVMe
 NVRAM
 OCI
 OData
@@ -212,6 +222,7 @@ PKI
 PNG
 Pongo
 POSIX
+PPA
 pre
 preseed
 proxied

doc/api-extensions.md

@@ -2760,3 +2760,7 @@ This introduces a new `io.bus` property for compatible network devices allowing
 ## `disk_io_bus_usb`
 
 Adds a new `usb` value for `io.bus` on `disk` devices.
+
+## `storage_driver_linstor`
+
+This adds a LINSTOR storage driver.

doc/config_options.txt

@@ -2603,6 +2603,41 @@ Specify the volume using the syntax `POOL/VOLUME`.
 Specify the volume using the syntax `POOL/VOLUME`.
 ```
 
+```{config:option} storage.linstor.ca_cert server-miscellaneous
+:scope: "global"
+:shortdesc: "LINSTOR SSL certificate authority"
+:type: "string"
+
+```
+
+```{config:option} storage.linstor.client_cert server-miscellaneous
+:scope: "global"
+:shortdesc: "LINSTOR SSL client certificate"
+:type: "string"
+
+```
+
+```{config:option} storage.linstor.client_key server-miscellaneous
+:scope: "global"
+:shortdesc: "LINSTOR SSL client key"
+:type: "string"
+
+```
+
+```{config:option} storage.linstor.controller_connection server-miscellaneous
+:scope: "global"
+:shortdesc: "LINSTOR controller connection string"
+:type: "string"
+
+```
+
+```{config:option} storage.linstor.satellite.name server-miscellaneous
+:scope: "global"
+:shortdesc: "LINSTOR satellite node name override"
+:type: "string"
+Set this option to the name of the local LINSTOR satellite node, should it be different from the Incus server name.
+```
+
 <!-- config group server-miscellaneous end -->
 <!-- config group server-oidc start -->
 ```{config:option} oidc.audience server-oidc

doc/explanation/storage.md

@@ -24,6 +24,7 @@ The following storage drivers are supported:
 - [Ceph RBD - `ceph`](storage-ceph)
 - [CephFS - `cephfs`](storage-cephfs)
 - [Ceph Object - `cephobject`](storage-cephobject)
+- [LINSTOR - `linstor`](storage-linstor)
 
 See the following how-to guides for additional information:
 
@@ -36,12 +37,12 @@ See the following how-to guides for additional information:
 Where the Incus data is stored depends on the configuration and the selected storage driver.
 Depending on the storage driver that is used, Incus can either share the file system with its host or keep its data separate.
 
-Storage location         | Directory | Btrfs    | LVM (all) | ZFS      | Ceph (all) |
-:---                     | :-:       | :-:      | :-:       | :-:      | :-:        |
-Shared with the host     | ✓  | ✓ | -         | ✓ | -          |
-Dedicated disk/partition | -         | ✓ | ✓  | ✓ | -          |
-Loop disk                | -         | ✓ | ✓  | ✓ | -          |
-Remote storage           | -         | -        | ✓  | -        | ✓   |
+Storage location         | Directory | Btrfs    | LVM (all) | ZFS      | Ceph (all) | LINSTOR  |
+:---                     | :-:       | :-:      | :-:       | :-:      | :-:        | :-:      |
+Shared with the host     | ✓  | ✓ | -         | ✓ | -          | -        |
+Dedicated disk/partition | -         | ✓ | ✓  | ✓ | -          | ✓ |
+Loop disk                | -         | ✓ | ✓  | ✓ | -          | ✓ |
+Remote storage           | -         | -        | ✓  | -        | ✓   | ✓ |
 
 #### Shared with the host
 
@@ -54,7 +55,7 @@ This option is supported for the `dir` driver, the `btrfs` driver (if the host i
 
 Having Incus use an empty partition on your main disk or a full dedicated disk keeps its storage completely independent from the host.
 
-This option is supported  for the `btrfs` driver, the `lvm` driver and the `zfs` driver.
+This option is supported  for the `btrfs` driver, the `lvm` driver, the `zfs` driver and the `linstor` driver.
 
 #### Loop disk
 
@@ -72,6 +73,7 @@ You can increase their size though; see {ref}`storage-resize-pool`.
 
 The `ceph`, `cephfs` and `cephobject` drivers store the data in a completely independent Ceph storage cluster that must be set up separately.
 The `lvmcluster` driver relies on a shared block device being available to all cluster members and on a pre-existing `lvmlockd` setup.
+The `linstor` driver stores the data in a LINSTOR storage cluster that must be setup separately.
 
 (storage-default-pool)=
 ### Default storage pool