new filekv storage backend and related changes (#177)

Author: jessepeterson

cli/cli.go

@@ -8,7 +8,9 @@ import (
 
 	"github.com/micromdm/nanomdm/storage"
 	"github.com/micromdm/nanomdm/storage/allmulti"
+	"github.com/micromdm/nanomdm/storage/diskv"
 	"github.com/micromdm/nanomdm/storage/file"
+	"github.com/micromdm/nanomdm/storage/inmem"
 	"github.com/micromdm/nanomdm/storage/mysql"
 	"github.com/micromdm/nanomdm/storage/pgsql"
 
@@ -17,6 +19,11 @@ import (
 	"github.com/micromdm/nanolib/log"
 )
 
+var (
+	ErrNoStorageOptions = errors.New("storage backend does not support options, please specify no (or empty) options")
+	ErrMissingDSN       = errors.New("missing required DSN")
+)
+
 type StringAccumulator []string
 
 func (s *StringAccumulator) String() string {
@@ -47,8 +54,8 @@ func (s *Storage) Parse(logger log.Logger) (storage.AllStorage, error) {
 	}
 	// default storage and DSN pair
 	if len(s.Storage) < 1 {
-		s.Storage = append(s.Storage, "file")
-		s.DSN = append(s.DSN, "db")
+		s.Storage = append(s.Storage, "filekv")
+		s.DSN = append(s.DSN, "dbkv")
 	}
 	var mdmStorage []storage.AllStorage
 	for idx, storage := range s.Storage {
@@ -63,7 +70,13 @@ func (s *Storage) Parse(logger log.Logger) (storage.AllStorage, error) {
 		)
 		switch storage {
 		case "file":
-			fileStorage, err := fileStorageConfig(dsn, options)
+			if options != "enable_deprecated=1" {
+				return nil, errors.New("file backend is deprecated; specify storage options to force enable")
+			}
+			if dsn == "" {
+				return nil, ErrMissingDSN
+			}
+			fileStorage, err := file.New(dsn)
 			if err != nil {
 				return nil, err
 			}
@@ -80,6 +93,19 @@ func (s *Storage) Parse(logger log.Logger) (storage.AllStorage, error) {
 				return nil, err
 			}
 			mdmStorage = append(mdmStorage, pgsqlStorage)
+		case "inmem":
+			if options != "" {
+				return nil, ErrNoStorageOptions
+			}
+			mdmStorage = append(mdmStorage, inmem.New())
+		case "filekv":
+			if dsn == "" {
+				return nil, ErrMissingDSN
+			}
+			if options != "" {
+				return nil, ErrNoStorageOptions
+			}
+			mdmStorage = append(mdmStorage, diskv.New(dsn))
 		default:
 			return nil, fmt.Errorf("unknown storage: %s", storage)
 		}
@@ -97,15 +123,6 @@ func (s *Storage) Parse(logger log.Logger) (storage.AllStorage, error) {
 	), nil
 }
 
-var NoStorageOptions = errors.New("storage backend does not support options, please specify no (or empty) options")
-
-func fileStorageConfig(dsn, options string) (*file.FileStorage, error) {
-	if options != "" {
-		return nil, NoStorageOptions
-	}
-	return file.New(dsn)
-}
-
 func mysqlStorageConfig(dsn, options string, logger log.Logger) (*mysql.MySQLStorage, error) {
 	logger = logger.With("storage", "mysql")
 	opts := []mysql.Option{

cmd/nano2nano/main.go

@@ -82,6 +82,14 @@ func main() {
 					logger.Info("msg", "sending to migration endpoint", "err", err)
 				}
 			}
+		case *mdm.SetBootstrapToken:
+			logger.Info(logsFromEnrollment("SetBootstrapToken", &v.Enrollment)...)
+			if !skipServer {
+				if err := httpPut(client, *flURL, *flAPIKey, v.Raw); err != nil {
+					fmt.Println(string(v.Raw))
+					logger.Info("msg", "sending to migration endpoint", "err", err)
+				}
+			}
 		case error:
 			logger.Info("msg", "receiving checkin", "err", v)
 		default:

docs/operations-guide.md

@@ -65,17 +65,34 @@ Enable additional debug logging.
 
 ### -storage, -storage-dsn, & -storage-options
 
-The `-storage`, `-storage-dsn`, & `-storage-options` flags together configure the storage backend(s). `-storage` specifies the name of the backend while `-storage-dsn` specifies the backend data source name (e.g. the connection string). The optional `-storage-options` flag specifies options for the backend if it supports them. If no storage flags are supplied then it is as if you specified `-storage file -storage-dsn db` meaning we use the `file` storage backend with `db` as its DSN.
+The `-storage`, `-storage-dsn`, & `-storage-options` flags together configure the storage backend(s). `-storage` specifies the name of the backend while `-storage-dsn` specifies the backend data source name (e.g. the connection string). The optional `-storage-options` flag specifies options for the backend if it supports them. If no storage flags are supplied then it is as if you specified `-storage filekv -storage-dsn dbkv` meaning we use the `filekv` storage backend with `dbkv` as its DSN.
 
-_Note:_ NanoMDM versions v0.5.0 and below used the `-dsn` flag while later versions switched to the `-storage-dsn` flag.
+> [!NOTE]
+> NanoMDM **versions v0.5 and below** used the `-dsn` flag while later versions now use the `-storage-dsn` flag.
+
+##### filekv storage backend
+
+* `-storage filekv`
+
+Configures the `filekv` storage backend. This manages enrollment and command queue data within plain filesystem files and directories using a key-value storage system. It has zero dependencies, no options, and should run out of the box. The `-storage-dsn` flag specifies the filesystem directory for the database. If no DSN is specified then `dbkv` is used as a default.
+
+*Example:* `-storage filekv -storage-dsn /path/to/my/db`
 
 #### file storage backend
 
 * `-storage file`
 
-Configures the `file` storage backend. This manages enrollment and command data within plain filesystem files and directories. It has zero dependencies and should run out of the box. The `-storage-dsn` flag specifies the filesystem directory for the database. The `file` backend has no storage options.
+> [!WARNING]
+> The `file` storage backend is deprecated in NanoMDM **versions after v0.7** and will be removed in a future release.
+
+Configures the `file` storage backend. This manages enrollment and command data within plain filesystem files and directories. It has zero dependencies and should run out of the box. The `-storage-dsn` flag specifies the filesystem directory for the database.
+
+Options are specified as a comma-separated list of "key=value" pairs. Supported options:
 
-*Example:* `-storage file -storage-dsn /path/to/my/db`
+* `enable_deprecated=1`
+  * This option enables the file backend. Without this switch the `file` backend is disabled.
+
+*Example:* `-storage file -storage-dsn /path/to/my/db -storage-options enable_deprecated=1`
 
 #### mysql storage backend
 
@@ -106,15 +123,26 @@ Options are specified as a comma-separated list of "key=value" pairs. The pgsql
 
 *Example:* `-storage pgsql -storage-dsn postgres://postgres:toor@localhost/nanomdm -storage-options delete=1`
 
+#### in-memory storage backend
+
+* `-storage inmem`
+
+Configure the `inmem` in-memory storage backend. This manages enrollment and command queue data entirely in *volatile* memeory. There are no options and the DSN is ignored.
+
+> [!CAUTION]
+> All data is lost when the server process exits when using the in-memory storage backend.
+
+*Example:* `-storage inmem`
+
 #### multi-storage backend
 
-You can configure multiple storage backends to be used simultaneously. Specifying multiple sets of `-storage`, `-storage-dsn`, & `-storage-options` flags will configure the "multi-storage" adapter. The flags must be specified in sets and are related to each other in the order they're specified: for example the first `-storage` flag corresponds to the first `-storage-dsn` flag and so forth.
+You can configure multiple storage backends to be used simultaneously. Specifying multiple sets of `-storage`, `-storage-dsn`, & `-storage-options` flags will configure the "multi-storage" adapter. The flags must be specified in sets and are related to each other in the order they're specified: for example the first `-storage` flag corresponds to the first `-storage-dsn` flag and so forth. Note that empty options must be specified even if the backend is not using them.
 
 Be aware that only the first storage backend will be "used" when interacting with the system, all other storage backends are called to, but any *results* are discarded. In other words consider them write-only. Also beware that you will have very bizaare results if you change to using multiple storage backends in the midst of existing enrollments. You will receive errors about missing database rows or data. A storage backend needs to be around when a device (or all devices) initially enroll(s). There is no "sync" or backfill system with multiple storage backends (see the migration ability if you need this).
 
-The multi-storage backend is really only useful if you've always been using multiple storage backends or if you're doing some type of development or testing (perhaps creating a new storage backend).
+The multi-storage backend is only really useful if you've always been using multiple storage backends or if you're doing some type of development or testing (perhaps creating a new storage backend).
 
-For example to use both a `file` *and* `mysql` backend your command line might look like: `-storage file -storage-dsn db -storage mysql -storage-dsn nanomdm:nanomdm/mymdmdb`. You can also mix and match backends, or mutliple of the same backend. Behavior is undefined (and probably very bad) if you specify two backends of the same type with the same DSN.
+For example to use both a `filekv` *and* `mysql` backend your command line might look like: `-storage filekv -storage-dsn dbkv -storage mysql -storage-dsn nanomdm:nanomdm/mymdmdb`. You can also mix and match backends, or mutliple of the same backend. Behavior is undefined (and probably very bad) if you specify two backends of the same type with the same DSN (i.e. sharing the same data source).
 
 ### -dump
 

go.mod

@@ -6,14 +6,16 @@ require (
 	github.com/RobotsAndPencils/buford v0.14.0
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/lib/pq v1.10.9
-	github.com/micromdm/nanolib v0.2.0
+	github.com/micromdm/nanolib v0.3.0
 	github.com/micromdm/plist v0.2.1
+	github.com/peterbourgon/diskv/v3 v3.0.1
 	github.com/smallstep/pkcs7 v0.2.1
 	golang.org/x/net v0.34.0
 )
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/google/btree v1.0.0 // indirect
 	golang.org/x/crypto v0.33.0 // indirect
 	golang.org/x/text v0.22.0 // indirect
 )

go.sum

@@ -5,15 +5,17 @@ github.com/RobotsAndPencils/buford v0.14.0/go.mod h1:F5FvdB/nkMby8Pge6HFpPHgLOeU
 github.com/aai/gocrypto v0.0.0-20160205191751-93df0c47f8b8/go.mod h1:nE/FnVUmtbP0EbgMVCUtDrm1+86H47QfJIdcmZb+J1s=
 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/micromdm/nanolib v0.2.0 h1:g5GHQuUpS82WIAB15LyenjF/0/WSUNJMe5XZfCJSXq4=
-github.com/micromdm/nanolib v0.2.0/go.mod h1:FwBKCvvphgYvbdUZ+qw5kay7NHJcg6zPi8W7kXNajmE=
+github.com/micromdm/nanolib v0.3.0 h1:65xIafn1hP4izGTcLhECpNIVAaccBur7t6wvIrOewSs=
+github.com/micromdm/nanolib v0.3.0/go.mod h1:FwBKCvvphgYvbdUZ+qw5kay7NHJcg6zPi8W7kXNajmE=
 github.com/micromdm/plist v0.2.1 h1:4SoSMOVAyzv1ThT8IKLgXLJEKezLkcVDN6wivqTTFdo=
 github.com/micromdm/plist v0.2.1/go.mod h1:flkfm0od6GzyXBqI28h5sgEyi3iPO28W2t1Zm9LpwWs=
+github.com/peterbourgon/diskv/v3 v3.0.1 h1:x06SQA46+PKIUftmEujdwSEpIx8kR+M9eLYsUxeYveU=
 github.com/peterbourgon/diskv/v3 v3.0.1/go.mod h1:kJ5Ny7vLdARGU3WUuy6uzO6T0nb/2gWcT1JiBvRmb5o=
 github.com/smallstep/pkcs7 v0.2.1 h1:6Kfzr/QizdIuB6LSv8y1LJdZ3aPSfTNhTLqAx9CTLfA=
 github.com/smallstep/pkcs7 v0.2.1/go.mod h1:RcXHsMfL+BzH8tRhmrF1NkkpebKpq3JEM66cOFxanf0=

storage/all.go

@@ -0,0 +1,11 @@
+package storage
+
+import "github.com/micromdm/nanomdm/mdm"
+
+type BootstrapTokenStore interface {
+	StoreBootstrapToken(r *mdm.Request, msg *mdm.SetBootstrapToken) error
+
+	// RetrieveBootstrapToken retrieves the previously-escrowed Bootstrap Token.
+	// If a token has not yet been escrowed then a nil token and no error should be returned.
+	RetrieveBootstrapToken(r *mdm.Request, msg *mdm.GetBootstrapToken) (*mdm.BootstrapToken, error)
+}

storage/bstoken.go

@@ -0,0 +1,35 @@
+package storage
+
+import (
+	"context"
+
+	"github.com/micromdm/nanomdm/mdm"
+)
+
+// CertAuthStore stores and retrieves cert-to-enrollment associations.
+// The request enrollment ID should be normalized for just the device channel.
+// The hash parameter, when present, is likely (but not required) to be
+// a 64-charachter hex string representation of a SHA-256 digest.
+type CertAuthStore interface {
+	// HasCertHash checks if hash has ever been associated to any enrollment.
+	HasCertHash(r *mdm.Request, hash string) (has bool, err error)
+
+	// EnrollmentHasCertHash checks that r.ID has any hash associated.
+	// The hash parameter can usually be ignored.
+	EnrollmentHasCertHash(r *mdm.Request, hash string) (bool, error)
+
+	// IsCertHashAssociated checks that r.ID is associated to hash.
+	IsCertHashAssociated(r *mdm.Request, hash string) (bool, error)
+
+	// AssociateCertHash associates r.ID with hash.
+	// Here hash is a cryptographic hash of the request certificate.
+	AssociateCertHash(r *mdm.Request, hash string) error
+}
+
+type CertAuthRetriever interface {
+	// EnrollmentFromHash retrieves a normalized enrollment ID from a cert hash.
+	// The hash parameter, when present, is likely (but not required) to be
+	// a 64-charachter hex string representation of a SHA-256 digest.
+	// Implementations should return an empty string if no result is found.
+	EnrollmentFromHash(ctx context.Context, hash string) (string, error)
+}

storage/certauth.go

@@ -0,0 +1,64 @@
+// Package diskv implements a NanoMDM storage backend using the diskv key-value store.
+package diskv
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/micromdm/nanomdm/storage/kv"
+
+	nlkv "github.com/micromdm/nanolib/storage/kv"
+	"github.com/micromdm/nanolib/storage/kv/kvdiskv"
+	"github.com/micromdm/nanolib/storage/kv/kvtxn"
+	"github.com/peterbourgon/diskv/v3"
+)
+
+// Diskv is a storage backend that uses diskv.
+type Diskv struct {
+	*kv.KV
+}
+
+// Split2X2Transform splits key into a path like /00/01 for a key of "0001".
+// The key will be prefixed with zeros if its length is less than 4.
+func Split2X2Transform(key string) []string {
+	if len(key) < 4 {
+		key = strings.Repeat("0", 4-len(key)) + key
+	}
+	return []string{key[0:2], key[2:4]}
+}
+
+// StripPrefixTransform wraps next in a function that trims prefix from the key.
+func StripPrefixTransform(next diskv.TransformFunction, prefix string) diskv.TransformFunction {
+	return func(key string) []string {
+		return next(strings.TrimPrefix(key, prefix))
+	}
+}
+
+func newBucket(path, name string) nlkv.TxnBucketWithCRUD {
+	return newBucketWithTransform(path, name, Split2X2Transform)
+}
+
+func newBucketWithTransform(path, name string, transform diskv.TransformFunction) nlkv.TxnBucketWithCRUD {
+	return kvtxn.New(kvdiskv.New(diskv.New(diskv.Options{
+		BasePath:     filepath.Join(path, name),
+		Transform:    transform,
+		CacheSizeMax: 1024 * 1024,
+	})))
+}
+
+// New creates a new storage backend that uses diskv.
+func New(path string) *Diskv {
+	return &Diskv{KV: kv.New(
+		newBucket(path, "users"),
+		newBucket(path, "cert_auth"),
+		newBucket(path, "queue"),
+		// try to store the push certs with transformed keys of the UUID within the Topic
+		newBucketWithTransform(
+			path,
+			"push_cert",
+			StripPrefixTransform(Split2X2Transform, "com.apple.mgmt.External."),
+		),
+		newBucket(path, "devices"),
+		newBucket(path, "enrollments"),
+	)}
+}

storage/diskv/diskv.go

@@ -0,0 +1,12 @@
+package diskv
+
+import (
+	"context"
+	"testing"
+
+	"github.com/micromdm/nanomdm/test/e2e"
+)
+
+func TestDiskv(t *testing.T) {
+	t.Run("e2e", func(t *testing.T) { e2e.TestE2E(t, context.Background(), New(t.TempDir())) })
+}

storage/diskv/diskv_test.go

@@ -36,12 +36,11 @@ func (s *FileStorage) RetrieveMigrationCheckins(_ context.Context, c chan<- inte
 			if err != nil {
 				c <- err
 			}
-			// if an Authenticate doesn't exist then this is a
-			// user-channel enrollment. skip it for this loop
-			if !userLoop && !authExists {
+			// an Authenticate should not exist for a user-channel
+			// enrollment. skip it if found.
+			if userLoop && authExists {
 				continue
-			}
-			if !userLoop {
+			} else if !userLoop && authExists {
 				sendCheckinMessage(e, AuthenticateFilename, c)
 			}
 			tokExists, err := e.fileExists(TokenUpdateFilename)

storage/file/migrate.go

@@ -0,0 +1,26 @@
+// Package inmem implements an in-memory NanoMDM storage backend.
+package inmem
+
+import (
+	"github.com/micromdm/nanomdm/storage/kv"
+
+	"github.com/micromdm/nanolib/storage/kv/kvmap"
+	"github.com/micromdm/nanolib/storage/kv/kvtxn"
+)
+
+// InMem is an in-memory storage backend.
+type InMem struct {
+	*kv.KV
+}
+
+// New creates a new in-memory storage backend.
+func New() *InMem {
+	return &InMem{KV: kv.New(
+		kvtxn.New(kvmap.New()),
+		kvtxn.New(kvmap.New()),
+		kvtxn.New(kvmap.New()),
+		kvtxn.New(kvmap.New()),
+		kvtxn.New(kvmap.New()),
+		kvtxn.New(kvmap.New()),
+	)}
+}