Prototype temporary directory

Signed-off-by: Jan Rodák <[email protected]>

Author: Honny1

drivers/aufs/aufs.go

@@ -45,6 +45,7 @@ import (
 	mountpk "github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/containers/storage/pkg/system"
+	"github.com/containers/storage/pkg/tempdir"
 	"github.com/containers/storage/pkg/unshare"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/sirupsen/logrus"
@@ -781,3 +782,9 @@ func (a *Driver) SupportsShifting() bool {
 func (a *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) {
 	return graphdriver.DedupResult{}, nil
 }
+
+// DeferredRemove is not implemented.
+// It calls Remove directly.
+func (a *Driver) DeferredRemove(id string) (tempdir.CleanupTempDirFunc, error) {
+	return nil, a.Remove(id)
+}

drivers/btrfs/btrfs.go

@@ -36,6 +36,7 @@ import (
 	"github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/containers/storage/pkg/system"
+	"github.com/containers/storage/pkg/tempdir"
 	"github.com/docker/go-units"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/sirupsen/logrus"
@@ -678,3 +679,9 @@ func (d *Driver) AdditionalImageStores() []string {
 func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) {
 	return graphdriver.DedupResult{}, nil
 }
+
+// DeferredRemove is not implemented.
+// It calls Remove directly.
+func (d *Driver) DeferredRemove(id string) (tempdir.CleanupTempDirFunc, error) {
+	return nil, d.Remove(id)
+}

drivers/driver.go

@@ -13,6 +13,7 @@ import (
 	"github.com/containers/storage/pkg/directory"
 	"github.com/containers/storage/pkg/fileutils"
 	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/tempdir"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/sirupsen/logrus"
 	"github.com/vbatts/tar-split/tar/storage"
@@ -124,6 +125,7 @@ type ProtoDriver interface {
 	CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *CreateOpts, readWrite bool) error
 	// Remove attempts to remove the filesystem layer with this id.
 	Remove(id string) error
+	DeferredRemove(id string) (tempdir.CleanupTempDirFunc, error)
 	// Get returns the mountpoint for the layered filesystem referred
 	// to by this id. You can optionally specify a mountLabel or "".
 	// Optionally it gets the mappings used to create the layer.

drivers/overlay/overlay.go

@@ -34,6 +34,7 @@ import (
 	"github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/containers/storage/pkg/system"
+	"github.com/containers/storage/pkg/tempdir"
 	"github.com/containers/storage/pkg/unshare"
 	units "github.com/docker/go-units"
 	digest "github.com/opencontainers/go-digest"
@@ -1314,15 +1315,16 @@ func (d *Driver) Remove(id string) error {
 	lid, err := os.ReadFile(path.Join(dir, "link"))
 	if err == nil {
 		if err := os.RemoveAll(path.Join(d.home, linkDir, string(lid))); err != nil {
-			logrus.Debugf("Failed to remove link: %v", err)
+			logrus.Debugf("Failed to Add to stage Directory link: %v", err)
 		}
 	}
 
 	d.releaseAdditionalLayerByID(id)
 
-	if err := system.EnsureRemoveAll(dir); err != nil && !os.IsNotExist(err) {
-		return err
+	if err := system.EnsureRemoveAll(dir); err != nil {
+		return fmt.Errorf("failed to add to stage directory: %w", err)
 	}
+
 	if d.quotaCtl != nil {
 		d.quotaCtl.ClearQuota(dir)
 		if d.imageStore != "" {
@@ -1332,6 +1334,35 @@ func (d *Driver) Remove(id string) error {
 	return nil
 }
 
+func (d *Driver) DeferredRemove(id string) (tempdir.CleanupTempDirFunc, error) {
+	t, err := tempdir.NewTempDir(filepath.Join(d.homeDirForImageStore(), stagingDir))
+	if err != nil {
+		return nil, err
+	}
+
+	dir := d.dir(id)
+	lid, err := os.ReadFile(path.Join(dir, "link"))
+	if err == nil {
+		if err := t.Add(path.Join(d.home, linkDir, string(lid))); err != nil {
+			logrus.Debugf("Failed to Add to stage Directory link: %v", err)
+		}
+	}
+
+	d.releaseAdditionalLayerByID(id)
+
+	if err := t.Add(dir); err != nil {
+		return t.Cleanup, fmt.Errorf("failed to add to stage directory: %w", err)
+	}
+
+	if d.quotaCtl != nil {
+		d.quotaCtl.ClearQuota(dir)
+		if d.imageStore != "" {
+			d.quotaCtl.ClearQuota(d.imageStore)
+		}
+	}
+	return t.Cleanup, nil
+}
+
 // recreateSymlinks goes through the driver's home directory and checks if the diff directory
 // under each layer has a symlink created for it under the linkDir. If the symlink does not
 // exist, it creates them
@@ -1358,8 +1389,8 @@ func (d *Driver) recreateSymlinks() error {
 		// Check that for each layer, there's a link in "l" with the name in
 		// the layer's "link" file that points to the layer's "diff" directory.
 		for _, dir := range dirs {
-			// Skip over the linkDir and anything that is not a directory
-			if dir.Name() == linkDir || !dir.IsDir() {
+			// Skip over the linkDir and anything that is not a directory or tempDir
+			if dir.Name() == linkDir || !dir.IsDir() || dir.Name() == stagingDir { // Note for review: Not sure if skipping stagingDir is correct
 				continue
 			}
 			// Read the "link" file under each layer to get the name of the symlink

drivers/vfs/driver.go

@@ -17,6 +17,7 @@ import (
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/containers/storage/pkg/system"
+	"github.com/containers/storage/pkg/tempdir"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/sirupsen/logrus"
 	"github.com/vbatts/tar-split/tar/storage"
@@ -241,7 +242,18 @@ func (d *Driver) dir(id string) string {
 
 // Remove deletes the content from the directory for a given id.
 func (d *Driver) Remove(id string) error {
-	return system.EnsureRemoveAll(d.dir(id))
+	return os.RemoveAll(d.dir(id))
+}
+
+func (d *Driver) DeferredRemove(id string) (tempdir.CleanupTempDirFunc, error) {
+	t, err := tempdir.NewTempDir(d.home)
+	if err != nil {
+		return nil, err
+	}
+	if err := t.Add(d.dir(id)); err != nil {
+		return nil, err
+	}
+	return t.Cleanup, nil
 }
 
 // Get returns the directory for the given id.

drivers/windows/windows.go

@@ -32,6 +32,7 @@ import (
 	"github.com/containers/storage/pkg/longpath"
 	"github.com/containers/storage/pkg/reexec"
 	"github.com/containers/storage/pkg/system"
+	"github.com/containers/storage/pkg/tempdir"
 	units "github.com/docker/go-units"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows"
@@ -1014,3 +1015,9 @@ func parseStorageOpt(storageOpt map[string]string) (*storageOptions, error) {
 	}
 	return &options, nil
 }
+
+// DeferredRemove is not implemented.
+// It calls Remove directly.
+func (d *Driver) DeferredRemove(id string) (tempdir.CleanupTempDirFunc, error) {
+	return nil, d.Remove(id)
+}

drivers/zfs/zfs.go

@@ -17,6 +17,7 @@ import (
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/parsers"
+	"github.com/containers/storage/pkg/tempdir"
 	zfs "github.com/mistifyio/go-zfs/v3"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/sirupsen/logrus"
@@ -406,6 +407,12 @@ func (d *Driver) Remove(id string) error {
 	return nil
 }
 
+// DeferredRemove is not implemented.
+// It calls Remove directly.
+func (d *Driver) DeferredRemove(id string) (tempdir.CleanupTempDirFunc, error) {
+	return nil, d.Remove(id)
+}
+
 // Get returns the mountpoint for the given id after creating the target directories if necessary.
 func (d *Driver) Get(id string, options graphdriver.MountOpts) (_ string, retErr error) {
 	mountpoint := d.mountPath(id)

layers.go

@@ -26,6 +26,7 @@ import (
 	"github.com/containers/storage/pkg/stringid"
 	"github.com/containers/storage/pkg/system"
 	"github.com/containers/storage/pkg/tarlog"
+	"github.com/containers/storage/pkg/tempdir"
 	"github.com/containers/storage/pkg/truncindex"
 	"github.com/klauspost/pgzip"
 	digest "github.com/opencontainers/go-digest"
@@ -293,6 +294,9 @@ type rwLayerStore interface {
 	// Delete deletes a layer with the specified name or ID.
 	Delete(id string) error
 
+	// Delete deletes a layer with the specified name or ID.
+	DeferredDelete(id string) (tempdir.CleanupTempDirFunc, error)
+
 	// Wipe deletes all layers.
 	Wipe() error
 
@@ -411,11 +415,12 @@ func newMultipleLockFile(l ...*lockfile.LockFile) *multipleLockFile {
 type layerStore struct {
 	// The following fields are only set when constructing layerStore, and must never be modified afterwards.
 	// They are safe to access without any other locking.
-	lockfile       *multipleLockFile  // lockfile.IsReadWrite can be used to distinguish between read-write and read-only layer stores.
-	mountsLockfile *lockfile.LockFile // Can _only_ be obtained with inProcessLock held.
-	rundir         string
-	jsonPath       [numLayerLocationIndex]string
-	layerdir       string
+	lockfile        *multipleLockFile  // lockfile.IsReadWrite can be used to distinguish between read-write and read-only layer stores.
+	mountsLockfile  *lockfile.LockFile // Can _only_ be obtained with inProcessLock held.
+	rundir          string
+	jsonPath        [numLayerLocationIndex]string
+	layerdir        string
+	tempDirRootPath string
 
 	inProcessLock sync.RWMutex // Can _only_ be obtained with lockfile held.
 	// The following fields can only be read/written with read/write ownership of inProcessLock, respectively.
@@ -794,6 +799,12 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) {
 	layers := []*Layer{}
 	ids := make(map[string]*Layer)
 
+	if r.lockfile.IsReadWrite() {
+		if err := tempdir.RecoverStaleDirs(r.tempDirRootPath); err != nil {
+			return false, err
+		}
+	}
+
 	for locationIndex := range numLayerLocationIndex {
 		location := layerLocationFromIndex(locationIndex)
 		rpath := r.jsonPath[locationIndex]
@@ -935,7 +946,12 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) {
 		// Now actually delete the layers
 		for _, layer := range layersToDelete {
 			logrus.Warnf("Found incomplete layer %q, deleting it", layer.ID)
-			err := r.deleteInternal(layer.ID)
+			cleanupFunc, err := r.deleteInternal(layer.ID)
+			defer func() {
+				if err := cleanupFunc(); err != nil {
+					logrus.Errorf("Error cleaning up temporary directories: %v", err)
+				}
+			}()
 			if err != nil {
 				// Don't return the error immediately, because deleteInternal does not saveLayers();
 				// Even if deleting one incomplete layer fails, call saveLayers() so that other possible successfully
@@ -1158,7 +1174,8 @@ func (s *store) newLayerStore(rundir, layerdir, imagedir string, driver drivers.
 			layersImageDir,
 			filepath.Join(volatileDir, "volatile-layers.json"),
 		},
-		layerdir: layerdir,
+		layerdir:        layerdir,
+		tempDirRootPath: s.graphRoot,
 
 		byid:    make(map[string]*Layer),
 		byname:  make(map[string]*Layer),
@@ -1920,13 +1937,15 @@ func layerHasIncompleteFlag(layer *Layer) bool {
 }
 
 // Requires startWriting.
-func (r *layerStore) deleteInternal(id string) error {
+// Caller MUST run CleanupTemporaryDirectories() after this. Ideally outside of
+// the startWriting.
+func (r *layerStore) deleteInternal(id string) (tempdir.CleanupTempDirFunc, error) {
 	if !r.lockfile.IsReadWrite() {
-		return fmt.Errorf("not allowed to delete layers at %q: %w", r.layerdir, ErrStoreIsReadOnly)
+		return nil, fmt.Errorf("not allowed to delete layers at %q: %w", r.layerdir, ErrStoreIsReadOnly)
 	}
 	layer, ok := r.lookup(id)
 	if !ok {
-		return ErrLayerUnknown
+		return nil, ErrLayerUnknown
 	}
 	// Ensure that if we are interrupted, the layer will be cleaned up.
 	if !layerHasIncompleteFlag(layer) {
@@ -1935,16 +1954,33 @@ func (r *layerStore) deleteInternal(id string) error {
 		}
 		layer.Flags[incompleteFlag] = true
 		if err := r.saveFor(layer); err != nil {
-			return err
+			return nil, err
 		}
 	}
+	tempDirectory, err := tempdir.NewTempDir(r.tempDirRootPath)
+	if err != nil {
+		return nil, err
+	}
 	// We never unset incompleteFlag; below, we remove the entire object from r.layers.
 	id = layer.ID
-	if err := r.driver.Remove(id); err != nil && !errors.Is(err, os.ErrNotExist) {
-		return err
+	var cleanFunc tempdir.CleanupTempDirFunc
+	if cleanFunc, err = r.driver.DeferredRemove(id); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return cleanFunc, err
+	}
+
+	finalCleanFunc := func() error {
+		if err := tempDirectory.Cleanup(); err != nil {
+			return err
+		}
+		if err := cleanFunc(); err != nil {
+			return err
+		}
+		return nil
 	}
-	os.Remove(r.tspath(id))
-	os.RemoveAll(r.datadir(id))
+	_ = tempDirectory.Add(r.tspath(id))
+	// os.Remove(r.tspath(id))
+	_ = tempDirectory.Add(r.datadir(id))
+	// os.RemoveAll(r.datadir(id))
 	delete(r.byid, id)
 	for _, name := range layer.Names {
 		delete(r.byname, name)
@@ -1968,7 +2004,7 @@ func (r *layerStore) deleteInternal(id string) error {
 	}) {
 		selinux.ReleaseLabel(mountLabel)
 	}
-	return nil
+	return finalCleanFunc, nil
 }
 
 // Requires startWriting.
@@ -2006,12 +2042,42 @@ func (r *layerStore) Delete(id string) error {
 			return err
 		}
 	}
-	if err := r.deleteInternal(id); err != nil {
-		return err
+	cleanFunc, err := r.deleteInternal(id)
+	rmErr := cleanFunc()
+	if err != nil || rmErr != nil {
+		return errors.Join(err, rmErr)
 	}
 	return r.saveFor(layer)
 }
 
+// Requires startWriting.
+// Caller MUST run CleanupTemporaryDirectories() after this. Ideally outside of
+// the startWriting.
+func (r *layerStore) DeferredDelete(id string) (tempdir.CleanupTempDirFunc, error) {
+	layer, ok := r.lookup(id)
+	if !ok {
+		return nil, ErrLayerUnknown
+	}
+	id = layer.ID
+	// The layer may already have been explicitly unmounted, but if not, we
+	// should try to clean that up before we start deleting anything at the
+	// driver level.
+	for {
+		_, err := r.unmount(id, false, true)
+		if err == ErrLayerNotMounted {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+	cleanFunc, err := r.deleteInternal(id)
+	if err != nil {
+		return cleanFunc, err
+	}
+	return cleanFunc, r.saveFor(layer)
+}
+
 // Requires startReading or startWriting.
 func (r *layerStore) Exists(id string) bool {
 	_, ok := r.lookup(id)