chunked: allow conversion without zstd compression

This commit introduces the capability to convert tar layers
to the zstd:chunked format, without performing any zstd compression.

This allows using zstd:chunked without the cost of compression and
decompression.

Closes: https://issues.redhat.com/browse/RUN-3056

Signed-off-by: Giuseppe Scrivano <[email protected]>

Author: giuseppe

pkg/chunked/compression_linux.go

@@ -185,7 +185,10 @@ func openTmpFileNoTmpFile(tmpDir string) (*os.File, error) {
 // Returns (manifest blob, parsed manifest, tar-split file or nil, manifest offset).
 // The opened tar-split file’s position is unspecified.
 // It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert.
-func readZstdChunkedManifest(tmpDir string, blobStream ImageSourceSeekable, tocDigest digest.Digest, annotations map[string]string) (_ []byte, _ *minimal.TOC, _ *os.File, _ int64, retErr error) {
+// The compressed parameter indicates whether the manifest and tar-split data are zstd-compressed
+// (true) or stored uncompressed (false).  Uncompressed data is used only for an optimization to convert
+// a regular OCI layer to zstd:chunked when convert_images is set, and it is not used for distributed images.
+func readZstdChunkedManifest(tmpDir string, blobStream ImageSourceSeekable, tocDigest digest.Digest, annotations map[string]string, compressed bool) (_ []byte, _ *minimal.TOC, _ *os.File, _ int64, retErr error) {
 	offsetMetadata := annotations[minimal.ManifestInfoKey]
 	if offsetMetadata == "" {
 		return nil, nil, nil, 0, fmt.Errorf("%q annotation missing", minimal.ManifestInfoKey)
@@ -261,7 +264,7 @@ func readZstdChunkedManifest(tmpDir string, blobStream ImageSourceSeekable, tocD
 		return nil, nil, nil, 0, err
 	}
 
-	decodedBlob, err := decodeAndValidateBlob(manifest, manifestLengthUncompressed, tocDigest.String())
+	decodedBlob, err := decodeAndValidateBlob(manifest, manifestLengthUncompressed, tocDigest.String(), compressed)
 	if err != nil {
 		return nil, nil, nil, 0, fmt.Errorf("validating and decompressing TOC: %w", err)
 	}
@@ -288,7 +291,7 @@ func readZstdChunkedManifest(tmpDir string, blobStream ImageSourceSeekable, tocD
 				decodedTarSplit.Close()
 			}
 		}()
-		if err := decodeAndValidateBlobToStream(tarSplit, decodedTarSplit, toc.TarSplitDigest.String()); err != nil {
+		if err := decodeAndValidateBlobToStream(tarSplit, decodedTarSplit, toc.TarSplitDigest.String(), compressed); err != nil {
 			return nil, nil, nil, 0, fmt.Errorf("validating and decompressing tar-split: %w", err)
 		}
 		// We use the TOC for creating on-disk files, but the tar-split for creating metadata
@@ -487,11 +490,15 @@ func validateBlob(blob []byte, expectedCompressedChecksum string) error {
 	return nil
 }
 
-func decodeAndValidateBlob(blob []byte, lengthUncompressed uint64, expectedCompressedChecksum string) ([]byte, error) {
+func decodeAndValidateBlob(blob []byte, lengthUncompressed uint64, expectedCompressedChecksum string, compressed bool) ([]byte, error) {
 	if err := validateBlob(blob, expectedCompressedChecksum); err != nil {
 		return nil, err
 	}
 
+	if !compressed {
+		return blob, nil
+	}
+
 	decoder, err := zstd.NewReader(nil)
 	if err != nil {
 		return nil, err
@@ -502,11 +509,16 @@ func decodeAndValidateBlob(blob []byte, lengthUncompressed uint64, expectedCompr
 	return decoder.DecodeAll(blob, b)
 }
 
-func decodeAndValidateBlobToStream(blob []byte, w *os.File, expectedCompressedChecksum string) error {
+func decodeAndValidateBlobToStream(blob []byte, w *os.File, expectedCompressedChecksum string, compressed bool) error {
 	if err := validateBlob(blob, expectedCompressedChecksum); err != nil {
 		return err
 	}
 
+	if !compressed {
+		_, err := w.Write(blob)
+		return err
+	}
+
 	decoder, err := zstd.NewReader(bytes.NewReader(blob))
 	if err != nil {
 		return err

pkg/chunked/compressor/compressor.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/containers/storage/pkg/chunked/internal/minimal"
 	"github.com/containers/storage/pkg/ioutils"
-	"github.com/klauspost/compress/zstd"
 	"github.com/opencontainers/go-digest"
 	"github.com/vbatts/tar-split/archive/tar"
 	"github.com/vbatts/tar-split/tar/asm"
@@ -202,7 +201,7 @@ type tarSplitData struct {
 	compressed          *bytes.Buffer
 	digester            digest.Digester
 	uncompressedCounter *ioutils.WriteCounter
-	zstd                *zstd.Encoder
+	zstd                minimal.ZstdWriter
 	packer              storage.Packer
 }
 

pkg/chunked/storage_linux.go

@@ -170,8 +170,7 @@ func (c *chunkedDiffer) convertTarToZstdChunked(destDirectory string, payload *o
 	}
 
 	newAnnotations := make(map[string]string)
-	level := 1
-	chunked, err := compressor.ZstdCompressor(f, newAnnotations, &level)
+	chunked, err := compressor.NoCompression(f, newAnnotations)
 	if err != nil {
 		f.Close()
 		return 0, nil, "", nil, err
@@ -341,7 +340,7 @@ func makeConvertFromRawDiffer(store storage.Store, blobDigest digest.Digest, blo
 // makeZstdChunkedDiffer sets up a chunkedDiffer for a zstd:chunked layer.
 // It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert.
 func makeZstdChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, annotations map[string]string, iss ImageSourceSeekable, pullOptions pullOptions) (_ *chunkedDiffer, retErr error) {
-	manifest, toc, tarSplit, tocOffset, err := readZstdChunkedManifest(store.RunRoot(), iss, tocDigest, annotations)
+	manifest, toc, tarSplit, tocOffset, err := readZstdChunkedManifest(store.RunRoot(), iss, tocDigest, annotations, true)
 	if err != nil { // May be ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert
 		return nil, fmt.Errorf("read zstd:chunked manifest: %w", err)
 	}
@@ -1448,7 +1447,9 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 		if err != nil {
 			return graphdriver.DriverWithDifferOutput{}, err
 		}
+
 		c.uncompressedTarSize = tarSize
+
 		// fileSource is a O_TMPFILE file descriptor, so we
 		// need to keep it open until the entire file is processed.
 		defer fileSource.Close()
@@ -1464,7 +1465,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 		if tocDigest == nil {
 			return graphdriver.DriverWithDifferOutput{}, fmt.Errorf("internal error: just-created zstd:chunked missing TOC digest")
 		}
-		manifest, toc, tarSplit, tocOffset, err := readZstdChunkedManifest(dest, fileSource, *tocDigest, annotations)
+		manifest, toc, tarSplit, tocOffset, err := readZstdChunkedManifest(dest, fileSource, *tocDigest, annotations, false)
 		if err != nil {
 			return graphdriver.DriverWithDifferOutput{}, fmt.Errorf("read zstd:chunked manifest: %w", err)
 		}
@@ -1473,7 +1474,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 		stream = fileSource
 
 		// fill the chunkedDiffer with the data we just read.
-		c.fileType = fileTypeZstdChunked
+		c.fileType = fileTypeNoCompression
 		c.manifest = manifest
 		c.toc = toc
 		c.tarSplit = tarSplit

pkg/chunked/zstdchunked_test.go

@@ -184,7 +184,7 @@ func TestGenerateAndParseManifest(t *testing.T) {
 	tocDigest, err := toc.GetTOCDigest(annotations)
 	require.NoError(t, err)
 	require.NotNil(t, tocDigest)
-	manifest, decodedTOC, _, _, err := readZstdChunkedManifest(t.TempDir(), s, *tocDigest, annotations)
+	manifest, decodedTOC, _, _, err := readZstdChunkedManifest(t.TempDir(), s, *tocDigest, annotations, true)
 	require.NoError(t, err)
 
 	var toc minimal.TOC