history: add support for exporting multiple and all records

Signed-off-by: Tonis Tiigi <[email protected]>

Author: tonistiigi

commands/history/export.go

@@ -14,14 +14,16 @@ import (
 	"github.com/docker/buildx/util/confutil"
 	"github.com/docker/buildx/util/desktop/bundle"
 	"github.com/docker/cli/cli/command"
+	"github.com/moby/buildkit/client"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
 type exportOptions struct {
 	builder string
-	ref     string
+	refs    []string
 	output  string
+	all     bool
 }
 
 func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) error {
@@ -40,40 +42,60 @@ func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) e
 		}
 	}
 
-	recs, err := queryRecords(ctx, opts.ref, nodes, &queryOptions{
-		CompletedOnly: true,
-	})
-	if err != nil {
-		return err
+	if len(opts.refs) == 0 {
+		opts.refs = []string{""}
 	}
 
-	if len(recs) == 0 {
-		if opts.ref == "" {
-			return errors.New("no records found")
+	var res []historyRecord
+	for _, ref := range opts.refs {
+		recs, err := queryRecords(ctx, ref, nodes, &queryOptions{
+			CompletedOnly: true,
+		})
+		if err != nil {
+			return err
 		}
-		return errors.Errorf("no record found for ref %q", opts.ref)
-	}
 
-	if opts.ref == "" {
-		slices.SortFunc(recs, func(a, b historyRecord) int {
-			return b.CreatedAt.AsTime().Compare(a.CreatedAt.AsTime())
-		})
-	}
+		if len(recs) == 0 {
+			if ref == "" {
+				return errors.New("no records found")
+			}
+			return errors.Errorf("no record found for ref %q", ref)
+		}
 
-	recs = recs[:1]
+		if ref == "" {
+			slices.SortFunc(recs, func(a, b historyRecord) int {
+				return b.CreatedAt.AsTime().Compare(a.CreatedAt.AsTime())
+			})
+		}
+
+		if opts.all {
+			res = append(res, recs...)
+			break
+		} else {
+			res = append(res, recs[0])
+		}
+	}
 
 	ls, err := localstate.New(confutil.NewConfig(dockerCli))
 	if err != nil {
 		return err
 	}
 
-	c, err := recs[0].node.Driver.Client(ctx)
-	if err != nil {
-		return err
+	visited := map[*builder.Node]struct{}{}
+	var clients []*client.Client
+	for _, rec := range res {
+		if _, ok := visited[rec.node]; ok {
+			continue
+		}
+		c, err := rec.node.Driver.Client(ctx)
+		if err != nil {
+			return err
+		}
+		clients = append(clients, c)
 	}
 
-	toExport := make([]*bundle.Record, 0, len(recs))
-	for _, rec := range recs {
+	toExport := make([]*bundle.Record, 0, len(res))
+	for _, rec := range res {
 		var defaultPlatform string
 		if p := rec.node.Platforms; len(p) > 0 {
 			defaultPlatform = platforms.FormatAll(platforms.Normalize(p[0]))
@@ -110,7 +132,7 @@ func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) e
 		}
 	}
 
-	return bundle.Export(ctx, c, w, toExport)
+	return bundle.Export(ctx, clients, w, toExport)
 }
 
 func exportCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command {
@@ -119,11 +141,11 @@ func exportCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "export [OPTIONS] [REF]",
 		Short: "Export a build into Docker Desktop bundle",
-		Args:  cobra.MaximumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if len(args) > 0 {
-				options.ref = args[0]
+			if options.all && len(args) > 0 {
+				return errors.New("cannot specify refs when using --all")
 			}
+			options.refs = args
 			options.builder = *rootOpts.Builder
 			return runExport(cmd.Context(), dockerCli, options)
 		},
@@ -132,6 +154,7 @@ func exportCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command {
 
 	flags := cmd.Flags()
 	flags.StringVarP(&options.output, "output", "o", "", "Output file path")
+	flags.BoolVar(&options.all, "all", false, "Export all records for the builder")
 
 	return cmd
 }

docs/reference/buildx_history_export.md

@@ -7,6 +7,7 @@ Export a build into Docker Desktop bundle
 
 | Name             | Type     | Default | Description                              |
 |:-----------------|:---------|:--------|:-----------------------------------------|
+| `--all`          | `bool`   |         | Export all records for the builder       |
 | `--builder`      | `string` |         | Override the configured builder instance |
 | `-D`, `--debug`  | `bool`   |         | Enable debug logging                     |
 | `-o`, `--output` | `string` |         | Output file path                         |

util/desktop/bundle/content.go

@@ -0,0 +1,78 @@
+package bundle
+
+import (
+	"context"
+
+	"github.com/containerd/containerd/v2/core/content"
+	cerrdefs "github.com/containerd/errdefs"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+type nsFallbackStore struct {
+	main content.Store
+	fb   content.Store
+}
+
+var _ content.Store = &nsFallbackStore{}
+
+func (c *nsFallbackStore) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
+	info, err := c.main.Info(ctx, dgst)
+	if err != nil {
+		if cerrdefs.IsNotFound(err) {
+			return c.fb.Info(ctx, dgst)
+		}
+	}
+	return info, err
+}
+
+func (c *nsFallbackStore) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
+	return c.main.Update(ctx, info, fieldpaths...)
+}
+
+func (c *nsFallbackStore) Walk(ctx context.Context, fn content.WalkFunc, filters ...string) error {
+	seen := make(map[digest.Digest]struct{})
+	err := c.main.Walk(ctx, func(i content.Info) error {
+		seen[i.Digest] = struct{}{}
+		return fn(i)
+	}, filters...)
+	if err != nil {
+		return err
+	}
+	return c.fb.Walk(ctx, func(i content.Info) error {
+		if _, ok := seen[i.Digest]; ok {
+			return nil
+		}
+		return fn(i)
+	}, filters...)
+}
+
+func (c *nsFallbackStore) Delete(ctx context.Context, dgst digest.Digest) error {
+	return c.main.Delete(ctx, dgst)
+}
+
+func (c *nsFallbackStore) Status(ctx context.Context, ref string) (content.Status, error) {
+	return c.main.Status(ctx, ref)
+}
+
+func (c *nsFallbackStore) ListStatuses(ctx context.Context, filters ...string) ([]content.Status, error) {
+	return c.main.ListStatuses(ctx, filters...)
+}
+
+func (c *nsFallbackStore) Abort(ctx context.Context, ref string) error {
+	return c.main.Abort(ctx, ref)
+}
+
+func (c *nsFallbackStore) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
+	ra, err := c.main.ReaderAt(ctx, desc)
+	if err != nil {
+		if cerrdefs.IsNotFound(err) {
+			return c.fb.ReaderAt(ctx, desc)
+		}
+	}
+	return ra, err
+}
+
+func (c *nsFallbackStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
+	return c.main.Writer(ctx, opts...)
+}

util/desktop/bundle/export.go

@@ -32,8 +32,23 @@ type Record struct {
 	StateGroup      *localstate.StateGroup `json:"stateGroup,omitempty"`
 }
 
-func Export(ctx context.Context, c *client.Client, w io.Writer, records []*Record) error {
-	store := proxy.NewContentStore(c.ContentClient())
+func Export(ctx context.Context, c []*client.Client, w io.Writer, records []*Record) error {
+	var store content.Store
+	for _, c := range c {
+		s := proxy.NewContentStore(c.ContentClient())
+		if store == nil {
+			store = s
+			break
+		}
+		store = &nsFallbackStore{
+			main: store,
+			fb:   s,
+		}
+	}
+	if store == nil {
+		return errors.New("no buildkit client found")
+	}
+
 	mp := contentutil.NewMultiProvider(store)
 
 	desc, err := export(ctx, mp, records)

util/desktop/bundle/trace.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"regexp"
 	"strings"
+	"sync"
 
 	"github.com/containerd/containerd/v2/core/content"
 	"github.com/docker/buildx/util/otelutil"
@@ -17,15 +18,22 @@ import (
 )
 
 var (
-	sensitiveKeys = []string{"ghtoken", "token", "access_key_id", "secret_access_key", "session_token"}
-	reAttrs       = regexp.MustCompile(`(?i)(` + strings.Join(sensitiveKeys, "|") + `)=[^ ,]+`)
-	reGhs         = regexp.MustCompile(`ghs_[A-Za-z0-9]{36}`)
+	reOnce                  sync.Once
+	reAttrs, reGhs, reGhpat *regexp.Regexp
 )
 
 func sanitizeCommand(value string) string {
+	reOnce.Do(func() {
+		sensitiveKeys := []string{"ghtoken", "token", "access_key_id", "secret_access_key", "session_token"}
+		reAttrs = regexp.MustCompile(`(?i)(` + strings.Join(sensitiveKeys, "|") + `)=[^ ,]+`)
+		reGhs = regexp.MustCompile(`(?:ghu|ghs)_[A-Za-z0-9]{36}`)
+		reGhpat = regexp.MustCompile(`github_pat_\w{82}`)
+	})
+
 	value = reAttrs.ReplaceAllString(value, "${1}=xxxxx")
-	// reGhs is just double proofing. Not really needed.
+	// reGhs/reGhpat is just double proofing. Not really needed.
 	value = reGhs.ReplaceAllString(value, "xxxxx")
+	value = reGhpat.ReplaceAllString(value, "xxxxx")
 	return value
 }