container driver: copy ca and user tls registries certs

Signed-off-by: CrazyMax <[email protected]>

Author: crazy-max

docs/reference/buildx_create.md

@@ -84,6 +84,11 @@ Specifies the configuration file for the buildkitd daemon to use. The configurat
 can be overridden by [`--buildkitd-flags`](#buildkitd-flags).
 See an [example buildkitd configuration file](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md).
 
+Note that if you create a `docker-container` builder and have specified
+certificates for registries in the `buildkitd.toml` configuration, the files
+will be copied into the container under `/etc/buildkit/certs` and configuration
+will be updated to reflect that.
+
 ### <a name="driver"></a> Set the builder driver to use (--driver)
 
 ```

driver/docker-container/driver.go

@@ -1,17 +1,18 @@
 package docker
 
 import (
-	"archive/tar"
 	"bytes"
 	"context"
 	"io"
 	"io/ioutil"
 	"net"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/docker/buildx/driver"
 	"github.com/docker/buildx/driver/bkimage"
+	"github.com/docker/buildx/util/confutil"
 	"github.com/docker/buildx/util/imagetools"
 	"github.com/docker/buildx/util/progress"
 	"github.com/docker/docker/api/types"
@@ -20,6 +21,7 @@ import (
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/network"
 	dockerclient "github.com/docker/docker/client"
+	dockerarchive "github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/util/tracing/detect"
@@ -28,12 +30,6 @@ import (
 
 const (
 	volumeStateSuffix = "_state"
-
-	// containerStateDir is the location where buildkitd inside the container
-	// stores its state. The container driver creates a Linux container, so
-	// this should match the location for Linux, as defined in:
-	// https://github.com/moby/buildkit/blob/v0.9.0/util/appdefaults/appdefaults_unix.go#L11-L15
-	containerBuildKitRootDir = "/var/lib/buildkit"
 )
 
 type Driver struct {
@@ -119,7 +115,7 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 				{
 					Type:   mount.TypeVolume,
 					Source: d.Name + volumeStateSuffix,
-					Target: containerBuildKitRootDir,
+					Target: confutil.DefaultBuildKitStateDir,
 				},
 			},
 		}
@@ -139,11 +135,12 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 			return err
 		}
 		if f := d.InitConfig.ConfigFile; f != "" {
-			buf, err := readFileToTar(f)
+			configFiles, err := confutil.LoadConfigFiles(f)
 			if err != nil {
 				return err
 			}
-			if err := d.DockerAPI.CopyToContainer(ctx, d.Name, "/", buf, dockertypes.CopyToContainerOptions{}); err != nil {
+			defer os.RemoveAll(configFiles)
+			if err := d.copyToContainer(ctx, configFiles+"/.", "/"); err != nil {
 				return err
 			}
 		}
@@ -205,6 +202,40 @@ func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error {
 	return rc.Close()
 }
 
+// copyToContainer is based on the implementation from docker/cli
+// https://github.com/docker/cli/blob/master/cli/command/container/cp.go
+func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir string) error {
+	srcPath, err := resolveLocalPath(srcPath)
+	if err != nil {
+		return err
+	}
+	srcStat, err := os.Lstat(srcPath)
+	if err != nil {
+		return err
+	}
+	srcInfo := dockerarchive.CopyInfo{
+		Path:   srcPath,
+		Exists: true,
+		IsDir:  srcStat.IsDir(),
+	}
+	srcArchive, err := dockerarchive.TarResource(srcInfo)
+	if err != nil {
+		return err
+	}
+	defer srcArchive.Close()
+
+	return d.DockerAPI.CopyToContainer(ctx, d.Name, dstDir, srcArchive, dockertypes.CopyToContainerOptions{
+		AllowOverwriteDirWithFile: false,
+	})
+}
+
+func resolveLocalPath(localPath string) (absPath string, err error) {
+	if absPath, err = filepath.Abs(localPath); err != nil {
+		return
+	}
+	return dockerarchive.PreserveTrailingDotOrSeparator(absPath, localPath, filepath.Separator), nil
+}
+
 func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) {
 	execConfig := types.ExecConfig{
 		Cmd:          cmd,
@@ -366,29 +397,6 @@ func (d *demux) Read(dt []byte) (int, error) {
 	return d.Reader.Read(dt)
 }
 
-func readFileToTar(fn string) (*bytes.Buffer, error) {
-	buf := bytes.NewBuffer(nil)
-	tw := tar.NewWriter(buf)
-	dt, err := ioutil.ReadFile(fn)
-	if err != nil {
-		return nil, err
-	}
-	if err := tw.WriteHeader(&tar.Header{
-		Name: "/etc/buildkit/buildkitd.toml",
-		Size: int64(len(dt)),
-		Mode: 0644,
-	}); err != nil {
-		return nil, err
-	}
-	if _, err := tw.Write(dt); err != nil {
-		return nil, err
-	}
-	if err := tw.Close(); err != nil {
-		return nil, err
-	}
-	return buf, nil
-}
-
 type logWriter struct {
 	logger progress.SubLogger
 	stream int

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/moby/buildkit v0.9.1-0.20211019185819-8778943ac3da
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
+	github.com/pelletier/go-toml v1.9.4
 	github.com/pkg/errors v0.9.1
 	github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002
 	github.com/sirupsen/logrus v1.8.1

util/confutil/config.go

@@ -0,0 +1,25 @@
+package confutil
+
+import (
+	"os"
+
+	"github.com/pelletier/go-toml"
+	"github.com/pkg/errors"
+)
+
+// loadConfigTree loads BuildKit config toml tree
+func loadConfigTree(fp string) (*toml.Tree, error) {
+	f, err := os.Open(fp)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, errors.Wrapf(err, "failed to load config from %s", fp)
+	}
+	defer f.Close()
+	t, err := toml.LoadReader(f)
+	if err != nil {
+		return t, errors.Wrap(err, "failed to parse config")
+	}
+	return t, nil
+}

util/confutil/container.go

@@ -0,0 +1,144 @@
+package confutil
+
+import (
+	"io"
+	"os"
+	"path"
+
+	"github.com/pelletier/go-toml"
+	"github.com/pkg/errors"
+)
+
+const (
+	// DefaultBuildKitStateDir and DefaultBuildKitConfigDir are the location
+	// where buildkitd inside the container stores its state. Some drivers
+	// create a Linux container, so this should match the location for Linux,
+	// as defined in: https://github.com/moby/buildkit/blob/v0.9.0/util/appdefaults/appdefaults_unix.go#L11-L15
+	DefaultBuildKitStateDir  = "/var/lib/buildkit"
+	DefaultBuildKitConfigDir = "/etc/buildkit"
+)
+
+// LoadConfigFiles creates a temp directory with BuildKit config and
+// registry certificates ready to be copied to a container.
+func LoadConfigFiles(bkconfig string) (string, error) {
+	if _, err := os.Stat(bkconfig); errors.Is(err, os.ErrNotExist) {
+		return "", errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig)
+	} else if err != nil {
+		return "", errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig)
+	}
+
+	// Load config tree
+	btoml, err := loadConfigTree(bkconfig)
+	if err != nil {
+		return "", err
+	}
+
+	// Temp dir that will be copied to the container
+	tmpDir, err := os.MkdirTemp("", "buildkitd-config")
+	if err != nil {
+		return "", err
+	}
+
+	// Create BuildKit config folders
+	tmpBuildKitConfigDir := path.Join(tmpDir, DefaultBuildKitConfigDir)
+	tmpBuildKitCertsDir := path.Join(tmpBuildKitConfigDir, "certs")
+	if err := os.MkdirAll(tmpBuildKitCertsDir, 0700); err != nil {
+		return "", err
+	}
+
+	// Iterate through registry config to copy certs and update
+	// BuildKit config with the underlying certs' path in the container.
+	//
+	// The following BuildKit config:
+	//
+	// [registry."myregistry.io"]
+	//   ca=["/etc/config/myca.pem"]
+	//   [[registry."myregistry.io".keypair]]
+	//     key="/etc/config/key.pem"
+	//     cert="/etc/config/cert.pem"
+	//
+	// will be translated in the container as:
+	//
+	// [registry."myregistry.io"]
+	//   ca=["/etc/buildkit/certs/myregistry.io/myca.pem"]
+	//   [[registry."myregistry.io".keypair]]
+	//     key="/etc/buildkit/certs/myregistry.io/key.pem"
+	//     cert="/etc/buildkit/certs/myregistry.io/cert.pem"
+	if btoml.Has("registry") {
+		for regName := range btoml.GetArray("registry").(*toml.Tree).Values() {
+			regConf := btoml.GetPath([]string{"registry", regName}).(*toml.Tree)
+			if regConf == nil {
+				continue
+			}
+			regCertsDir := path.Join(tmpBuildKitCertsDir, regName)
+			if err := os.Mkdir(regCertsDir, 0755); err != nil {
+				return "", err
+			}
+			if regConf.Has("ca") {
+				regCAs := regConf.GetArray("ca").([]string)
+				if len(regCAs) > 0 {
+					var cas []string
+					for _, ca := range regCAs {
+						cas = append(cas, path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(ca)))
+						if err := copyfile(ca, path.Join(regCertsDir, path.Base(ca))); err != nil {
+							return "", err
+						}
+					}
+					regConf.Set("ca", cas)
+				}
+			}
+			if regConf.Has("keypair") {
+				regKeyPairs := regConf.GetArray("keypair").([]*toml.Tree)
+				if len(regKeyPairs) == 0 {
+					continue
+				}
+				for _, kp := range regKeyPairs {
+					if kp == nil {
+						continue
+					}
+					key := kp.Get("key").(string)
+					if len(key) > 0 {
+						kp.Set("key", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(key)))
+						if err := copyfile(key, path.Join(regCertsDir, path.Base(key))); err != nil {
+							return "", err
+						}
+					}
+					cert := kp.Get("cert").(string)
+					if len(cert) > 0 {
+						kp.Set("cert", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(cert)))
+						if err := copyfile(cert, path.Join(regCertsDir, path.Base(cert))); err != nil {
+							return "", err
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// Write BuildKit config
+	bkfile, err := os.OpenFile(path.Join(tmpBuildKitConfigDir, "buildkitd.toml"), os.O_CREATE|os.O_WRONLY, 0600)
+	if err != nil {
+		return "", err
+	}
+	_, err = btoml.WriteTo(bkfile)
+	if err != nil {
+		return "", err
+	}
+
+	return tmpDir, nil
+}
+
+func copyfile(src string, dst string) error {
+	sf, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sf.Close()
+	df, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0600)
+	if err != nil {
+		return err
+	}
+	defer df.Close()
+	_, err = io.Copy(df, sf)
+	return err
+}