container driver: copy ca and user tls registries certs

Signed-off-by: CrazyMax <[email protected]>

Author: crazy-max

docs/reference/buildx_create.md

@@ -84,6 +84,11 @@ Specifies the configuration file for the buildkitd daemon to use. The configurat
 can be overridden by [`--buildkitd-flags`](#buildkitd-flags).
 See an [example buildkitd configuration file](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md).
 
+Also note that if you create a `docker-container` builder and have specified
+certificates for registries in the `buildkitd.toml` configuration, their location
+will be translated in the configuration to fit in the container under
+`/etc/buildkit/certs`.
+
 ### <a name="driver"></a> Set the builder driver to use (--driver)
 
 ```

driver/docker-container/driver.go

@@ -1,39 +1,37 @@
 package docker
 
 import (
-	"archive/tar"
 	"bytes"
 	"context"
 	"io"
 	"io/ioutil"
 	"net"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/docker/buildx/driver"
 	"github.com/docker/buildx/driver/bkimage"
+	"github.com/docker/buildx/util/bkutil"
 	"github.com/docker/buildx/util/imagetools"
 	"github.com/docker/buildx/util/progress"
+	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	dockertypes "github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/network"
 	dockerclient "github.com/docker/docker/client"
+	dockerarchive "github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/docker/docker/pkg/system"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/util/tracing/detect"
 	"github.com/pkg/errors"
 )
 
 const (
 	volumeStateSuffix = "_state"
-
-	// containerStateDir is the location where buildkitd inside the container
-	// stores its state. The container driver creates a Linux container, so
-	// this should match the location for Linux, as defined in:
-	// https://github.com/moby/buildkit/blob/v0.9.0/util/appdefaults/appdefaults_unix.go#L11-L15
-	containerBuildKitRootDir = "/var/lib/buildkit"
 )
 
 type Driver struct {
@@ -119,7 +117,7 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 				{
 					Type:   mount.TypeVolume,
 					Source: d.Name + volumeStateSuffix,
-					Target: containerBuildKitRootDir,
+					Target: bkutil.ContainerBuildKitRootDir,
 				},
 			},
 		}
@@ -139,11 +137,12 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 			return err
 		}
 		if f := d.InitConfig.ConfigFile; f != "" {
-			buf, err := readFileToTar(f)
+			ctnConfigDir, err := bkutil.ContainerConfDir(f)
 			if err != nil {
 				return err
 			}
-			if err := d.DockerAPI.CopyToContainer(ctx, d.Name, "/", buf, dockertypes.CopyToContainerOptions{}); err != nil {
+			defer os.RemoveAll(ctnConfigDir)
+			if err := d.copyToContainer(ctx, ctnConfigDir+"/.", "/"); err != nil {
 				return err
 			}
 		}
@@ -205,6 +204,89 @@ func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error {
 	return rc.Close()
 }
 
+// copyToContainer is based on the implementation from docker/cli
+// https://github.com/docker/cli/blob/master/cli/command/container/cp.go
+func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstPath string) error {
+	var err error
+
+	// Get an absolute source path.
+	srcPath, err = resolveLocalPath(srcPath)
+	if err != nil {
+		return err
+	}
+
+	// Prepare the destination.
+	dstInfo := dockerarchive.CopyInfo{Path: dstPath}
+	dstStat, err := d.DockerAPI.ContainerStatPath(ctx, d.Name, dstPath)
+
+	// If the destination is a symbolic link, we should evaluate it.
+	if err == nil && dstStat.Mode&os.ModeSymlink != 0 {
+		linkTarget := dstStat.LinkTarget
+		if !system.IsAbs(linkTarget) {
+			dstParent, _ := dockerarchive.SplitPathDirEntry(dstPath)
+			linkTarget = filepath.Join(dstParent, linkTarget)
+		}
+		dstInfo.Path = linkTarget
+		if dstStat, err = d.DockerAPI.ContainerStatPath(ctx, d.Name, linkTarget); err != nil {
+			return err
+		}
+	}
+
+	// Validate the destination path.
+	if err := command.ValidateOutputPathFileMode(dstStat.Mode); err != nil {
+		return errors.Wrapf(err, `destination "%s:%s" must be a directory or a regular file`, d.Name, dstPath)
+	}
+
+	// Ignore any error and assume that the parent directory of the destination
+	// path exists, in which case the copy may still succeed. If there is any
+	// type of conflict (e.g., non-directory overwriting an existing directory
+	// or vice versa) the extraction will fail. If the destination simply did
+	// not exist, but the parent directory does, the extraction will still
+	// succeed.
+	dstInfo.Exists, dstInfo.IsDir = true, dstStat.Mode.IsDir()
+
+	// Prepare source copy info.
+	srcInfo, err := dockerarchive.CopyInfoSourcePath(srcPath, true)
+	if err != nil {
+		return err
+	}
+
+	srcArchive, err := dockerarchive.TarResource(srcInfo)
+	if err != nil {
+		return err
+	}
+	defer srcArchive.Close()
+
+	// With the stat info about the local source as well as the
+	// destination, we have enough information to know whether we need to
+	// alter the archive that we upload so that when the server extracts
+	// it to the specified directory in the container we get the desired
+	// copy behavior.
+
+	// See comments in the implementation of `dockerarchive.PrepareArchiveCopy`
+	// for exactly what goes into deciding how and whether the source
+	// archive needs to be altered for the correct copy behavior when it is
+	// extracted. This function also infers from the source and destination
+	// info which directory to extract to, which may be the parent of the
+	// destination that the user specified.
+	dstDir, preparedArchive, err := dockerarchive.PrepareArchiveCopy(srcArchive, srcInfo, dstInfo)
+	if err != nil {
+		return err
+	}
+	defer preparedArchive.Close()
+
+	return d.DockerAPI.CopyToContainer(ctx, d.Name, dstDir, preparedArchive, dockertypes.CopyToContainerOptions{
+		AllowOverwriteDirWithFile: false,
+	})
+}
+
+func resolveLocalPath(localPath string) (absPath string, err error) {
+	if absPath, err = filepath.Abs(localPath); err != nil {
+		return
+	}
+	return dockerarchive.PreserveTrailingDotOrSeparator(absPath, localPath, filepath.Separator), nil
+}
+
 func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) {
 	execConfig := types.ExecConfig{
 		Cmd:          cmd,
@@ -366,29 +448,6 @@ func (d *demux) Read(dt []byte) (int, error) {
 	return d.Reader.Read(dt)
 }
 
-func readFileToTar(fn string) (*bytes.Buffer, error) {
-	buf := bytes.NewBuffer(nil)
-	tw := tar.NewWriter(buf)
-	dt, err := ioutil.ReadFile(fn)
-	if err != nil {
-		return nil, err
-	}
-	if err := tw.WriteHeader(&tar.Header{
-		Name: "/etc/buildkit/buildkitd.toml",
-		Size: int64(len(dt)),
-		Mode: 0644,
-	}); err != nil {
-		return nil, err
-	}
-	if _, err := tw.Write(dt); err != nil {
-		return nil, err
-	}
-	if err := tw.Close(); err != nil {
-		return nil, err
-	}
-	return buf, nil
-}
-
 type logWriter struct {
 	logger progress.SubLogger
 	stream int

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/moby/buildkit v0.9.1-0.20211019185819-8778943ac3da
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
+	github.com/pelletier/go-toml v1.9.4
 	github.com/pkg/errors v0.9.1
 	github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002
 	github.com/sirupsen/logrus v1.8.1

util/bkutil/config.go

@@ -0,0 +1,25 @@
+package bkutil
+
+import (
+	"os"
+
+	"github.com/pelletier/go-toml"
+	"github.com/pkg/errors"
+)
+
+// LoadConfigTree loads BuildKit config toml tree
+func LoadConfigTree(fp string) (*toml.Tree, error) {
+	f, err := os.Open(fp)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, errors.Wrapf(err, "failed to load config from %s", fp)
+	}
+	defer f.Close()
+	t, err := toml.LoadReader(f)
+	if err != nil {
+		return t, errors.Wrap(err, "failed to parse config")
+	}
+	return t, nil
+}

util/bkutil/container.go

@@ -0,0 +1,157 @@
+package bkutil
+
+import (
+	"io"
+	"os"
+	"path"
+
+	"github.com/pelletier/go-toml"
+	"github.com/pkg/errors"
+)
+
+const (
+	// ContainerBuildKitRootDir and ContainerBuildKitConfigDir are the location
+	// where buildkitd inside the container stores its state. Some drivers
+	// create a Linux container, so this should match the location for Linux,
+	// as defined in: https://github.com/moby/buildkit/blob/v0.9.0/util/appdefaults/appdefaults_unix.go#L11-L15
+	ContainerBuildKitRootDir   = "/var/lib/buildkit"
+	ContainerBuildKitConfigDir = "/etc/buildkit"
+)
+
+// ContainerConfDir creates a temp directory with BuildKit config and
+// registry certificates ready to be copied to a container.
+func ContainerConfDir(bkconfig string) (string, error) {
+	if _, err := os.Stat(bkconfig); errors.Is(err, os.ErrNotExist) {
+		return "", errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig)
+	} else if err != nil {
+		return "", errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig)
+	}
+
+	// Load config tree
+	btoml, err := LoadConfigTree(bkconfig)
+	if err != nil {
+		return "", err
+	}
+
+	// Temp dir that will be copied to the container
+	tmpDir, err := os.MkdirTemp("", "buildkitd-config")
+	if err != nil {
+		return "", err
+	}
+
+	// Create BuildKit config folders
+	tmpBuildKitConfigDir := path.Join(tmpDir, ContainerBuildKitConfigDir)
+	tmpBuildKitCertsDir := path.Join(tmpBuildKitConfigDir, "certs")
+	if err := os.MkdirAll(tmpBuildKitCertsDir, 0755); err != nil {
+		return "", err
+	}
+
+	// Iterate through registry config to copy certs and update
+	// BuildKit config with the underlying certs' path in the container.
+	//
+	// The following BuildKit config:
+	//
+	// [registry."myregistry.io"]
+	//   ca=["/etc/config/myca.pem"]
+	//   [[registry."myregistry.io".keypair]]
+	//     key="/etc/config/key.pem"
+	//     cert="/etc/config/cert.pem"
+	//
+	// will be translated in the container as:
+	//
+	// [registry."myregistry.io"]
+	//   ca=["/etc/buildkit/certs/myregistry.io/myca.pem"]
+	//   [[registry."myregistry.io".keypair]]
+	//     key="/etc/buildkit/certs/myregistry.io/key.pem"
+	//     cert="/etc/buildkit/certs/myregistry.io/cert.pem"
+	if btoml.Has("registry") {
+		for regName := range btoml.GetArray("registry").(*toml.Tree).Values() {
+			regConf := btoml.GetPath([]string{"registry", regName}).(*toml.Tree)
+			if regConf == nil {
+				continue
+			}
+			regCertsDir := path.Join(tmpBuildKitCertsDir, regName)
+			if err := os.Mkdir(regCertsDir, 0755); err != nil {
+				return "", err
+			}
+			if regConf.Has("ca") {
+				regCAs := regConf.GetArray("ca").([]string)
+				if len(regCAs) > 0 {
+					var cas []string
+					for _, ca := range regCAs {
+						cas = append(cas, path.Join(ContainerBuildKitConfigDir, "certs", regName, path.Base(ca)))
+						if err := copyfile(ca, path.Join(regCertsDir, path.Base(ca))); err != nil {
+							return "", err
+						}
+					}
+					regConf.Set("ca", cas)
+				}
+			}
+			if regConf.Has("keypair") {
+				regKeyPairs := regConf.GetArray("keypair").([]*toml.Tree)
+				if len(regKeyPairs) == 0 {
+					continue
+				}
+				for _, kp := range regKeyPairs {
+					if kp == nil {
+						continue
+					}
+					key := kp.Get("key").(string)
+					if len(key) > 0 {
+						kp.Set("key", path.Join(ContainerBuildKitConfigDir, "certs", regName, path.Base(key)))
+						if err := copyfile(key, path.Join(regCertsDir, path.Base(key))); err != nil {
+							return "", err
+						}
+					}
+					cert := kp.Get("cert").(string)
+					if len(cert) > 0 {
+						kp.Set("cert", path.Join(ContainerBuildKitConfigDir, "certs", regName, path.Base(cert)))
+						if err := copyfile(cert, path.Join(regCertsDir, path.Base(cert))); err != nil {
+							return "", err
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// Write BuildKit config
+	bkfile, err := os.OpenFile(path.Join(tmpBuildKitConfigDir, "buildkitd.toml"), os.O_CREATE|os.O_WRONLY, 0600)
+	if err != nil {
+		return "", err
+	}
+	_, err = btoml.WriteTo(bkfile)
+	if err != nil {
+		return "", err
+	}
+
+	return tmpDir, nil
+}
+
+func copyfile(src string, dst string) error {
+	si, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+	if si.Mode()&os.ModeSymlink != 0 {
+		if src, err = os.Readlink(src); err != nil {
+			return err
+		}
+		si, err = os.Stat(src)
+		if err != nil {
+			return err
+		}
+	}
+	sf, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sf.Close()
+	df, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, si.Mode())
+	if err != nil {
+		return err
+	}
+	defer df.Close()
+	_, err = io.Copy(df, sf)
+	return err
+}