container driver: copy ca and user tls registries certs

Signed-off-by: CrazyMax <[email protected]>

Author: crazy-max

docs/reference/buildx_create.md

@@ -84,6 +84,10 @@ Specifies the configuration file for the buildkitd daemon to use. The configurat
 can be overridden by [`--buildkitd-flags`](#buildkitd-flags).
 See an [example buildkitd configuration file](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md).
 
+Also note that if you create a `docker-container` builder and have specified
+certificates for registries in the `buildkitd.toml` configuration, these will
+be copied into the container under `/etc/buildkit/certs`.
+
 ### <a name="driver"></a> Set the builder driver to use (--driver)
 
 ```

driver/docker-container/driver.go

@@ -1,28 +1,33 @@
 package docker
 
 import (
-	"archive/tar"
 	"bytes"
 	"context"
 	"io"
 	"io/ioutil"
 	"net"
 	"os"
+	"path"
+	"path/filepath"
 	"time"
 
 	"github.com/docker/buildx/driver"
 	"github.com/docker/buildx/driver/bkimage"
 	"github.com/docker/buildx/util/imagetools"
 	"github.com/docker/buildx/util/progress"
+	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
 	dockertypes "github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/network"
 	dockerclient "github.com/docker/docker/client"
+	dockerarchive "github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/docker/docker/pkg/system"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/util/tracing/detect"
+	"github.com/pelletier/go-toml"
 	"github.com/pkg/errors"
 )
 
@@ -33,7 +38,8 @@ const (
 	// stores its state. The container driver creates a Linux container, so
 	// this should match the location for Linux, as defined in:
 	// https://github.com/moby/buildkit/blob/v0.9.0/util/appdefaults/appdefaults_unix.go#L11-L15
-	containerBuildKitRootDir = "/var/lib/buildkit"
+	containerBuildKitRootDir   = "/var/lib/buildkit"
+	containerBuildKitConfigDir = "/etc/buildkit"
 )
 
 type Driver struct {
@@ -139,12 +145,14 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 			return err
 		}
 		if f := d.InitConfig.ConfigFile; f != "" {
-			buf, err := readFileToTar(f)
-			if err != nil {
-				return err
-			}
-			if err := d.DockerAPI.CopyToContainer(ctx, d.Name, "/", buf, dockertypes.CopyToContainerOptions{}); err != nil {
-				return err
+			if _, err := os.Stat(d.InitConfig.ConfigFile); err == nil {
+				if err := d.copyBuildKitConfToContainer(ctx, f); err != nil {
+					return err
+				}
+			} else if errors.Is(err, os.ErrNotExist) {
+				return errors.Wrapf(err, "buildkit configuration file not found: %s", d.InitConfig.ConfigFile)
+			} else {
+				return errors.Wrapf(err, "invalid buildkit configuration file: %s", d.InitConfig.ConfigFile)
 			}
 		}
 		if err := d.start(ctx, l); err != nil {
@@ -205,6 +213,205 @@ func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error {
 	return rc.Close()
 }
 
+// copyToContainer is based on the implementation from docker/cli
+// https://github.com/docker/cli/blob/master/cli/command/container/cp.go
+func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstPath string) error {
+	var err error
+
+	// Get an absolute source path.
+	srcPath, err = resolveLocalPath(srcPath)
+	if err != nil {
+		return err
+	}
+
+	// Prepare the destination.
+	dstInfo := dockerarchive.CopyInfo{Path: dstPath}
+	dstStat, err := d.DockerAPI.ContainerStatPath(ctx, d.Name, dstPath)
+
+	// If the destination is a symbolic link, we should evaluate it.
+	if err == nil && dstStat.Mode&os.ModeSymlink != 0 {
+		linkTarget := dstStat.LinkTarget
+		if !system.IsAbs(linkTarget) {
+			dstParent, _ := dockerarchive.SplitPathDirEntry(dstPath)
+			linkTarget = filepath.Join(dstParent, linkTarget)
+		}
+		dstInfo.Path = linkTarget
+		if dstStat, err = d.DockerAPI.ContainerStatPath(ctx, d.Name, linkTarget); err != nil {
+			return err
+		}
+	}
+
+	// Validate the destination path.
+	if err := command.ValidateOutputPathFileMode(dstStat.Mode); err != nil {
+		return errors.Wrapf(err, `destination "%s:%s" must be a directory or a regular file`, d.Name, dstPath)
+	}
+
+	// Ignore any error and assume that the parent directory of the destination
+	// path exists, in which case the copy may still succeed. If there is any
+	// type of conflict (e.g., non-directory overwriting an existing directory
+	// or vice versa) the extraction will fail. If the destination simply did
+	// not exist, but the parent directory does, the extraction will still
+	// succeed.
+	dstInfo.Exists, dstInfo.IsDir = true, dstStat.Mode.IsDir()
+
+	// Prepare source copy info.
+	srcInfo, err := dockerarchive.CopyInfoSourcePath(srcPath, true)
+	if err != nil {
+		return err
+	}
+
+	srcArchive, err := dockerarchive.TarResource(srcInfo)
+	if err != nil {
+		return err
+	}
+	defer srcArchive.Close()
+
+	// With the stat info about the local source as well as the
+	// destination, we have enough information to know whether we need to
+	// alter the archive that we upload so that when the server extracts
+	// it to the specified directory in the container we get the desired
+	// copy behavior.
+
+	// See comments in the implementation of `dockerarchive.PrepareArchiveCopy`
+	// for exactly what goes into deciding how and whether the source
+	// archive needs to be altered for the correct copy behavior when it is
+	// extracted. This function also infers from the source and destination
+	// info which directory to extract to, which may be the parent of the
+	// destination that the user specified.
+	dstDir, preparedArchive, err := dockerarchive.PrepareArchiveCopy(srcArchive, srcInfo, dstInfo)
+	if err != nil {
+		return err
+	}
+	defer preparedArchive.Close()
+
+	return d.DockerAPI.CopyToContainer(ctx, d.Name, dstDir, preparedArchive, dockertypes.CopyToContainerOptions{
+		AllowOverwriteDirWithFile: false,
+	})
+}
+
+func resolveLocalPath(localPath string) (absPath string, err error) {
+	if absPath, err = filepath.Abs(localPath); err != nil {
+		return
+	}
+	return dockerarchive.PreserveTrailingDotOrSeparator(absPath, localPath, filepath.Separator), nil
+}
+
+// copyBuildKitConfToContainer copy BuildKit config and registry
+// certificates to the containers
+func (d *Driver) copyBuildKitConfToContainer(ctx context.Context, bkconfig string) error {
+	// Load BuildKit config tree
+	btoml, err := loadBuildKitConfigTree(bkconfig)
+	if err != nil {
+		return err
+	}
+
+	// Temp dir that will be copied to the container
+	tmpDir, err := os.MkdirTemp("", "buildkitd-config")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	// Create BuildKit config folders
+	tmpBuildKitConfigDir := path.Join(tmpDir, containerBuildKitConfigDir)
+	tmpBuildKitCertsDir := path.Join(tmpBuildKitConfigDir, "certs")
+	if err := os.MkdirAll(tmpBuildKitCertsDir, 0755); err != nil {
+		return err
+	}
+
+	// Iterate through registry config to copy certs and update
+	// BuildKit config with the underlying certs' path in the container.
+	//
+	// The following BuildKit config:
+	//
+	// [registry."myregistry.io"]
+	//   ca=["/etc/config/myca.pem"]
+	//   [[registry."myregistry.io".keypair]]
+	//     key="/etc/config/key.pem"
+	//     cert="/etc/config/cert.pem"
+	//
+	// will be translated in the container as:
+	//
+	// [registry."myregistry.io"]
+	//   ca=["/etc/buildkit/certs/myregistry.io/myca.pem"]
+	//   [[registry."myregistry.io".keypair]]
+	//     key="/etc/buildkit/certs/myregistry.io/key.pem"
+	//     cert="/etc/buildkit/certs/myregistry.io/cert.pem"
+	for regName := range btoml.GetArray("registry").(*toml.Tree).Values() {
+		regConf := btoml.GetPath([]string{"registry", regName}).(*toml.Tree)
+		if regConf == nil {
+			continue
+		}
+
+		// Create registry certs folder
+		regCertsDir := path.Join(tmpBuildKitCertsDir, regName)
+		if err := os.Mkdir(regCertsDir, 0755); err != nil {
+			return err
+		}
+
+		regCAs := regConf.GetArray("ca").([]string)
+		if len(regCAs) > 0 {
+			var cas []string
+			for _, ca := range regCAs {
+				cas = append(cas, path.Join(containerBuildKitConfigDir, "certs", regName, path.Base(ca)))
+				if err := copyfile(ca, path.Join(regCertsDir, path.Base(ca))); err != nil {
+					return err
+				}
+			}
+			regConf.Set("ca", cas)
+		}
+
+		regKeyPairs := regConf.GetArray("keypair").([]*toml.Tree)
+		if len(regKeyPairs) > 0 {
+			for _, kp := range regKeyPairs {
+				key := kp.Get("key").(string)
+				if len(key) > 0 {
+					kp.Set("key", path.Join(containerBuildKitConfigDir, "certs", regName, path.Base(key)))
+					if err := copyfile(key, path.Join(regCertsDir, path.Base(key))); err != nil {
+						return err
+					}
+				}
+				cert := kp.Get("cert").(string)
+				if len(cert) > 0 {
+					kp.Set("cert", path.Join(containerBuildKitConfigDir, "certs", regName, path.Base(cert)))
+					if err := copyfile(cert, path.Join(regCertsDir, path.Base(cert))); err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
+	// Write BuildKit config
+	bkfile, err := os.OpenFile(path.Join(tmpBuildKitConfigDir, "buildkitd.toml"), os.O_CREATE|os.O_WRONLY, 0600)
+	if err != nil {
+		return err
+	}
+	_, err = btoml.WriteTo(bkfile)
+	if err != nil {
+		return err
+	}
+
+	return d.copyToContainer(ctx, tmpDir+"/.", "/")
+}
+
+// loadBuildKitConfigTree loads toml BuildKit config tree
+func loadBuildKitConfigTree(fp string) (*toml.Tree, error) {
+	f, err := os.Open(fp)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, errors.Wrapf(err, "failed to load config from %s", fp)
+	}
+	defer f.Close()
+	t, err := toml.LoadReader(f)
+	if err != nil {
+		return t, errors.Wrap(err, "failed to parse config")
+	}
+	return t, nil
+}
+
 func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) {
 	execConfig := types.ExecConfig{
 		Cmd:          cmd,
@@ -366,29 +573,6 @@ func (d *demux) Read(dt []byte) (int, error) {
 	return d.Reader.Read(dt)
 }
 
-func readFileToTar(fn string) (*bytes.Buffer, error) {
-	buf := bytes.NewBuffer(nil)
-	tw := tar.NewWriter(buf)
-	dt, err := ioutil.ReadFile(fn)
-	if err != nil {
-		return nil, err
-	}
-	if err := tw.WriteHeader(&tar.Header{
-		Name: "/etc/buildkit/buildkitd.toml",
-		Size: int64(len(dt)),
-		Mode: 0644,
-	}); err != nil {
-		return nil, err
-	}
-	if _, err := tw.Write(dt); err != nil {
-		return nil, err
-	}
-	if err := tw.Close(); err != nil {
-		return nil, err
-	}
-	return buf, nil
-}
-
 type logWriter struct {
 	logger progress.SubLogger
 	stream int
@@ -398,3 +582,35 @@ func (l *logWriter) Write(dt []byte) (int, error) {
 	l.logger.Log(l.stream, dt)
 	return len(dt), nil
 }
+
+func copyfile(src string, dst string) error {
+	si, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	if si.Mode()&os.ModeSymlink != 0 {
+		if src, err = os.Readlink(src); err != nil {
+			return err
+		}
+		si, err = os.Stat(src)
+		if err != nil {
+			return err
+		}
+	}
+
+	sf, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sf.Close()
+
+	df, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, si.Mode())
+	if err != nil {
+		return err
+	}
+	defer df.Close()
+
+	_, err = io.Copy(df, sf)
+	return err
+}

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/moby/buildkit v0.9.1-0.20211008210008-ba673bbdab4f
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
+	github.com/pelletier/go-toml v1.9.4
 	github.com/pkg/errors v0.9.1
 	github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002
 	github.com/sirupsen/logrus v1.8.1