Skip to content

Commit d311561

Browse files
tonistiigitonistiigi
authored
Merge pull request #824 from tonistiigi/config-files-store2
store snapshot of config files on create
2 parents 44e180b + 40121c6 commit d311561

File tree

9 files changed

+146
-99
lines changed

9 files changed

+146
-99
lines changed

‎commands/util.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
225225
}
226226
}
227227

228-
d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.ConfigFile, n.DriverOpts, n.Platforms, contextPathHash)
228+
d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, contextPathHash)
229229
if err != nil {
230230
di.Err = err
231231
return nil
@@ -360,7 +360,7 @@ func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, defaultOnly b
360360
}
361361
}
362362

363-
d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, "", nil, nil, contextPathHash)
363+
d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, nil, nil, nil, contextPathHash)
364364
if err != nil {
365365
return nil, err
366366
}

‎driver/docker-container/driver.go

+37-11
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import (
77
"io/ioutil"
88
"net"
99
"os"
10+
"path"
11+
"path/filepath"
1012
"time"
1113

1214
"github.com/docker/buildx/driver"
@@ -134,15 +136,8 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
134136
if err != nil {
135137
return err
136138
}
137-
if f := d.InitConfig.ConfigFile; f != "" {
138-
configFiles, err := confutil.LoadConfigFiles(f)
139-
if err != nil {
140-
return err
141-
}
142-
defer os.RemoveAll(configFiles)
143-
if err := d.copyToContainer(ctx, configFiles, "/"); err != nil {
144-
return err
145-
}
139+
if err := d.copyToContainer(ctx, d.InitConfig.Files); err != nil {
140+
return err
146141
}
147142
if err := d.start(ctx, l); err != nil {
148143
return err
@@ -202,15 +197,22 @@ func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error {
202197
return rc.Close()
203198
}
204199

205-
func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir string) error {
200+
func (d *Driver) copyToContainer(ctx context.Context, files map[string][]byte) error {
201+
srcPath, err := writeConfigFiles(files)
202+
if err != nil {
203+
return err
204+
}
205+
if srcPath != "" {
206+
defer os.RemoveAll(srcPath)
207+
}
206208
srcArchive, err := dockerarchive.TarWithOptions(srcPath, &dockerarchive.TarOptions{
207209
ChownOpts: &idtools.Identity{UID: 0, GID: 0},
208210
})
209211
if err != nil {
210212
return err
211213
}
212214
defer srcArchive.Close()
213-
return d.DockerAPI.CopyToContainer(ctx, d.Name, dstDir, srcArchive, dockertypes.CopyToContainerOptions{})
215+
return d.DockerAPI.CopyToContainer(ctx, d.Name, "/", srcArchive, dockertypes.CopyToContainerOptions{})
214216
}
215217

216218
func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) {
@@ -383,3 +385,27 @@ func (l *logWriter) Write(dt []byte) (int, error) {
383385
l.logger.Log(l.stream, dt)
384386
return len(dt), nil
385387
}
388+
389+
func writeConfigFiles(m map[string][]byte) (_ string, err error) {
390+
// Temp dir that will be copied to the container
391+
tmpDir, err := os.MkdirTemp("", "buildkitd-config")
392+
if err != nil {
393+
return "", err
394+
}
395+
defer func() {
396+
if err != nil {
397+
os.RemoveAll(tmpDir)
398+
}
399+
}()
400+
for f, dt := range m {
401+
f = path.Join(confutil.DefaultBuildKitConfigDir, f)
402+
p := filepath.Join(tmpDir, f)
403+
if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
404+
return "", err
405+
}
406+
if err := os.WriteFile(p, dt, 0600); err != nil {
407+
return "", err
408+
}
409+
}
410+
return tmpDir, nil
411+
}

‎driver/docker/factory.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
4444
if cfg.DockerAPI == nil {
4545
return nil, errors.Errorf("docker driver requires docker API access")
4646
}
47-
if cfg.ConfigFile != "" {
47+
if len(cfg.Files) > 0 {
4848
return nil, errors.Errorf("setting config file is not supported for docker driver, use dockerd configuration file")
4949
}
5050

‎driver/kubernetes/driver.go

+9-9
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ type Driver struct {
4141
factory driver.Factory
4242
minReplicas int
4343
deployment *appsv1.Deployment
44-
configMap *corev1.ConfigMap
44+
configMaps []*corev1.ConfigMap
4545
clientset *kubernetes.Clientset
4646
deploymentClient clientappsv1.DeploymentInterface
4747
podClient clientcorev1.PodInterface
@@ -65,16 +65,16 @@ func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
6565
return errors.Wrapf(err, "error for bootstrap %q", d.deployment.Name)
6666
}
6767

68-
if d.configMap != nil {
68+
for _, cfg := range d.configMaps {
6969
// create ConfigMap first if exists
70-
_, err = d.configMapClient.Create(ctx, d.configMap, metav1.CreateOptions{})
70+
_, err = d.configMapClient.Create(ctx, cfg, metav1.CreateOptions{})
7171
if err != nil {
7272
if !apierrors.IsAlreadyExists(err) {
73-
return errors.Wrapf(err, "error while calling configMapClient.Create for %q", d.configMap.Name)
73+
return errors.Wrapf(err, "error while calling configMapClient.Create for %q", cfg.Name)
7474
}
75-
_, err = d.configMapClient.Update(ctx, d.configMap, metav1.UpdateOptions{})
75+
_, err = d.configMapClient.Update(ctx, cfg, metav1.UpdateOptions{})
7676
if err != nil {
77-
return errors.Wrapf(err, "error while calling configMapClient.Update for %q", d.configMap.Name)
77+
return errors.Wrapf(err, "error while calling configMapClient.Update for %q", cfg.Name)
7878
}
7979
}
8080
}
@@ -171,10 +171,10 @@ func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
171171
return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
172172
}
173173
}
174-
if d.configMap != nil {
175-
if err := d.configMapClient.Delete(ctx, d.configMap.Name, metav1.DeleteOptions{}); err != nil {
174+
for _, cfg := range d.configMaps {
175+
if err := d.configMapClient.Delete(ctx, cfg.Name, metav1.DeleteOptions{}); err != nil {
176176
if !apierrors.IsNotFound(err) {
177-
return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", d.configMap.Name)
177+
return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", cfg.Name)
178178
}
179179
}
180180
}

‎driver/kubernetes/factory.go

+2-10
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package kubernetes
22

33
import (
44
"context"
5-
"os"
65
"strconv"
76
"strings"
87

@@ -74,18 +73,11 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
7473
BuildkitFlags: cfg.BuildkitFlags,
7574
Rootless: false,
7675
Platforms: cfg.Platforms,
76+
ConfigFiles: cfg.Files,
7777
}
7878

7979
deploymentOpt.Qemu.Image = bkimage.QemuImage
8080

81-
if cfg.ConfigFile != "" {
82-
buildkitConfig, err := os.ReadFile(cfg.ConfigFile)
83-
if err != nil {
84-
return nil, err
85-
}
86-
deploymentOpt.BuildkitConfig = buildkitConfig
87-
}
88-
8981
loadbalance := LoadbalanceSticky
9082

9183
for k, v := range cfg.DriverOpts {
@@ -147,7 +139,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
147139
}
148140
}
149141

150-
d.deployment, d.configMap, err = manifest.NewDeployment(deploymentOpt)
142+
d.deployment, d.configMaps, err = manifest.NewDeployment(deploymentOpt)
151143
if err != nil {
152144
return nil, err
153145
}

‎driver/kubernetes/manifest/manifest.go

+45-14
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package manifest
22

33
import (
4+
"fmt"
5+
"path"
46
"strings"
57

68
"github.com/docker/buildx/util/platformutil"
@@ -25,9 +27,8 @@ type DeploymentOpt struct {
2527
}
2628

2729
BuildkitFlags []string
28-
// BuildkitConfig
29-
// when not empty, will create configmap with buildkit.toml and mounted
30-
BuildkitConfig []byte
30+
// files mounted at /etc/buildkitd
31+
ConfigFiles map[string][]byte
3132

3233
Rootless bool
3334
NodeSelector map[string]string
@@ -43,7 +44,7 @@ const (
4344
AnnotationPlatform = "buildx.docker.com/platform"
4445
)
4546

46-
func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMap, err error) {
47+
func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.ConfigMap, err error) {
4748
labels := map[string]string{
4849
"app": opt.Name,
4950
}
@@ -103,38 +104,36 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMa
103104
},
104105
},
105106
}
106-
107-
if len(opt.BuildkitConfig) > 0 {
108-
c = &corev1.ConfigMap{
107+
for _, cfg := range splitConfigFiles(opt.ConfigFiles) {
108+
cc := &corev1.ConfigMap{
109109
TypeMeta: metav1.TypeMeta{
110110
APIVersion: corev1.SchemeGroupVersion.String(),
111111
Kind: "ConfigMap",
112112
},
113113
ObjectMeta: metav1.ObjectMeta{
114114
Namespace: opt.Namespace,
115-
Name: opt.Name + "-config",
115+
Name: opt.Name + "-" + cfg.name,
116116
Annotations: annotations,
117117
},
118-
Data: map[string]string{
119-
"buildkitd.toml": string(opt.BuildkitConfig),
120-
},
118+
Data: cfg.files,
121119
}
122120

123121
d.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{
124-
Name: "config",
125-
MountPath: "/etc/buildkit",
122+
Name: cfg.name,
123+
MountPath: path.Join("/etc/buildkit", cfg.path),
126124
}}
127125

128126
d.Spec.Template.Spec.Volumes = []corev1.Volume{{
129127
Name: "config",
130128
VolumeSource: corev1.VolumeSource{
131129
ConfigMap: &corev1.ConfigMapVolumeSource{
132130
LocalObjectReference: corev1.LocalObjectReference{
133-
Name: c.Name,
131+
Name: cc.Name,
134132
},
135133
},
136134
},
137135
}}
136+
c = append(c, cc)
138137
}
139138

140139
if opt.Qemu.Install {
@@ -208,3 +207,35 @@ func toRootless(d *appsv1.Deployment) error {
208207
d.Spec.Template.ObjectMeta.Annotations["container.seccomp.security.alpha.kubernetes.io/"+containerName] = "unconfined"
209208
return nil
210209
}
210+
211+
type config struct {
212+
name string
213+
path string
214+
files map[string]string
215+
}
216+
217+
func splitConfigFiles(m map[string][]byte) []config {
218+
var c []config
219+
idx := map[string]int{}
220+
nameIdx := 0
221+
for k, v := range m {
222+
dir := path.Dir(k)
223+
i, ok := idx[dir]
224+
if !ok {
225+
idx[dir] = len(c)
226+
i = len(c)
227+
name := "config"
228+
if dir != "." {
229+
nameIdx++
230+
name = fmt.Sprintf("%s-%d", name, nameIdx)
231+
}
232+
c = append(c, config{
233+
path: dir,
234+
name: name,
235+
files: map[string]string{},
236+
})
237+
}
238+
c[i].files[path.Base(k)] = string(v)
239+
}
240+
return c
241+
}

‎driver/manager.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ type InitConfig struct {
5353
DockerAPI dockerclient.APIClient
5454
KubeClientConfig KubeClientConfig
5555
BuildkitFlags []string
56-
ConfigFile string
56+
Files map[string][]byte
5757
DriverOpts map[string]string
5858
Auth Auth
5959
Platforms []specs.Platform
@@ -103,17 +103,17 @@ func GetFactory(name string, instanceRequired bool) Factory {
103103
return nil
104104
}
105105

106-
func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, config string, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
106+
func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
107107
ic := InitConfig{
108108
DockerAPI: api,
109109
KubeClientConfig: kcc,
110110
Name: name,
111111
BuildkitFlags: flags,
112-
ConfigFile: config,
113112
DriverOpts: do,
114113
Auth: auth,
115114
Platforms: platforms,
116115
ContextPathHash: contextPathHash,
116+
Files: files,
117117
}
118118
if f == nil {
119119
var err error

‎store/nodegroup.go

+12-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55

66
"github.com/containerd/containerd/platforms"
7+
"github.com/docker/buildx/util/confutil"
78
"github.com/docker/buildx/util/platformutil"
89
specs "github.com/opencontainers/image-spec/specs-go/v1"
910
"github.com/pkg/errors"
@@ -21,8 +22,9 @@ type Node struct {
2122
Endpoint string
2223
Platforms []specs.Platform
2324
Flags []string
24-
ConfigFile string
2525
DriverOpts map[string]string
26+
27+
Files map[string][]byte
2628
}
2729

2830
func (ng *NodeGroup) Leave(name string) error {
@@ -88,10 +90,18 @@ func (ng *NodeGroup) Update(name, endpoint string, platforms []string, endpoints
8890
Name: name,
8991
Endpoint: endpoint,
9092
Platforms: pp,
91-
ConfigFile: configFile,
9293
Flags: flags,
9394
DriverOpts: do,
9495
}
96+
97+
if configFile != "" {
98+
files, err := confutil.LoadConfigFiles(configFile)
99+
if err != nil {
100+
return err
101+
}
102+
n.Files = files
103+
}
104+
95105
ng.Nodes = append(ng.Nodes, n)
96106

97107
if err := ng.validateDuplicates(endpoint, len(ng.Nodes)-1); err != nil {

0 commit comments

Comments
 (0)