[release/1.7 backport] Add --pull option to build command

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
     - uses: actions/[email protected]
     - uses: actions/setup-go@v5
       with:
-        go-version: 1.21.x
+        go-version: 1.22.x
     - name: "Compile binaries"
       run: make artifacts
     - name: "SHA256SUMS"

.github/workflows/test.yml

@@ -8,7 +8,7 @@ on:
   pull_request:
 
 env:
-  GO_VERSION: 1.21.x
+  GO_VERSION: 1.22.x
 
 jobs:
   project:

Dockerfile

@@ -44,7 +44,7 @@ ARG TINI_VERSION=v0.19.0
 ARG BUILDG_VERSION=v0.4.1
 
 # Test deps
-ARG GO_VERSION=1.21
+ARG GO_VERSION=1.22
 ARG UBUNTU_VERSION=22.04
 ARG CONTAINERIZED_SYSTEMD_VERSION=v0.1.1
 ARG GOTESTSUM_VERSION=v1.11.0
@@ -65,6 +65,18 @@ ARG TARGETARCH
 RUN xx-apt-get update && \
   xx-apt-get install -y binutils gcc libc6-dev libbtrfs-dev libseccomp-dev
 
+# runc still requires Go 1.21
+# https://github.com/opencontainers/runc/issues/4233
+FROM --platform=$BUILDPLATFORM golang:1.21-bullseye AS build-base-debian-go121
+COPY --from=xx / /
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+  apt-get install -y git pkg-config dpkg-dev
+ARG TARGETARCH
+# libseccomp: for runc
+RUN xx-apt-get update && \
+  xx-apt-get install -y binutils gcc libc6-dev libseccomp-dev
+
 FROM build-base-debian AS build-containerd
 ARG TARGETARCH
 ARG CONTAINERD_VERSION
@@ -76,7 +88,9 @@ RUN git checkout ${CONTAINERD_VERSION} && \
 RUN GO=xx-go make STATIC=1 && \
   cp -a bin/containerd bin/containerd-shim-runc-v2 bin/ctr /out/$TARGETARCH
 
-FROM build-base-debian AS build-runc
+# runc still requires Go 1.21
+# https://github.com/opencontainers/runc/issues/4233
+FROM build-base-debian-go121 AS build-runc
 ARG RUNC_VERSION
 ARG TARGETARCH
 RUN git clone https://github.com/opencontainers/runc.git /go/src/github.com/opencontainers/runc

Dockerfile.d/SHA256SUMS.d/cni-plugins-v1.4.1

@@ -1,2 +1,2 @@
-1511f6c003ace805eafeb1132727791326283cff88a923d76329e1892bba7a10  cni-plugins-linux-amd64-v1.4.1.tgz
-72644e13557cda8a5b39baf97fc5e93d23fdf7baba7700000e7e9efd8bdf9234  cni-plugins-linux-arm64-v1.4.1.tgz
+2a0ea7072d1806b8526489bcd3b4847a06ab010ee32ba3c3d4e5a3235d3eb138  cni-plugins-linux-amd64-v1.4.1.tgz
+56fe62d73942cffd8f119d2b8ecb6a062e85f529a3dbfc7aa5cd83c2c01929a7  cni-plugins-linux-arm64-v1.4.1.tgz

cmd/nerdctl/builder_build.go

@@ -50,6 +50,7 @@ If Dockerfile is not present and -f is not specified, it will look for Container
 	buildCommand.Flags().Bool("no-cache", false, "Do not use cache when building the image")
 	buildCommand.Flags().StringP("output", "o", "", "Output destination (format: type=local,dest=path)")
 	buildCommand.Flags().String("progress", "auto", "Set type of progress output (auto, plain, tty). Use plain to show container output")
+	buildCommand.Flags().Bool("pull", false, "On true, always attempt to pull latest image version from remote. Default uses buildkit's default.")
 	buildCommand.Flags().StringArray("secret", nil, "Secret file to expose to the build: id=mysecret,src=/local/secret")
 	buildCommand.Flags().StringArray("allow", nil, "Allow extra privileged entitlement, e.g. network.host, security.insecure")
 	buildCommand.RegisterFlagCompletionFunc("allow", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
@@ -129,6 +130,14 @@ func processBuildCommandFlag(cmd *cobra.Command, args []string) (types.BuilderBu
 	if err != nil {
 		return types.BuilderBuildOptions{}, err
 	}
+	var pull *bool
+	if cmd.Flags().Changed("pull") {
+		pullFlag, err := cmd.Flags().GetBool("pull")
+		if err != nil {
+			return types.BuilderBuildOptions{}, err
+		}
+		pull = &pullFlag
+	}
 	secret, err := cmd.Flags().GetStringArray("secret")
 	if err != nil {
 		return types.BuilderBuildOptions{}, err
@@ -177,6 +186,7 @@ func processBuildCommandFlag(cmd *cobra.Command, args []string) (types.BuilderBu
 		BuildArgs:    buildArgs,
 		Label:        label,
 		NoCache:      noCache,
+		Pull:         pull,
 		Secret:       secret,
 		Allow:        allow,
 		SSH:          ssh,

cmd/nerdctl/builder_linux_test.go

@@ -20,6 +20,8 @@ import (
 	"bytes"
 	"fmt"
 	"os"
+	"os/exec"
+	"path/filepath"
 	"testing"
 
 	"github.com/containerd/nerdctl/pkg/testutil"
@@ -40,3 +42,92 @@ CMD ["echo", "nerdctl-builder-debug-test-string"]
 
 	base.Cmd("builder", "debug", buildCtx).CmdOption(testutil.WithStdin(bytes.NewReader([]byte("c\n")))).AssertOK()
 }
+
+func TestBuildWithPull(t *testing.T) {
+	testutil.DockerIncompatible(t)
+	testutil.RequiresBuild(t)
+
+	oldImage := testutil.BusyboxImage
+	oldImageSha := "141c253bc4c3fd0a201d32dc1f493bcf3fff003b6df416dea4f41046e0f37d47"
+	newImage := testutil.AlpineImage
+
+	buildkitConfig := fmt.Sprintf(`[worker.oci]
+enabled = false
+
+[worker.containerd]
+enabled = true
+namespace = "%s"`, testutil.Namespace)
+
+	cleanup := useBuildkitConfig(t, buildkitConfig)
+	defer cleanup()
+
+	testCases := []struct {
+		name string
+		pull string
+	}{
+		{
+			name: "build with local image",
+			pull: "false",
+		},
+		{
+			name: "build with newest image",
+			pull: "true",
+		},
+		{
+			name: "build with buildkit default",
+			// buildkit default pulls from remote
+			pull: "default",
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			base := testutil.NewBase(t)
+			defer base.Cmd("builder", "prune").AssertOK()
+			base.Cmd("image", "prune", "--force", "--all").AssertOK()
+
+			base.Cmd("pull", oldImage).Run()
+			base.Cmd("tag", oldImage, newImage).Run()
+
+			dockerfile := fmt.Sprintf(`FROM %s`, newImage)
+			tmpDir := t.TempDir()
+			err := os.WriteFile(filepath.Join(tmpDir, "Dockerfile"), []byte(dockerfile), 0644)
+			assert.NilError(t, err)
+
+			buildCtx, err := createBuildContext(dockerfile)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			buildCmd := []string{"build", buildCtx}
+			switch tc.pull {
+			case "false":
+				buildCmd = append(buildCmd, "--pull=false")
+				base.Cmd(buildCmd...).AssertErrContains(oldImageSha)
+			case "true":
+				buildCmd = append(buildCmd, "--pull=true")
+				base.Cmd(buildCmd...).AssertErrNotContains(oldImageSha)
+			case "default":
+				base.Cmd(buildCmd...).AssertErrNotContains(oldImageSha)
+			}
+		})
+	}
+}
+
+func useBuildkitConfig(t *testing.T, config string) (cleanup func()) {
+	buildkitConfigPath := "/etc/buildkit/buildkitd.toml"
+
+	currConfig, err := exec.Command("cat", buildkitConfigPath).Output()
+	assert.NilError(t, err)
+
+	os.WriteFile(buildkitConfigPath, []byte(config), 0644)
+	_, err = exec.Command("systemctl", "restart", "buildkit").Output()
+	assert.NilError(t, err)
+
+	return func() {
+		assert.NilError(t, os.WriteFile(buildkitConfigPath, currConfig, 0644))
+		_, err = exec.Command("systemctl", "restart", "buildkit").Output()
+		assert.NilError(t, err)
+	}
+}

docs/command-reference.md

@@ -680,6 +680,7 @@ Flags:
   - :whale: `type=tar[,dest=path/to/output.tar]`: Raw tar ball
   - :whale: `type=image,name=example.com/image,push=true`: Push to a registry (see [`buildctl build`](https://github.com/moby/buildkit/tree/v0.9.0#imageregistry) documentation)
 - :whale: `--progress=(auto|plain|tty)`: Set type of progress output (auto, plain, tty). Use plain to show container output
+- :whale: `--pull=(true|false)`: On true, always attempt to pull latest image version from remote. Default uses buildkit's default.
 - :whale: `--secret`: Secret file to expose to the build: id=mysecret,src=/local/secret
 - :whale: `--allow`: Allow extra privileged entitlement, e.g. network.host, security.insecure  (It’s required to configure the buildkitd to enable the feature, see [`buildkitd.toml`](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md) documentation)
 - :whale: `--ssh`: SSH agent socket or keys to expose to the build (format: `default|<id>[=<socket>|<key>[,<key>]]`)

pkg/api/types/builder_types.go

@@ -65,6 +65,8 @@ type BuilderBuildOptions struct {
 	BuildContext string
 	// NetworkMode mode for the build context
 	NetworkMode string
+	// Pull determines if we should try to pull latest image from remote. Default is buildkit's default.
+	Pull *bool
 }
 
 // BuilderPruneOptions specifies options for `nerdctl builder prune`.

pkg/cmd/builder/build.go

@@ -324,6 +324,15 @@ func generateBuildctlArgs(ctx context.Context, client *containerd.Client, option
 		buildctlArgs = append(buildctlArgs, "--no-cache")
 	}
 
+	if options.Pull != nil {
+		switch *options.Pull {
+		case true:
+			buildctlArgs = append(buildctlArgs, "--opt=image-resolve-mode=pull")
+		case false:
+			buildctlArgs = append(buildctlArgs, "--opt=image-resolve-mode=local")
+		}
+	}
+
 	for _, s := range strutil.DedupeStrSlice(options.Secret) {
 		buildctlArgs = append(buildctlArgs, "--secret="+s)
 	}

pkg/testutil/testutil.go

@@ -387,6 +387,14 @@ func (c *Cmd) AssertOutContains(s string) {
 	c.Assert(expected)
 }
 
+func (c *Cmd) AssertErrContains(s string) {
+	c.Base.T.Helper()
+	expected := icmd.Expected{
+		Err: s,
+	}
+	c.Assert(expected)
+}
+
 func (c *Cmd) AssertCombinedOutContains(s string) {
 	c.Base.T.Helper()
 	res := c.Run()
@@ -430,6 +438,15 @@ func (c *Cmd) AssertOutNotContains(s string) {
 	})
 }
 
+func (c *Cmd) AssertErrNotContains(s string) {
+	c.AssertOutWithFunc(func(stderr string) error {
+		if strings.Contains(stderr, s) {
+			return fmt.Errorf("expected stdout to not contain %q", s)
+		}
+		return nil
+	})
+}
+
 func (c *Cmd) AssertOutExactly(s string) {
 	c.Base.T.Helper()
 	fn := func(stdout string) error {