Merge branch 'master' into render-4-validate

Author: yuwenma

cmd/skaffold/app/cmd/cmd.go

@@ -284,14 +284,14 @@ func isHouseKeepingMessagesAllowed(cmd *cobra.Command) bool {
 	if cmd.Annotations == nil {
 		return false
 	}
-	return cmd.Annotations[HouseKeepingMessagesAllowedAnnotation] == "true"
+	return cmd.Annotations[HouseKeepingMessagesAllowedAnnotation] == fmt.Sprintf("%t", true)
 }
 
 func allowHouseKeepingMessages(cmd *cobra.Command) {
 	if cmd.Annotations == nil {
 		cmd.Annotations = make(map[string]string)
 	}
-	cmd.Annotations[HouseKeepingMessagesAllowedAnnotation] = "true"
+	cmd.Annotations[HouseKeepingMessagesAllowedAnnotation] = fmt.Sprintf("%t", true)
 }
 
 func preReleaseVersion(s string) bool {

cmd/skaffold/app/cmd/inspect_build_env.go

@@ -23,6 +23,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/inspect"
 	buildEnv "github.com/GoogleContainerTools/skaffold/pkg/skaffold/inspect/buildEnv"
 )
@@ -34,6 +35,12 @@ var buildEnvFlags = struct {
 	timeout     string
 	concurrency int
 
+	// Local
+	push             config.BoolOrUndefined
+	tryImportMissing config.BoolOrUndefined
+	useDockerCLI     config.BoolOrUndefined
+	useBuildkit      config.BoolOrUndefined
+
 	// Google Cloud Build
 	projectID   string
 	diskSizeGb  int64
@@ -74,7 +81,7 @@ func cmdBuildEnvAdd() *cobra.Command {
 	return NewCmd("add").
 		WithDescription("Add a new build environment to the default pipeline or to a new or existing profile.").
 		WithPersistentFlagAdder(cmdBuildEnvAddFlags).
-		WithCommands(cmdBuildEnvAddGcb(), cmdBuildEnvAddCluster())
+		WithCommands(cmdBuildEnvAddLocal(), cmdBuildEnvAddGcb(), cmdBuildEnvAddCluster())
 }
 
 func cmdBuildEnvAddGcb() *cobra.Command {
@@ -89,6 +96,18 @@ Use the '--module' filter to specify the individual module to target. Otherwise,
 		NoArgs(addGcbBuildEnv)
 }
 
+func cmdBuildEnvAddLocal() *cobra.Command {
+	return NewCmd("local").
+		WithDescription("Add a new Local build environment definition").
+		WithLongDescription(`Add a new Local build environment definition.
+Without the '--profile' flag the new environment definition is added to the default pipeline. With the '--profile' flag it will create a new profile with this build env definition. 
+In these respective scenarios, it will fail if the build env definition for the default pipeline or the named profile already exists. To override an existing definition use 'skaffold inspect build-env modify' command instead. 
+Use the '--module' filter to specify the individual module to target. Otherwise, it'll be applied to all modules defined in the target file. Also, with the '--profile' flag if the target config imports other configs as dependencies, then the new profile will be recursively created in all the imported configs also.`).
+		WithExample("Add a new profile named 'local' targeting the local build environment with option to push images and using buildkit", "inspect build-env add local --profile local --push true --useBuildkit true -f skaffold.yaml").
+		WithFlagAdder(cmdBuildEnvLocalFlags).
+		NoArgs(addLocalBuildEnv)
+}
+
 func cmdBuildEnvAddCluster() *cobra.Command {
 	return NewCmd("cluster").
 		WithDescription("Add a new Cluster build environment definition").
@@ -105,6 +124,10 @@ func listBuildEnv(ctx context.Context, out io.Writer) error {
 	return buildEnv.PrintBuildEnvsList(ctx, out, printBuildEnvsListOptions())
 }
 
+func addLocalBuildEnv(ctx context.Context, out io.Writer) error {
+	return buildEnv.AddLocalBuildEnv(ctx, out, localBuildEnvOptions())
+}
+
 func addGcbBuildEnv(ctx context.Context, out io.Writer) error {
 	return buildEnv.AddGcbBuildEnv(ctx, out, addGcbBuildEnvOptions())
 }
@@ -117,6 +140,20 @@ func cmdBuildEnvAddFlags(f *pflag.FlagSet) {
 	f.StringVarP(&buildEnvFlags.profile, "profile", "p", "", `Profile name to add the new build env definition in. If the profile name doesn't exist then the profile will be created in all the target configs. If this flag is not specified then the build env is added to the default pipeline of the target configs.`)
 }
 
+func cmdBuildEnvLocalFlags(f *pflag.FlagSet) {
+	var flags []*pflag.Flag
+	flags = append(flags, f.VarPF(&buildEnvFlags.push, "push", "", `Set to true to push images to a registry`))
+	flags = append(flags, f.VarPF(&buildEnvFlags.tryImportMissing, "tryImportMissing", "", `Set to true to to attempt importing artifacts from Docker (either a local or remote registry) if not in the build cache`))
+	flags = append(flags, f.VarPF(&buildEnvFlags.useDockerCLI, "useDockerCLI", "", `Set to true to use 'docker' command-line interface instead of Docker Engine APIs`))
+	flags = append(flags, f.VarPF(&buildEnvFlags.useBuildkit, "useBuildkit", "", `Set to true to use BuildKit to build Docker images`))
+	f.IntVar(&buildEnvFlags.concurrency, "concurrency", -1, `number of artifacts to build concurrently. 0 means "no-limit"`)
+
+	// support *bool flags without a value to be interpreted as `true`; like `--push` instead of `--push=true`
+	for _, f := range flags {
+		f.NoOptDefVal = "true"
+	}
+}
+
 func cmdBuildEnvAddGcbFlags(f *pflag.FlagSet) {
 	f.StringVar(&buildEnvFlags.projectID, "projectId", "", `ID of the Cloud Platform Project.`)
 	f.Int64Var(&buildEnvFlags.diskSizeGb, "diskSizeGb", 0, `Disk size of the VM that runs the build`)
@@ -162,6 +199,23 @@ func printBuildEnvsListOptions() inspect.Options {
 		},
 	}
 }
+
+func localBuildEnvOptions() inspect.Options {
+	return inspect.Options{
+		Filename:  inspectFlags.fileName,
+		OutFormat: inspectFlags.outFormat,
+		Modules:   inspectFlags.modules,
+		BuildEnvOptions: inspect.BuildEnvOptions{
+			Profile:          buildEnvFlags.profile,
+			Push:             buildEnvFlags.push.Value(),
+			TryImportMissing: buildEnvFlags.tryImportMissing.Value(),
+			UseDockerCLI:     buildEnvFlags.useDockerCLI.Value(),
+			UseBuildkit:      buildEnvFlags.useBuildkit.Value(),
+			Concurrency:      buildEnvFlags.concurrency,
+		},
+	}
+}
+
 func addGcbBuildEnvOptions() inspect.Options {
 	return inspect.Options{
 		Filename:  inspectFlags.fileName,
@@ -177,6 +231,7 @@ func addGcbBuildEnvOptions() inspect.Options {
 		},
 	}
 }
+
 func addClusterBuildEnvOptions() inspect.Options {
 	return inspect.Options{
 		Filename:  inspectFlags.fileName,

docs/content/en/docs/design/_index.md

@@ -23,6 +23,7 @@ provides built-in support for the following tools:
 * **Tag**
   * Git tagger
   * Sha256 tagger
+  * Input Digest tagger
   * Env Template tagger
   * DateTime tagger
 * **Deploy**

docs/content/en/docs/environment/templating.md

@@ -23,6 +23,7 @@ List of fields that support templating:
 * `deploy.kubectl.defaultNamespace`
 * `deploy.kustomize.defaultNamespace`
 * `portForward.namespace`
+* `portForward.resourceName`
 
 _Please note, this list is not exhaustive._
 

docs/content/en/docs/pipeline-stages/taggers.md

@@ -8,14 +8,17 @@ aliases: [/docs/how-tos/taggers]
 
 Skaffold supports multiple taggers or tag policies for tagging images:
 
- + the `gitCommit` tagger uses git commits/references to tag images.
- + the `sha256` tagger uses `latest` to tag images.
- + the `envTemplate` tagger uses environment variables to tag images.
+ + the `gitCommit` tagger uses git commits/references.
+ + the `inputDigest` tagger uses a digest of the artifact source files.
+ + the `envTemplate` tagger uses environment variables.
  + the `datetime` tagger uses current date and time, with a configurable pattern.
  + the `customTemplate` tagger uses a combination of the existing taggers as components in a template.
+ + the `sha256` tagger uses `latest`.
 
 The default tagger, if none is specified in the `skaffold.yaml`, is the `gitCommit` tagger.
 
+The tags can be overridden with a fixed tag with the `--tag` option on the command-line.
+
 ### Configuration
 
 The tag policy is specified in the `tagPolicy` field of the `build` section
@@ -68,22 +71,24 @@ specified explicitly:
 
 {{< schema root="GitTagger" >}}
 
-## `sha256`: uses `latest` to tag images
 
-`sha256` is a misleading name. It is named like that because, in the end, when Skaffold
-deploys to a remote cluster, the image's `sha256` digest is used in addition to `:latest`
-in order to create an immutable image reference.
+## `inputDigest`: uses a digest of the artifact source to tag images
+
+The `inputDigest` tagger tags images with a digest of the artifact
+source files.  The source files are the dependencies calculated by the
+configured builder.
 
 ### Example
 
 The following `build` section instructs Skaffold to build a
-Docker image `gcr.io/k8s-skaffold/example` with the `sha256` tag policy:
+Docker image `gcr.io/k8s-skaffold/example` with the `inputDigest` tag policy:
 
-{{% readfile file="samples/taggers/sha256.yaml" %}}
+{{% readfile file="samples/taggers/inputDigest.yaml" %}}
 
 ### Configuration
 
-`sha256` tag policy features no options.
+`inputDigest` tag policy features no options.
+
 
 ## `envTemplate`: uses values of environment variables as tags
 
@@ -176,4 +181,21 @@ Suppose the current time is `15:04:09.999 January 2nd, 2006` and the abbreviated
 
 The tag template uses the [Golang Templating Syntax](https://golang.org/pkg/text/template/).
 As showcased in the example, `customTemplate` tag policy features one
-**required** parameter, `template`, which is the tag template to use. To learn more about templating support in the skaffold.yaml, see [Templated fields]({{< relref "../environment/templating.md" >}})
+**required** parameter, `template`, which is the tag template to use. To learn more about templating support in the skaffold.yaml, see [Templated fields]({{< relref "../environment/templating.md" >}})
+
+## `sha256`: uses `latest` to tag images
+
+`sha256` is a misleading name. It is named like that because, in the end, when Skaffold
+deploys to a remote cluster, the image's `sha256` digest is used in addition to `:latest`
+in order to create an immutable image reference.
+
+### Example
+
+The following `build` section instructs Skaffold to build a
+Docker image `gcr.io/k8s-skaffold/example` with the `sha256` tag policy:
+
+{{% readfile file="samples/taggers/sha256.yaml" %}}
+
+### Configuration
+
+`sha256` tag policy features no options.

docs/content/en/samples/taggers/inputDigest.yaml

@@ -0,0 +1,5 @@
+build:
+  tagPolicy:
+    inputDigest: {}
+  artifacts:
+  - image: gcr.io/k8s-skaffold/example

docs/content/en/schemas/v2beta17.json

@@ -1689,14 +1689,14 @@
         },
         "skipBuildDependencies": {
           "type": "boolean",
-          "description": "should build dependencies be skipped. Ignored when `remote: true`.",
-          "x-intellij-html-description": "should build dependencies be skipped. Ignored when <code>remote: true</code>.",
+          "description": "should build dependencies be skipped. Ignored for `remoteChart`.",
+          "x-intellij-html-description": "should build dependencies be skipped. Ignored for <code>remoteChart</code>.",
           "default": "false"
         },
         "upgradeOnChange": {
           "type": "boolean",
-          "description": "specifies whether to upgrade helm chart on code changes. Default is `true` when helm chart is local (`remote: false`). Default is `false` if `remote: true`.",
-          "x-intellij-html-description": "specifies whether to upgrade helm chart on code changes. Default is <code>true</code> when helm chart is local (<code>remote: false</code>). Default is <code>false</code> if <code>remote: true</code>."
+          "description": "specifies whether to upgrade helm chart on code changes. Default is `true` when helm chart is local (has `chartPath`). Default is `false` when helm chart is remote (has `remoteChart`).",
+          "x-intellij-html-description": "specifies whether to upgrade helm chart on code changes. Default is <code>true</code> when helm chart is local (has <code>chartPath</code>). Default is <code>false</code> when helm chart is remote (has <code>remoteChart</code>)."
         },
         "useHelmSecrets": {
           "type": "boolean",

pkg/skaffold/build/build_problems.go

@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"regexp"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
 	sErrors "github.com/GoogleContainerTools/skaffold/pkg/skaffold/errors"
@@ -38,14 +40,16 @@ const (
 )
 
 var (
-	nilSuggestions = func(cfg interface{}) []*proto.Suggestion { return nil }
+	unknownProjectErr = fmt.Sprintf(".*(%s.* unknown: Project.*)", PushImageErr)
+	nilSuggestions    = func(cfg interface{}) []*proto.Suggestion { return nil }
 	// for testing
 	getConfigForCurrentContext = config.GetConfigForCurrentKubectx
 	problems                   = []sErrors.Problem{
 		{
 			Regexp:  re(fmt.Sprintf(".*%s.* denied: .*", PushImageErr)),
 			ErrCode: proto.StatusCode_BUILD_PUSH_ACCESS_DENIED,
-			Description: func(error) string {
+			Description: func(err error) string {
+				logrus.Tracef("error building %s", err)
 				return "Build Failed. No push access to specified image repository"
 			},
 			Suggestion: suggestBuildPushAccessDeniedAction,
@@ -59,9 +63,14 @@ var (
 			Suggestion: nilSuggestions,
 		},
 		{
-			Regexp: re(fmt.Sprintf(".*%s.* unknown: Project", PushImageErr)),
-			Description: func(error) string {
-				return "Build Failed"
+			Regexp: re(unknownProjectErr),
+			Description: func(err error) string {
+				logrus.Tracef("error building %s", err)
+				matchExp := re(unknownProjectErr)
+				if match := matchExp.FindStringSubmatch(err.Error()); len(match) >= 2 {
+					return fmt.Sprintf("Build Failed. %s", match[1])
+				}
+				return fmt.Sprintf("Build Failed due to %s", err)
 			},
 			ErrCode: proto.StatusCode_BUILD_PROJECT_NOT_FOUND,
 			Suggestion: func(interface{}) []*proto.Suggestion {
@@ -75,6 +84,7 @@ var (
 			Regexp:  re(dockerConnectionFailed),
 			ErrCode: proto.StatusCode_BUILD_DOCKER_DAEMON_NOT_RUNNING,
 			Description: func(err error) string {
+				logrus.Tracef("error building %s", err)
 				matchExp := re(dockerConnectionFailed)
 				if match := matchExp.FindStringSubmatch(err.Error()); len(match) >= 2 {
 					return fmt.Sprintf("Build Failed. %s", match[1])

pkg/skaffold/build/build_problems_test.go

@@ -106,11 +106,11 @@ func TestBuildProblems(t *testing.T) {
 		},
 		{
 			description: "unknown project error",
-			err:         fmt.Errorf("build failed: could not push image: unknown: Project"),
-			expected:    "Build Failed. Check your GCR project.",
+			err:         fmt.Errorf("build failed: could not push image: unknown: Project test"),
+			expected:    "Build Failed. could not push image: unknown: Project test. Check your GCR project.",
 			expectedAE: &proto.ActionableErr{
 				ErrCode: proto.StatusCode_BUILD_PROJECT_NOT_FOUND,
-				Message: "build failed: could not push image: unknown: Project",
+				Message: "build failed: could not push image: unknown: Project test",
 				Suggestions: []*proto.Suggestion{{
 					SuggestionCode: proto.SuggestionCode_CHECK_GCLOUD_PROJECT,
 					Action:         "Check your GCR project",

pkg/skaffold/event/v2/event.go

@@ -290,16 +290,17 @@ func AutoTriggerDiff(phase constants.Phase, val bool) (bool, error) {
 	}
 }
 
-func TaskInProgress(task constants.Phase) {
+func TaskInProgress(task constants.Phase, description string) {
 	if task == constants.DevLoop {
 		handler.iteration++
 	}
 
 	handler.handleTaskEvent(&proto.TaskEvent{
-		Id:        fmt.Sprintf("%s-%d", task, handler.iteration),
-		Task:      string(task),
-		Iteration: int32(handler.iteration),
-		Status:    InProgress,
+		Id:          fmt.Sprintf("%s-%d", task, handler.iteration),
+		Task:        string(task),
+		Description: description,
+		Iteration:   int32(handler.iteration),
+		Status:      InProgress,
 	})
 }