Skip to content

Commit 84367d9

Browse files
authored
hooks: implement kubectl deploy hooks (GoogleContainerTools#6376)
* hooks: implement kubectl deploy hooks * appease linters
1 parent 9dd4f24 commit 84367d9

File tree

11 files changed

+368
-22
lines changed

11 files changed

+368
-22
lines changed

docs/content/en/schemas/v2beta21.json

+30-1
Original file line numberDiff line numberDiff line change
@@ -956,6 +956,29 @@
956956
"description": "describes a dependency on another skaffold configuration.",
957957
"x-intellij-html-description": "describes a dependency on another skaffold configuration."
958958
},
959+
"ContainerHook": {
960+
"required": [
961+
"command"
962+
],
963+
"properties": {
964+
"command": {
965+
"items": {
966+
"type": "string"
967+
},
968+
"type": "array",
969+
"description": "command to execute.",
970+
"x-intellij-html-description": "command to execute.",
971+
"default": "[]"
972+
}
973+
},
974+
"preferredOrder": [
975+
"command"
976+
],
977+
"additionalProperties": false,
978+
"type": "object",
979+
"description": "describes a lifecycle hook definition to execute on a container. The container name is inferred from the scope in which this hook is defined.",
980+
"x-intellij-html-description": "describes a lifecycle hook definition to execute on a container. The container name is inferred from the scope in which this hook is defined."
981+
},
959982
"CustomArtifact": {
960983
"properties": {
961984
"buildCommand": {
@@ -2537,6 +2560,11 @@
25372560
"description": "additional flags passed to `kubectl`.",
25382561
"x-intellij-html-description": "additional flags passed to <code>kubectl</code>."
25392562
},
2563+
"hooks": {
2564+
"$ref": "#/definitions/DeployHooks",
2565+
"description": "describes a set of lifecycle hooks that are executed before and after every deploy.",
2566+
"x-intellij-html-description": "describes a set of lifecycle hooks that are executed before and after every deploy."
2567+
},
25402568
"manifests": {
25412569
"items": {
25422570
"type": "string"
@@ -2560,7 +2588,8 @@
25602588
"manifests",
25612589
"remoteManifests",
25622590
"flags",
2563-
"defaultNamespace"
2591+
"defaultNamespace",
2592+
"hooks"
25642593
],
25652594
"additionalProperties": false,
25662595
"type": "object",
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: hooks-example-deployment
5+
spec:
6+
replicas: 1
7+
selector:
8+
matchLabels:
9+
app: hooks
10+
template:
11+
metadata:
12+
labels:
13+
app: hooks
14+
spec:
15+
containers:
16+
- name: hooks-example
17+
image: hooks-example

integration/examples/lifecycle-hooks/k8s-pod.yaml

-8
This file was deleted.

integration/examples/lifecycle-hooks/skaffold.yaml

+18-1
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,21 @@ build:
3232
deploy:
3333
kubectl:
3434
manifests:
35-
- k8s-pod.yaml
35+
- deployment.yaml
36+
hooks:
37+
before:
38+
- host:
39+
command: ["sh", "-c", "echo pre-deploy host hook running on $(hostname)!"]
40+
os: [darwin, linux]
41+
- container:
42+
# this will only run when there's a matching container from a previous deploy iteration like in `skaffold dev`
43+
command: ["sh", "-c", "echo pre-deploy container hook running on $(hostname)!"]
44+
containerName: hooks-example*
45+
podName: hooks-example-deployment*
46+
after:
47+
- host:
48+
command: ["sh", "-c", "echo post-deploy host hook running on $(hostname)!"]
49+
- container:
50+
command: ["sh", "-c", "echo post-deploy container hook running on $(hostname)!"]
51+
containerName: hooks-example* # use a glob pattern to prefix-match the container name and pod name for deployments, stateful-sets, etc.
52+
podName: hooks-example-deployment*

pkg/skaffold/deploy/deploy_mux.go

+19-2
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ type DeployerMux struct {
4545
deployers []Deployer
4646
}
4747

48+
type deployerWithHooks interface {
49+
PreDeployHooks(context.Context, io.Writer) error
50+
PostDeployHooks(context.Context, io.Writer) error
51+
}
52+
4853
func NewDeployerMux(deployers []Deployer, iterativeStatusCheck bool) Deployer {
4954
return DeployerMux{deployers: deployers, iterativeStatusCheck: iterativeStatusCheck}
5055
}
@@ -104,19 +109,31 @@ func (m DeployerMux) Deploy(ctx context.Context, w io.Writer, as []graph.Artifac
104109
eventV2.DeployInProgress(i)
105110
w = output.WithEventContext(w, constants.Deploy, strconv.Itoa(i))
106111
ctx, endTrace := instrumentation.StartTrace(ctx, "Deploy")
107-
112+
deployHooks, ok := deployer.(deployerWithHooks)
113+
if ok {
114+
if err := deployHooks.PreDeployHooks(ctx, w); err != nil {
115+
return err
116+
}
117+
}
108118
if err := deployer.Deploy(ctx, w, as); err != nil {
109119
eventV2.DeployFailed(i, err)
110120
endTrace(instrumentation.TraceEndError(err))
111121
return err
112122
}
113-
if m.iterativeStatusCheck {
123+
// Always run iterative status check if there are deploy hooks.
124+
// This is required otherwise the deploy hooks can get erreneously executed on older pods from a previous deployment.
125+
if ok || m.iterativeStatusCheck {
114126
if err := deployer.GetStatusMonitor().Check(ctx, w); err != nil {
115127
eventV2.DeployFailed(i, err)
116128
endTrace(instrumentation.TraceEndError(err))
117129
return err
118130
}
119131
}
132+
if ok {
133+
if err := deployHooks.PostDeployHooks(ctx, w); err != nil {
134+
return err
135+
}
136+
}
120137
eventV2.DeploySucceeded(i)
121138
endTrace()
122139
}

pkg/skaffold/deploy/kubectl/kubectl.go

+30-7
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,10 @@ import (
3838
deployutil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy/util"
3939
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/event"
4040
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph"
41+
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/hooks"
4142
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation"
4243
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
44+
k8slogger "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/logger"
4345
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/manifest"
4446
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/loader"
4547
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/log"
@@ -54,13 +56,13 @@ import (
5456
type Deployer struct {
5557
*latestV1.KubectlDeploy
5658

57-
accessor access.Accessor
58-
imageLoader loader.ImageLoader
59-
logger log.Logger
60-
debugger debug.Debugger
61-
statusMonitor status.Monitor
62-
syncer sync.Syncer
63-
59+
accessor access.Accessor
60+
imageLoader loader.ImageLoader
61+
logger k8slogger.Logger
62+
debugger debug.Debugger
63+
statusMonitor status.Monitor
64+
syncer sync.Syncer
65+
hookRunner hooks.Runner
6466
originalImages []graph.Artifact // the set of images marked as "local" by the Runner
6567
localImages []graph.Artifact // the set of images parsed from the Deployer's manifest set
6668
podSelector *kubernetes.ImageList
@@ -107,6 +109,7 @@ func NewDeployer(cfg Config, labeller *label.DefaultLabeller, d *latestV1.Kubect
107109
logger: logger,
108110
statusMonitor: component.NewMonitor(cfg, cfg.GetKubeContext(), labeller, &namespaces),
109111
syncer: component.NewSyncer(kubectl.CLI, &namespaces, logger.GetFormatter()),
112+
hookRunner: hooks.NewDeployRunner(kubectl.CLI, d.LifecycleHooks, namespaces, logger.GetFormatter(), hooks.NewDeployEnvOpts(labeller.GetRunID(), kubectl.KubeContext, namespaces)),
110113
workingDir: cfg.GetWorkingDir(),
111114
globalConfig: cfg.GlobalConfig(),
112115
defaultRepo: cfg.DefaultRepo(),
@@ -236,6 +239,26 @@ func (k *Deployer) Deploy(ctx context.Context, out io.Writer, builds []graph.Art
236239
return nil
237240
}
238241

242+
func (k *Deployer) PreDeployHooks(ctx context.Context, out io.Writer) error {
243+
childCtx, endTrace := instrumentation.StartTrace(ctx, "Deploy_PreHooks")
244+
if err := k.hookRunner.RunPreHooks(childCtx, out); err != nil {
245+
endTrace(instrumentation.TraceEndError(err))
246+
return err
247+
}
248+
endTrace()
249+
return nil
250+
}
251+
252+
func (k *Deployer) PostDeployHooks(ctx context.Context, out io.Writer) error {
253+
childCtx, endTrace := instrumentation.StartTrace(ctx, "Deploy_PostHooks")
254+
if err := k.hookRunner.RunPostHooks(childCtx, out); err != nil {
255+
endTrace(instrumentation.TraceEndError(err))
256+
return err
257+
}
258+
endTrace()
259+
return nil
260+
}
261+
239262
func (k *Deployer) manifestFiles(manifests []string) ([]string, error) {
240263
var nonURLManifests, gcsManifests []string
241264
for _, manifest := range manifests {

pkg/skaffold/hooks/container.go

+27
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"fmt"
2222
"io"
23+
"path"
2324

2425
"golang.org/x/sync/errgroup"
2526
v1 "k8s.io/api/core/v1"
@@ -52,6 +53,32 @@ func runningImageSelector(image string) containerSelector {
5253
}
5354
}
5455

56+
// namePatternSelector chooses containers that match the glob patterns for pod and container names
57+
func namePatternSelector(podName, containerName string) containerSelector {
58+
return func(p v1.Pod, c v1.Container) (bool, error) {
59+
if p.Status.Phase != v1.PodRunning {
60+
return false, nil
61+
}
62+
for _, status := range p.Status.ContainerStatuses {
63+
if status.Name == c.Name && status.State.Running == nil {
64+
return false, nil
65+
}
66+
}
67+
if matched, err := path.Match(podName, p.Name); err != nil {
68+
return false, fmt.Errorf("failed to evaluate pod name pattern %q due to error %w", podName, err)
69+
} else if podName != "" && !matched {
70+
return false, nil
71+
}
72+
73+
if matched, err := path.Match(containerName, c.Name); err != nil {
74+
return false, fmt.Errorf("failed to evaluate container name pattern %q due to error %w", containerName, err)
75+
} else if containerName != "" && !matched {
76+
return false, nil
77+
}
78+
return true, nil
79+
}
80+
}
81+
5582
// containerHook represents a lifecycle hook to be executed inside a running container
5683
type containerHook struct {
5784
cfg latestV1.ContainerHook

pkg/skaffold/hooks/deploy.go

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
/*
2+
Copyright 2021 The Skaffold Authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package hooks
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"io"
23+
"strings"
24+
"sync"
25+
26+
corev1 "k8s.io/api/core/v1"
27+
28+
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubectl"
29+
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/logger"
30+
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/output"
31+
v1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
32+
)
33+
34+
func NewDeployRunner(cli *kubectl.CLI, d v1.DeployHooks, namespaces []string, formatter logger.Formatter, opts DeployEnvOpts) Runner {
35+
return deployRunner{d, cli, namespaces, formatter, opts, new(sync.Map)}
36+
}
37+
38+
func NewDeployEnvOpts(runID string, kubeContext string, namespaces []string) DeployEnvOpts {
39+
return DeployEnvOpts{
40+
RunID: runID,
41+
KubeContext: kubeContext,
42+
Namespaces: strings.Join(namespaces, ","),
43+
}
44+
}
45+
46+
type deployRunner struct {
47+
v1.DeployHooks
48+
cli *kubectl.CLI
49+
namespaces []string
50+
formatter logger.Formatter
51+
opts DeployEnvOpts
52+
visitedPods *sync.Map // maintain a list of previous iteration pods, so that they can be skipped
53+
}
54+
55+
func (r deployRunner) RunPreHooks(ctx context.Context, out io.Writer) error {
56+
return r.run(ctx, out, r.PreHooks, phases.PreDeploy)
57+
}
58+
59+
func (r deployRunner) RunPostHooks(ctx context.Context, out io.Writer) error {
60+
return r.run(ctx, out, r.PostHooks, phases.PostDeploy)
61+
}
62+
63+
func (r deployRunner) getEnv() []string {
64+
common := getEnv(staticEnvOpts)
65+
deploy := getEnv(r.opts)
66+
return append(common, deploy...)
67+
}
68+
69+
func (r deployRunner) run(ctx context.Context, out io.Writer, hooks []v1.DeployHookItem, phase phase) error {
70+
if len(hooks) > 0 {
71+
output.Default.Fprintln(out, fmt.Sprintf("Starting %s hooks...", phase))
72+
}
73+
env := r.getEnv()
74+
for _, h := range hooks {
75+
if h.HostHook != nil {
76+
hook := hostHook{*h.HostHook, env}
77+
if err := hook.run(ctx, out); err != nil {
78+
return err
79+
}
80+
} else if h.ContainerHook != nil {
81+
hook := containerHook{
82+
cfg: v1.ContainerHook{Command: h.ContainerHook.Command},
83+
cli: r.cli,
84+
selector: filterPodsSelector(r.visitedPods, phase, namePatternSelector(h.ContainerHook.PodName, h.ContainerHook.ContainerName)),
85+
namespaces: r.namespaces,
86+
formatter: r.formatter,
87+
}
88+
if err := hook.run(ctx, out); err != nil {
89+
return err
90+
}
91+
}
92+
}
93+
if len(hooks) > 0 {
94+
output.Default.Fprintln(out, fmt.Sprintf("Completed %s hooks", phase))
95+
}
96+
return nil
97+
}
98+
99+
// filterPodsSelector filters the pods that have already been processed from a previous deploy iteration
100+
func filterPodsSelector(visitedPods *sync.Map, phase phase, selector containerSelector) containerSelector {
101+
return func(p corev1.Pod, c corev1.Container) (bool, error) {
102+
key := fmt.Sprintf("%s:%s", phase, p.GetName())
103+
if _, found := visitedPods.LoadOrStore(key, struct{}{}); found {
104+
return false, nil
105+
}
106+
return selector(p, c)
107+
}
108+
}

0 commit comments

Comments
 (0)