Skip to content

Commit 0442ef7

Browse files
willthameswillthames
committed
Add wait functionality for deployments and daemonsets
Allow terraform to correctly report whether a deployment or daemonset has timed out becoming ready (where ready means that all pods are up to date relative to the spec)
1 parent e7bcd53 commit 0442ef7

File tree

12 files changed

+361
-4
lines changed

12 files changed

+361
-4
lines changed

‎docs/resources/resource.md

+8-1
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ resource "kustomization_resource" "p0" {
6868
6969
# then loop through resources in ids_prio[1]
7070
# and set an explicit depends_on on kustomization_resource.p0
71+
# wait 2 minutes for any deployment or daemonset to become ready
7172
resource "kustomization_resource" "p1" {
7273
for_each = data.kustomization_build.test.ids_prio[1]
7374
@@ -76,6 +77,11 @@ resource "kustomization_resource" "p1" {
7677
? sensitive(data.kustomization_build.test.manifests[each.value])
7778
: data.kustomization_build.test.manifests[each.value]
7879
)
80+
wait = true
81+
timeouts {
82+
create = "2m"
83+
update = "2m"
84+
}
7985
8086
depends_on = [kustomization_resource.p0]
8187
}
@@ -99,4 +105,5 @@ resource "kustomization_resource" "p2" {
99105
## Argument Reference
100106

101107
- `manifest` - (Required) JSON encoded Kubernetes resource manifest.
102-
- 'timeouts' - (Optional) Overwrite `create` or `delete` timeout defaults. Defaults are 5 minutes for `create` and 10 minutes for `delete`.
108+
- `wait` - Whether to wait for pods to become ready (default false). Currently only has an effect for Deployments and DaemonSets.
109+
- 'timeouts' - (Optional) Overwrite `create`, `update` or `delete` timeout defaults. Defaults are 5 minutes for `create` and `update` and 10 minutes for `delete`.

‎kustomize/manifest.go

+95
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@ import (
99

1010
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
1111

12+
k8sappsv1 "k8s.io/api/apps/v1"
1213
k8serrors "k8s.io/apimachinery/pkg/api/errors"
1314
k8smeta "k8s.io/apimachinery/pkg/api/meta"
1415
k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
1517
k8sunstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1618
k8sruntime "k8s.io/apimachinery/pkg/runtime"
1719
k8sschema "k8s.io/apimachinery/pkg/runtime/schema"
@@ -20,13 +22,20 @@ import (
2022
"k8s.io/client-go/restmapper"
2123
)
2224

25+
var waitRefreshFunctions = map[string]waitRefreshFunction{
26+
"apps/Deployment": waitDeploymentRefresh,
27+
"apps/Daemonset": waitDaemonsetRefresh,
28+
}
29+
2330
type kManifestId struct {
2431
group string
2532
kind string
2633
namespace string
2734
name string
2835
}
2936

37+
type waitRefreshFunction func(km *kManifest) (interface{}, string, error)
38+
3039
func mustParseProviderId(str string) *kManifestId {
3140
kr, err := parseProviderId(str)
3241
if err != nil {
@@ -354,6 +363,92 @@ func (km *kManifest) waitDeleted(t time.Duration) error {
354363
return nil
355364
}
356365

366+
func daemonsetReady(u *k8sunstructured.Unstructured) (bool, error) {
367+
var daemonset k8sappsv1.DaemonSet
368+
if err := k8sruntime.DefaultUnstructuredConverter.FromUnstructured(u.UnstructuredContent(), &daemonset); err != nil {
369+
return false, err
370+
}
371+
if daemonset.Generation == daemonset.Status.ObservedGeneration &&
372+
daemonset.Status.UpdatedNumberScheduled == daemonset.Status.DesiredNumberScheduled &&
373+
daemonset.Status.NumberReady == daemonset.Status.DesiredNumberScheduled &&
374+
daemonset.Status.NumberUnavailable == 0 {
375+
return true, nil
376+
} else {
377+
return false, nil
378+
}
379+
}
380+
381+
func waitDaemonsetRefresh(km *kManifest) (interface{}, string, error) {
382+
resp, err := km.apiGet(k8smetav1.GetOptions{})
383+
if err != nil {
384+
if k8serrors.IsNotFound(err) {
385+
return nil, "missing", nil
386+
}
387+
return nil, "error", err
388+
}
389+
ready, err := daemonsetReady(resp)
390+
if err != nil {
391+
return nil, "error", err
392+
}
393+
if ready {
394+
return resp, "done", nil
395+
}
396+
return nil, "in progress", nil
397+
}
398+
399+
func deploymentReady(u *k8sunstructured.Unstructured) (bool, error) {
400+
var deployment k8sappsv1.Deployment
401+
if err := k8sruntime.DefaultUnstructuredConverter.FromUnstructured(u.UnstructuredContent(), &deployment); err != nil {
402+
return false, err
403+
}
404+
if deployment.Generation == deployment.Status.ObservedGeneration &&
405+
deployment.Status.AvailableReplicas == *deployment.Spec.Replicas &&
406+
deployment.Status.AvailableReplicas == deployment.Status.Replicas &&
407+
deployment.Status.UnavailableReplicas == 0 {
408+
return true, nil
409+
} else {
410+
return false, nil
411+
}
412+
}
413+
414+
func waitDeploymentRefresh(km *kManifest) (interface{}, string, error) {
415+
resp, err := km.apiGet(k8smetav1.GetOptions{})
416+
if err != nil {
417+
if k8serrors.IsNotFound(err) {
418+
return nil, "missing", nil
419+
}
420+
return nil, "error", err
421+
}
422+
ready, err := deploymentReady(resp)
423+
if err != nil {
424+
return nil, "error", err
425+
}
426+
if ready {
427+
return resp, "done", nil
428+
}
429+
return nil, "in progress", nil
430+
}
431+
432+
func (km *kManifest) waitCreatedOrUpdated(t time.Duration) error {
433+
gvk := km.gvk()
434+
if refresh, ok := waitRefreshFunctions[fmt.Sprintf("%s/%s", gvk.Group, gvk.Kind)]; ok {
435+
stateConf := &resource.StateChangeConf{
436+
Target: []string{"done"},
437+
Pending: []string{"in progress"},
438+
Timeout: t,
439+
Refresh: func() (interface{}, string, error) {
440+
return refresh(km)
441+
},
442+
}
443+
444+
_, err := stateConf.WaitForState()
445+
if err != nil {
446+
return km.fmtErr(fmt.Errorf("timed out creating/updating %s %s/%s: %s", gvk.Kind, km.namespace(), km.name(), err))
447+
}
448+
}
449+
return nil
450+
}
451+
357452
func (km *kManifest) fmtErr(err error) error {
358453
return fmt.Errorf(
359454
"%q: %s",

‎kustomize/resource_kustomization.go

+19-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import (
1414
k8smeta "k8s.io/apimachinery/pkg/api/meta"
1515
k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1616
k8sschema "k8s.io/apimachinery/pkg/runtime/schema"
17-
"k8s.io/apimachinery/pkg/util/validation/field"
17+
"k8s.io/apimachinery/pkg/util/validation/field"
1818
)
1919

2020
func kustomizationResource() *schema.Resource {
@@ -35,10 +35,16 @@ func kustomizationResource() *schema.Resource {
3535
Type: schema.TypeString,
3636
Required: true,
3737
},
38+
"wait": &schema.Schema{
39+
Type: schema.TypeBool,
40+
Default: false,
41+
Optional: true,
42+
},
3843
},
3944

4045
Timeouts: &schema.ResourceTimeout{
4146
Create: schema.DefaultTimeout(5 * time.Minute),
47+
Update: schema.DefaultTimeout(5 * time.Minute),
4248
Delete: schema.DefaultTimeout(10 * time.Minute),
4349
},
4450
}
@@ -103,6 +109,12 @@ func kustomizationResourceCreate(d *schema.ResourceData, m interface{}) error {
103109
return logError(err)
104110
}
105111

112+
if d.Get("wait").(bool) {
113+
if err = km.waitCreatedOrUpdated(d.Timeout(schema.TimeoutCreate)); err != nil {
114+
return logError(err)
115+
}
116+
}
117+
106118
id := string(resp.GetUID())
107119
d.SetId(id)
108120

@@ -327,6 +339,12 @@ func kustomizationResourceUpdate(d *schema.ResourceData, m interface{}) error {
327339
return logError(err)
328340
}
329341

342+
if d.Get("wait").(bool) {
343+
if err = kmm.waitCreatedOrUpdated(d.Timeout(schema.TimeoutUpdate)); err != nil {
344+
return logError(err)
345+
}
346+
}
347+
330348
id := string(resp.GetUID())
331349
d.SetId(id)
332350

‎kustomize/resource_kustomization_test.go

+169-2
Original file line numberDiff line numberDiff line change
@@ -490,8 +490,131 @@ resource "kustomization_resource" "scprov" {
490490
`
491491
}
492492

493-
//
494-
//
493+
func TestAccResourceKustomization_wait(t *testing.T) {
494+
495+
resource.Test(t, resource.TestCase{
496+
//PreCheck: func() { testAccPreCheck(t) },
497+
Providers: testAccProviders,
498+
Steps: []resource.TestStep{
499+
//
500+
//
501+
// Applying initial config with a svc and deployment in a namespace with wait
502+
{
503+
Config: testAccResourceKustomizationConfig_wait("test_kustomizations/wait/initial"),
504+
Check: resource.ComposeAggregateTestCheckFunc(
505+
testAccCheckManifestNestedString("kustomization_resource.dep1", "test", "spec", "selector", "matchLabels", "app"),
506+
testAccCheckDeploymentReady("kustomization_resource.dep1", "test-wait", "test"),
507+
),
508+
},
509+
//
510+
//
511+
// Applying modified config updating the deployment annotation with wait
512+
{
513+
Config: testAccResourceKustomizationConfig_wait("test_kustomizations/wait/modified"),
514+
Check: resource.ComposeAggregateTestCheckFunc(
515+
testAccCheckManifestNestedString("kustomization_resource.dep1", "this will cause a redeploy", "spec", "template", "metadata", "annotations", "new"),
516+
testAccCheckDeploymentReady("kustomization_resource.dep1", "test-wait", "test"),
517+
),
518+
},
519+
},
520+
})
521+
}
522+
523+
func testAccResourceKustomizationConfig_wait(path string) string {
524+
return testAccDataSourceKustomizationConfig_basic(path) + `
525+
resource "kustomization_resource" "ns" {
526+
manifest = data.kustomization_build.test.manifests["_/Namespace/_/test-wait"]
527+
}
528+
resource "kustomization_resource" "dep1" {
529+
manifest = data.kustomization_build.test.manifests["apps/Deployment/test-wait/test"]
530+
wait = true
531+
timeouts {
532+
create = "1m"
533+
update = "1m"
534+
}
535+
}
536+
`
537+
}
538+
539+
func TestAccResourceKustomization_wait_failure(t *testing.T) {
540+
541+
resource.Test(t, resource.TestCase{
542+
//PreCheck: func() { testAccPreCheck(t) },
543+
Providers: testAccProviders,
544+
Steps: []resource.TestStep{
545+
//
546+
//
547+
// Applying initial config with a svc and a failing deployment in a namespace with wait
548+
{
549+
Config: testAccResourceKustomizationConfig_wait_failure("test_kustomizations/wait-fail/initial"),
550+
Check: resource.ComposeAggregateTestCheckFunc(
551+
testAccCheckDeploymentNotReady("kustomization_resource.dep1", "test-wait-fail", "test"),
552+
),
553+
ExpectError: regexp.MustCompile("timed out creating/updating Deployment test-wait-fail/test:"),
554+
},
555+
},
556+
})
557+
}
558+
559+
func testAccResourceKustomizationConfig_wait_failure(path string) string {
560+
return testAccDataSourceKustomizationConfig_basic(path) + `
561+
resource "kustomization_resource" "ns" {
562+
manifest = data.kustomization_build.test.manifests["_/Namespace/_/test-wait-fail"]
563+
}
564+
resource "kustomization_resource" "dep1" {
565+
manifest = data.kustomization_build.test.manifests["apps/Deployment/test-wait-fail/test"]
566+
wait = true
567+
timeouts {
568+
create = "1m"
569+
}
570+
}
571+
`
572+
}
573+
574+
func TestAccResourceKustomization_nowait(t *testing.T) {
575+
576+
resource.Test(t, resource.TestCase{
577+
//PreCheck: func() { testAccPreCheck(t) },
578+
Providers: testAccProviders,
579+
Steps: []resource.TestStep{
580+
//
581+
//
582+
// Applying initial config with a svc and deployment in a namespace without wait
583+
// so shouldn't exist immediately after creation
584+
{
585+
Config: testAccResourceKustomizationConfig_nowait("test_kustomizations/nowait/initial"),
586+
Check: resource.ComposeAggregateTestCheckFunc(
587+
testAccCheckManifestNestedString("kustomization_resource.dep1", "test", "spec", "selector", "matchLabels", "app"),
588+
testAccCheckDeploymentNotReady("kustomization_resource.dep1", "test-nowait", "test"),
589+
),
590+
},
591+
//
592+
//
593+
// Applying modified config updating the deployment annotation without wait,
594+
// so we don't immediately expect the annotation to be present
595+
{
596+
Config: testAccResourceKustomizationConfig_nowait("test_kustomizations/nowait/modified"),
597+
Check: resource.ComposeAggregateTestCheckFunc(
598+
testAccCheckManifestNestedString("kustomization_resource.dep1", "this will cause a redeploy", "spec", "template", "metadata", "annotations", "new"),
599+
testAccCheckDeploymentNotReady("kustomization_resource.dep1", "test-nowait", "test"),
600+
),
601+
},
602+
},
603+
})
604+
}
605+
606+
func testAccResourceKustomizationConfig_nowait(path string) string {
607+
return testAccDataSourceKustomizationConfig_basic(path) + `
608+
resource "kustomization_resource" "ns" {
609+
manifest = data.kustomization_build.test.manifests["_/Namespace/_/test-nowait"]
610+
}
611+
612+
resource "kustomization_resource" "dep1" {
613+
manifest = data.kustomization_build.test.manifests["apps/Deployment/test-nowait/test"]
614+
}
615+
`
616+
}
617+
495618
// Upgrade_API_Version Test
496619
func TestAccResourceKustomization_upgradeAPIVersion(t *testing.T) {
497620

@@ -922,6 +1045,50 @@ func testAccCheckDeploymentPurged(n string) resource.TestCheckFunc {
9221045
}
9231046
}
9241047

1048+
func testAccCheckDeploymentReady(n string, namespace string, name string) resource.TestCheckFunc {
1049+
return func(s *terraform.State) error {
1050+
u, err := getResourceFromTestState(s, n)
1051+
if err != nil {
1052+
return err
1053+
}
1054+
1055+
resp, err := getResourceFromK8sAPI(u)
1056+
if err != nil {
1057+
return err
1058+
}
1059+
ready, err := deploymentReady(resp)
1060+
if err != nil {
1061+
return err
1062+
}
1063+
if !ready {
1064+
return fmt.Errorf("deployment %s in %s not ready", name, namespace)
1065+
}
1066+
return nil
1067+
}
1068+
}
1069+
1070+
func testAccCheckDeploymentNotReady(n string, namespace string, name string) resource.TestCheckFunc {
1071+
return func(s *terraform.State) error {
1072+
u, err := getResourceFromTestState(s, n)
1073+
if err != nil {
1074+
return err
1075+
}
1076+
1077+
resp, err := getResourceFromK8sAPI(u)
1078+
if err != nil {
1079+
return err
1080+
}
1081+
ready, err := deploymentReady(resp)
1082+
if err != nil {
1083+
return err
1084+
}
1085+
if ready {
1086+
return fmt.Errorf("deployment %s in %s unexpectedly ready", name, namespace)
1087+
}
1088+
return nil
1089+
}
1090+
}
1091+
9251092
func getResourceFromTestState(s *terraform.State, n string) (ur *k8sunstructured.Unstructured, err error) {
9261093
rs, ok := s.RootModule().Resources[n]
9271094
if !ok {

‎kustomize/test_kustomizations/nowait/initial/kustomization.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
4+
namespace: test-nowait
5+
6+
resources:
7+
- namespace.yaml
8+
- ../../_example_app

0 commit comments

Comments
 (0)