diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index de391f1..4b83fe5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,7 +26,7 @@ jobs: if: startsWith(matrix.runner, 'ubuntu-') uses: engineerd/setup-kind@v0.5.0 with: - version: "v0.9.0" + version: "v0.17.0" - name: Set up Go uses: actions/setup-go@v2 diff --git a/docs/resources/resource.md b/docs/resources/resource.md index c1a15ff..f3c7165 100644 --- a/docs/resources/resource.md +++ b/docs/resources/resource.md @@ -68,6 +68,7 @@ resource "kustomization_resource" "p0" { # then loop through resources in ids_prio[1] # and set an explicit depends_on on kustomization_resource.p0 +# wait 2 minutes for any deployment or daemonset to become ready resource "kustomization_resource" "p1" { for_each = data.kustomization_build.test.ids_prio[1] @@ -76,6 +77,11 @@ resource "kustomization_resource" "p1" { ? sensitive(data.kustomization_build.test.manifests[each.value]) : data.kustomization_build.test.manifests[each.value] ) + wait = true + timeouts { + create = "2m" + update = "2m" + } depends_on = [kustomization_resource.p0] } @@ -99,4 +105,5 @@ resource "kustomization_resource" "p2" { ## Argument Reference - `manifest` - (Required) JSON encoded Kubernetes resource manifest. -- 'timeouts' - (Optional) Overwrite `create` or `delete` timeout defaults. Defaults are 5 minutes for `create` and 10 minutes for `delete`. +- `wait` - Whether to wait for pods to become ready (default false). Currently only has an effect for Deployments and DaemonSets. +- 'timeouts' - (Optional) Overwrite `create`, `update` or `delete` timeout defaults. Defaults are 5 minutes for `create` and `update` and 10 minutes for `delete`. diff --git a/kustomize/manifest.go b/kustomize/manifest.go index 444179b..31765da 100644 --- a/kustomize/manifest.go +++ b/kustomize/manifest.go @@ -9,9 +9,11 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + k8sappsv1 "k8s.io/api/apps/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" k8smeta "k8s.io/apimachinery/pkg/api/meta" k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sunstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" k8sruntime "k8s.io/apimachinery/pkg/runtime" k8sschema "k8s.io/apimachinery/pkg/runtime/schema" @@ -20,6 +22,11 @@ import ( "k8s.io/client-go/restmapper" ) +var waitRefreshFunctions = map[string]waitRefreshFunction{ + "apps/Deployment": waitDeploymentRefresh, + "apps/Daemonset": waitDaemonsetRefresh, +} + type kManifestId struct { group string kind string @@ -27,6 +34,8 @@ type kManifestId struct { name string } +type waitRefreshFunction func(km *kManifest) (interface{}, string, error) + func mustParseProviderId(str string) *kManifestId { kr, err := parseProviderId(str) if err != nil { @@ -354,6 +363,95 @@ func (km *kManifest) waitDeleted(t time.Duration) error { return nil } +func daemonsetReady(u *k8sunstructured.Unstructured) (bool, error) { + var daemonset k8sappsv1.DaemonSet + if err := k8sruntime.DefaultUnstructuredConverter.FromUnstructured(u.UnstructuredContent(), &daemonset); err != nil { + return false, err + } + if daemonset.Generation == daemonset.Status.ObservedGeneration && + daemonset.Status.UpdatedNumberScheduled == daemonset.Status.DesiredNumberScheduled && + daemonset.Status.NumberReady == daemonset.Status.DesiredNumberScheduled && + daemonset.Status.NumberUnavailable == 0 { + return true, nil + } else { + return false, nil + } +} + +func waitDaemonsetRefresh(km *kManifest) (interface{}, string, error) { + resp, err := km.apiGet(k8smetav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, "missing", nil + } + return nil, "error", err + } + ready, err := daemonsetReady(resp) + if err != nil { + return nil, "error", err + } + if ready { + return resp, "done", nil + } + return nil, "in progress", nil +} + +func deploymentReady(u *k8sunstructured.Unstructured) (bool, error) { + var deployment k8sappsv1.Deployment + if err := k8sruntime.DefaultUnstructuredConverter.FromUnstructured(u.UnstructuredContent(), &deployment); err != nil { + return false, err + } + if deployment.Generation == deployment.Status.ObservedGeneration && + deployment.Status.AvailableReplicas == *deployment.Spec.Replicas && + deployment.Status.AvailableReplicas == deployment.Status.Replicas && + deployment.Status.UnavailableReplicas == 0 { + return true, nil + } else { + return false, nil + } +} + +func waitDeploymentRefresh(km *kManifest) (interface{}, string, error) { + resp, err := km.apiGet(k8smetav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, "missing", nil + } + return nil, "error", err + } + ready, err := deploymentReady(resp) + if err != nil { + return nil, "error", err + } + if ready { + return resp, "done", nil + } + return nil, "in progress", nil +} + +func (km *kManifest) waitCreatedOrUpdated(t time.Duration) error { + gvk := km.gvk() + if refresh, ok := waitRefreshFunctions[fmt.Sprintf("%s/%s", gvk.Group, gvk.Kind)]; ok { + delay := 10 * time.Second + stateConf := &resource.StateChangeConf{ + Target: []string{"done"}, + Pending: []string{"in progress"}, + Timeout: t, + Delay: delay, + NotFoundChecks: 2*int(t/delay) + 1, + Refresh: func() (interface{}, string, error) { + return refresh(km) + }, + } + + _, err := stateConf.WaitForState() + if err != nil { + return km.fmtErr(fmt.Errorf("timed out creating/updating %s %s/%s: %s", gvk.Kind, km.namespace(), km.name(), err)) + } + } + return nil +} + func (km *kManifest) fmtErr(err error) error { return fmt.Errorf( "%q: %s", diff --git a/kustomize/resource_kustomization.go b/kustomize/resource_kustomization.go index 2b67215..54980b5 100644 --- a/kustomize/resource_kustomization.go +++ b/kustomize/resource_kustomization.go @@ -35,10 +35,16 @@ func kustomizationResource() *schema.Resource { Type: schema.TypeString, Required: true, }, + "wait": &schema.Schema{ + Type: schema.TypeBool, + Default: false, + Optional: true, + }, }, Timeouts: &schema.ResourceTimeout{ Create: schema.DefaultTimeout(5 * time.Minute), + Update: schema.DefaultTimeout(5 * time.Minute), Delete: schema.DefaultTimeout(10 * time.Minute), }, } @@ -103,6 +109,12 @@ func kustomizationResourceCreate(d *schema.ResourceData, m interface{}) error { return logError(err) } + if d.Get("wait").(bool) { + if err = km.waitCreatedOrUpdated(d.Timeout(schema.TimeoutCreate)); err != nil { + return logError(err) + } + } + id := string(resp.GetUID()) d.SetId(id) @@ -327,6 +339,12 @@ func kustomizationResourceUpdate(d *schema.ResourceData, m interface{}) error { return logError(err) } + if d.Get("wait").(bool) { + if err = kmm.waitCreatedOrUpdated(d.Timeout(schema.TimeoutUpdate)); err != nil { + return logError(err) + } + } + id := string(resp.GetUID()) d.SetId(id) @@ -421,6 +439,7 @@ func kustomizationResourceImport(d *schema.ResourceData, m interface{}) ([]*sche } d.Set("manifest", lac) + d.Set("wait", d.Get("wait")) return []*schema.ResourceData{d}, nil } diff --git a/kustomize/resource_kustomization_test.go b/kustomize/resource_kustomization_test.go index 85c5380..9b30802 100644 --- a/kustomize/resource_kustomization_test.go +++ b/kustomize/resource_kustomization_test.go @@ -6,6 +6,7 @@ import ( "regexp" "strings" "testing" + "time" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" @@ -474,6 +475,135 @@ resource "kustomization_resource" "scprov" { ` } +func TestAccResourceKustomization_wait(t *testing.T) { + now := time.Now() + resource.Test(t, resource.TestCase{ + //PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + // + // + // Applying initial config with a svc and deployment in a namespace with wait + { + Config: testAccResourceKustomizationConfig_wait("test_kustomizations/wait/initial"), + Check: resource.ComposeAggregateTestCheckFunc( + assertDurationIsShorterThan(now, 5*time.Minute), + testAccCheckManifestNestedString("kustomization_resource.dep1", "test", "spec", "selector", "matchLabels", "app"), + testAccCheckDeploymentReady("kustomization_resource.dep1", "test-wait", "test"), + ), + }, + // + // + // Applying modified config updating the deployment annotation with wait + { + Config: testAccResourceKustomizationConfig_wait("test_kustomizations/wait/modified"), + Check: resource.ComposeAggregateTestCheckFunc( + assertDurationIsShorterThan(now, 1*time.Minute), + testAccCheckManifestNestedString("kustomization_resource.dep1", "this will cause a redeploy", "spec", "template", "metadata", "annotations", "new"), + testAccCheckDeploymentReady("kustomization_resource.dep1", "test-wait", "test"), + ), + }, + }, + }) +} + +func testAccResourceKustomizationConfig_wait(path string) string { + return testAccDataSourceKustomizationConfig_basic(path) + ` +resource "kustomization_resource" "ns" { + manifest = data.kustomization_build.test.manifests["_/Namespace/_/test-wait"] +} +resource "kustomization_resource" "dep1" { + manifest = data.kustomization_build.test.manifests["apps/Deployment/test-wait/test"] + wait = true + timeouts { + create = "1m" + update = "1m" + } +} +` +} + +func TestAccResourceKustomization_wait_failure(t *testing.T) { + now := time.Now() + + resource.Test(t, resource.TestCase{ + //PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + // + // + // Applying initial config with a svc and a failing deployment in a namespace with wait + { + Config: testAccResourceKustomizationConfig_wait_failure("test_kustomizations/wait-fail/initial"), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckDeploymentNotReady("kustomization_resource.dep1", "test-wait-fail", "test"), + assertDurationIsLongerThan(now, 1*time.Minute), + ), + ExpectError: regexp.MustCompile("timed out creating/updating Deployment test-wait-fail/test:"), + }, + }, + }) +} + +func testAccResourceKustomizationConfig_wait_failure(path string) string { + return testAccDataSourceKustomizationConfig_basic(path) + ` +resource "kustomization_resource" "ns" { + manifest = data.kustomization_build.test.manifests["_/Namespace/_/test-wait-fail"] +} +resource "kustomization_resource" "dep1" { + manifest = data.kustomization_build.test.manifests["apps/Deployment/test-wait-fail/test"] + wait = true + timeouts { + create = "1m" + } +} +` +} + +func TestAccResourceKustomization_nowait(t *testing.T) { + + resource.Test(t, resource.TestCase{ + //PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + // + // + // Applying initial config with a svc and deployment in a namespace without wait + // so shouldn't exist immediately after creation + { + Config: testAccResourceKustomizationConfig_nowait("test_kustomizations/nowait/initial"), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckManifestNestedString("kustomization_resource.dep1", "test", "spec", "selector", "matchLabels", "app"), + testAccCheckDeploymentNotReady("kustomization_resource.dep1", "test-nowait", "test"), + ), + }, + // + // + // Applying modified config updating the deployment annotation without wait, + // so we don't immediately expect the annotation to be present + { + Config: testAccResourceKustomizationConfig_nowait("test_kustomizations/nowait/modified"), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckManifestNestedString("kustomization_resource.dep1", "this will cause a redeploy", "spec", "template", "metadata", "annotations", "new"), + testAccCheckDeploymentNotReady("kustomization_resource.dep1", "test-nowait", "test"), + ), + }, + }, + }) +} + +func testAccResourceKustomizationConfig_nowait(path string) string { + return testAccDataSourceKustomizationConfig_basic(path) + ` +resource "kustomization_resource" "ns" { + manifest = data.kustomization_build.test.manifests["_/Namespace/_/test-nowait"] +} + +resource "kustomization_resource" "dep1" { + manifest = data.kustomization_build.test.manifests["apps/Deployment/test-nowait/test"] +} +` +} + // Upgrade_API_Version Test func TestAccResourceKustomization_upgradeAPIVersion(t *testing.T) { @@ -890,6 +1020,50 @@ func testAccCheckDeploymentPurged(n string) resource.TestCheckFunc { } } +func testAccCheckDeploymentReady(n string, namespace string, name string) resource.TestCheckFunc { + return func(s *terraform.State) error { + u, err := getResourceFromTestState(s, n) + if err != nil { + return err + } + + resp, err := getResourceFromK8sAPI(u) + if err != nil { + return err + } + ready, err := deploymentReady(resp) + if err != nil { + return err + } + if !ready { + return fmt.Errorf("deployment %s in %s not ready", name, namespace) + } + return nil + } +} + +func testAccCheckDeploymentNotReady(n string, namespace string, name string) resource.TestCheckFunc { + return func(s *terraform.State) error { + u, err := getResourceFromTestState(s, n) + if err != nil { + return err + } + + resp, err := getResourceFromK8sAPI(u) + if err != nil { + return err + } + ready, err := deploymentReady(resp) + if err != nil { + return err + } + if ready { + return fmt.Errorf("deployment %s in %s unexpectedly ready", name, namespace) + } + return nil + } +} + func getResourceFromTestState(s *terraform.State, n string) (ur *k8sunstructured.Unstructured, err error) { rs, ok := s.RootModule().Resources[n] if !ok { @@ -1116,3 +1290,23 @@ func testAccCheckManifestNestedString(n string, expected string, k ...string) re return nil } } + +func assertDurationIsLongerThan(start time.Time, duration time.Duration) resource.TestCheckFunc { + return func(s *terraform.State) error { + elapsed := time.Since(start) + if elapsed > duration { + return nil + } + return fmt.Errorf("elapsed time %s is not longer than %s", elapsed, duration) + } +} + +func assertDurationIsShorterThan(start time.Time, duration time.Duration) resource.TestCheckFunc { + return func(s *terraform.State) error { + elapsed := time.Since(start) + if elapsed < duration { + return nil + } + return fmt.Errorf("elapsed time %s is not shorter than %s", elapsed, duration) + } +} diff --git a/kustomize/test_kustomizations/nowait/initial/kustomization.yaml b/kustomize/test_kustomizations/nowait/initial/kustomization.yaml new file mode 100644 index 0000000..70bbbaa --- /dev/null +++ b/kustomize/test_kustomizations/nowait/initial/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: test-nowait + +resources: +- namespace.yaml +- ../../_example_app diff --git a/kustomize/test_kustomizations/nowait/initial/namespace.yaml b/kustomize/test_kustomizations/nowait/initial/namespace.yaml new file mode 100644 index 0000000..ab704d1 --- /dev/null +++ b/kustomize/test_kustomizations/nowait/initial/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-nowait diff --git a/kustomize/test_kustomizations/nowait/modified/kustomization.yaml b/kustomize/test_kustomizations/nowait/modified/kustomization.yaml new file mode 100644 index 0000000..e22a366 --- /dev/null +++ b/kustomize/test_kustomizations/nowait/modified/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../initial + +patches: + - target: + kind: Deployment + name: test + patch: | + - op: add + path: /spec/template/metadata/annotations + value: + new: this will cause a redeploy diff --git a/kustomize/test_kustomizations/wait-fail/initial/kustomization.yaml b/kustomize/test_kustomizations/wait-fail/initial/kustomization.yaml new file mode 100644 index 0000000..ecd0b25 --- /dev/null +++ b/kustomize/test_kustomizations/wait-fail/initial/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: test-wait-fail + +resources: +- namespace.yaml +- ../../_example_app + +images: + - name: nginx + newName: doesnotexist/definitelydoesntexist diff --git a/kustomize/test_kustomizations/wait-fail/initial/namespace.yaml b/kustomize/test_kustomizations/wait-fail/initial/namespace.yaml new file mode 100644 index 0000000..0e6fdb9 --- /dev/null +++ b/kustomize/test_kustomizations/wait-fail/initial/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-wait-fail diff --git a/kustomize/test_kustomizations/wait/initial/kustomization.yaml b/kustomize/test_kustomizations/wait/initial/kustomization.yaml new file mode 100644 index 0000000..7428808 --- /dev/null +++ b/kustomize/test_kustomizations/wait/initial/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: test-wait + +resources: +- namespace.yaml +- ../../_example_app diff --git a/kustomize/test_kustomizations/wait/initial/namespace.yaml b/kustomize/test_kustomizations/wait/initial/namespace.yaml new file mode 100644 index 0000000..ec081c3 --- /dev/null +++ b/kustomize/test_kustomizations/wait/initial/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-wait diff --git a/kustomize/test_kustomizations/wait/modified/kustomization.yaml b/kustomize/test_kustomizations/wait/modified/kustomization.yaml new file mode 100644 index 0000000..e22a366 --- /dev/null +++ b/kustomize/test_kustomizations/wait/modified/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../initial + +patches: + - target: + kind: Deployment + name: test + patch: | + - op: add + path: /spec/template/metadata/annotations + value: + new: this will cause a redeploy