Add dynamic key filtering to datadog chart baseline tests

Author: levan-m

test/common/common.go

@@ -1,7 +1,9 @@
 package common
 
 import (
-	appsv1 "k8s.io/api/apps/v1"
+	"bytes"
+	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -11,6 +13,9 @@ import (
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/stretchr/testify/require"
+	yaml "gopkg.in/yaml.v3"
+	appsv1 "k8s.io/api/apps/v1"
+	yaml2 "k8s.io/apimachinery/pkg/util/yaml"
 )
 
 type HelmCommand struct {
@@ -117,3 +122,48 @@ func Contains(str string, list []string) bool {
 	}
 	return false
 }
+
+func FilterYamlKeysMultiManifest(manifest string, filterKeys map[string]interface{}) (string, error) {
+	reader := strings.NewReader(manifest)
+	decoder := yaml2.NewYAMLOrJSONDecoder(reader, 4096)
+	builder := strings.Builder{}
+	for {
+		var obj map[string]interface{}
+		// We read the next YAML document from the input stream until we reach EOF.
+		// This is needed if Helm rendering contains multiple resource manifests.
+		err := decoder.Decode(&obj)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", fmt.Errorf("couldn't decode manifest for filtering dynamic keys: %s", err)
+		}
+
+		filterKeysRecursive(&obj, filterKeys)
+
+		var buf bytes.Buffer
+		enc := yaml.NewEncoder(&buf)
+		enc.SetIndent(2) // Adjust indentation (default is 4)
+		enc.Encode(obj)
+		enc.Close()
+		output := buf.String()
+
+		if err != nil {
+			return "", fmt.Errorf("couldn't marshal manifest after filtering: %s", err)
+		}
+		builder.WriteString(output)
+		builder.WriteString("---\n")
+	}
+	return builder.String(), nil
+}
+
+func filterKeysRecursive(yamlMap *map[string]interface{}, keys map[string]interface{}) {
+	for yamlKey := range *yamlMap {
+		if _, found := keys[yamlKey]; found {
+			// fmt.Println("deleting key", yamlKey)
+			delete(*yamlMap, yamlKey)
+		} else if nested, ok := (*yamlMap)[yamlKey].(map[string]interface{}); ok {
+			filterKeysRecursive(&nested, keys)
+		}
+	}
+}

test/datadog/baseline/agent-clusterchecks-deployment_default.yaml

@@ -7,7 +7,6 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: datadog
     app.kubernetes.io/version: "7"
-    helm.sh/chart: datadog-3.98.1
   name: datadog-clusterchecks
   namespace: datadog-agent
 spec:
@@ -23,9 +22,7 @@ spec:
     type: RollingUpdate
   template:
     metadata:
-      annotations:
-        checksum/clusteragent_token: 46c57eb56ed9b06c80f95be1db78e7ada68e05d60e70cd05adbe44dce4d458ad
-        checksum/install_info: 294ab445dd224b5cc1663ee0737da0a03d9e2d995bc77fd2d21fb358fe1b2eed
+      annotations: {}
       labels:
         admission.datadoghq.com/enabled: "false"
         app: datadog-clusterchecks

test/datadog/baseline/cluster-agent-deployment_default.yaml

@@ -7,7 +7,6 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: datadog
     app.kubernetes.io/version: "7"
-    helm.sh/chart: datadog-3.98.1
   name: datadog-cluster-agent
   namespace: datadog-agent
 spec:
@@ -23,12 +22,7 @@ spec:
     type: RollingUpdate
   template:
     metadata:
-      annotations:
-        checksum/api_key: b9ca61ea0157739747185859ecd177e8bd37d7614c90db55d68a3eeb7a23061e
-        checksum/application_key: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
-        checksum/clusteragent-configmap: 78ab32266531527cc470ed026dbb918f2d545a1744a8e0c78b63cedd3fab3d46
-        checksum/clusteragent_token: ba677e8b323af8e8a1bf50dd0aa672fbb7ea7bfcd8b5a6017f36a3a92efc1b93
-        checksum/install_info: 294ab445dd224b5cc1663ee0737da0a03d9e2d995bc77fd2d21fb358fe1b2eed
+      annotations: {}
       labels:
         admission.datadoghq.com/enabled: "false"
         app: datadog-cluster-agent

test/datadog/baseline/cluster-agent-deployment_default_advanced_AC_injection.yaml

@@ -7,7 +7,6 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: datadog
     app.kubernetes.io/version: "7"
-    helm.sh/chart: datadog-3.98.1
   name: datadog-cluster-agent
   namespace: datadog-agent
 spec:
@@ -23,12 +22,7 @@ spec:
     type: RollingUpdate
   template:
     metadata:
-      annotations:
-        checksum/api_key: b9ca61ea0157739747185859ecd177e8bd37d7614c90db55d68a3eeb7a23061e
-        checksum/application_key: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
-        checksum/clusteragent-configmap: 78ab32266531527cc470ed026dbb918f2d545a1744a8e0c78b63cedd3fab3d46
-        checksum/clusteragent_token: 22226c9c062bb4932462ccc9488667d2d2eaa50025fcd567eb1f4184110e2556
-        checksum/install_info: 294ab445dd224b5cc1663ee0737da0a03d9e2d995bc77fd2d21fb358fe1b2eed
+      annotations: {}
       labels:
         admission.datadoghq.com/enabled: "false"
         app: datadog-cluster-agent

test/datadog/baseline/cluster-agent-deployment_default_minimal_AC_injection.yaml

@@ -7,7 +7,6 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: datadog
     app.kubernetes.io/version: "7"
-    helm.sh/chart: datadog-3.98.1
   name: datadog-cluster-agent
   namespace: datadog-agent
 spec:
@@ -23,12 +22,7 @@ spec:
     type: RollingUpdate
   template:
     metadata:
-      annotations:
-        checksum/api_key: b9ca61ea0157739747185859ecd177e8bd37d7614c90db55d68a3eeb7a23061e
-        checksum/application_key: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
-        checksum/clusteragent-configmap: 78ab32266531527cc470ed026dbb918f2d545a1744a8e0c78b63cedd3fab3d46
-        checksum/clusteragent_token: 96e9e09ad2f5522df113058aa00bcce374e9676aab9eb164fb3cf1d9b4d65a14
-        checksum/install_info: 294ab445dd224b5cc1663ee0737da0a03d9e2d995bc77fd2d21fb358fe1b2eed
+      annotations: {}
       labels:
         admission.datadoghq.com/enabled: "false"
         app: datadog-cluster-agent

test/datadog/baseline/daemonset_default.yaml

@@ -7,7 +7,6 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: datadog
     app.kubernetes.io/version: "7"
-    helm.sh/chart: datadog-3.98.1
   name: datadog
   namespace: datadog-agent
 spec:
@@ -17,12 +16,7 @@ spec:
       app: datadog
   template:
     metadata:
-      annotations:
-        checksum/autoconf-config: 74234e98afe7498fb5daf1f36ac2d78acc339464f950703b8c019892f982b90b
-        checksum/checksd-config: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a
-        checksum/clusteragent_token: 0b52485673a4bc26ed99a829df0e4aa4aaad46740afa7f8875464f0c4714ebc6
-        checksum/confd-config: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a
-        checksum/install_info: 294ab445dd224b5cc1663ee0737da0a03d9e2d995bc77fd2d21fb358fe1b2eed
+      annotations: {}
       labels:
         admission.datadoghq.com/enabled: "false"
         app: datadog