feat(deploy): add secret/config map hash to pod templates

internal/controllers/common/common_utils.go

@@ -15,13 +15,17 @@
 package common
 
 import (
+	"cmp"
+	"context"
 	"crypto/sha256"
+	"encoding/json"
 	"fmt"
 	"hash/fnv"
 	"io/ioutil"
 	"math/rand"
 	"os"
 	"regexp"
+	"slices"
 	"strings"
 	"time"
 
@@ -31,6 +35,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 )
 
@@ -202,6 +208,123 @@ func checkResourceRequestWithLimit(requests, limits corev1.ResourceList) {
 	}
 }
 
+const annotationSecretHash = "io.cryostat/secret-hash"
+const annotationConfigMapHash = "io.cryostat/config-map-hash"
+
+// AnnotateWithObjRefHashes annotates the provided pod template with hashes of the secret and config map data used
+// by this pod template. This allows the pod template parent to automatically roll out a new revision when
+// the hashed data changes.
+func AnnotateWithObjRefHashes(ctx context.Context, client client.Client, namespace string, template *corev1.PodTemplateSpec) error {
+	if template.Annotations == nil {
+		template.Annotations = map[string]string{}
+	}
+
+	// Collect names of secrets and config maps used by this pod template
+	secrets := newObjectSet[string]()
+	configMaps := newObjectSet[string]()
+
+	// Look for secrets and config maps references in environment variables
+	for _, container := range template.Spec.Containers {
+		// Look through Env[].ValueFrom for secret/config map refs
+		for _, env := range container.Env {
+			if env.ValueFrom != nil {
+				if env.ValueFrom.SecretKeyRef != nil {
+					secrets.add(env.ValueFrom.SecretKeyRef.Name)
+				} else if env.ValueFrom.ConfigMapKeyRef != nil {
+					configMaps.add(env.ValueFrom.ConfigMapKeyRef.Name)
+				}
+			}
+		}
+		// Look through EnvFrom for secret/config map refs
+		for _, envFrom := range container.EnvFrom {
+			if envFrom.SecretRef != nil {
+				secrets.add(envFrom.SecretRef.Name)
+			} else if envFrom.ConfigMapRef != nil {
+				configMaps.add(envFrom.ConfigMapRef.Name)
+			}
+		}
+	}
+
+	// Look for secrets and config maps references in volumes
+	for _, vol := range template.Spec.Volumes {
+		if vol.Secret != nil {
+			// Look for secret volumes
+			secrets.add(vol.Secret.SecretName)
+		} else if vol.ConfigMap != nil {
+			// Look for config map volumes
+			configMaps.add(vol.ConfigMap.Name)
+		} else if vol.Projected != nil {
+			// Also look for secret/config map sources in projected volumes
+			for _, source := range vol.Projected.Sources {
+				if source.Secret != nil {
+					secrets.add(source.Secret.Name)
+				} else if source.ConfigMap != nil {
+					configMaps.add(source.ConfigMap.Name)
+				}
+			}
+		}
+	}
+
+	// Hash the discovered secrets and config maps
+	secretHash, err := hashSecrets(ctx, client, namespace, secrets)
+	if err != nil {
+		return err
+	}
+	configMapHash, err := hashConfigMaps(ctx, client, namespace, configMaps)
+	if err != nil {
+		return err
+	}
+
+	// Apply the hashes as annotations to the pod template
+	template.Annotations[annotationSecretHash] = *secretHash
+	template.Annotations[annotationConfigMapHash] = *configMapHash
+	return nil
+}
+
+func hashSecrets(ctx context.Context, client client.Client, namespace string, secrets *objectSet[string]) (*string, error) {
+	// Collect the JSON of all secret data, sorted by object name
+	combinedJSON := []byte{}
+	for _, name := range secrets.toSortedSlice() {
+		secret := &corev1.Secret{}
+		err := client.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, secret)
+		if err != nil {
+			return nil, err
+		}
+		// Marshal secret data as JSON. Keys are sorted, see: [json.Marshal]
+		buf, err := json.Marshal(secret.Data)
+		if err != nil {
+			return nil, err
+		}
+		combinedJSON = append(combinedJSON, buf...)
+	}
+	// Hash the JSON with SHA256
+	hashed := fmt.Sprintf("%x", sha256.Sum256(combinedJSON))
+	return &hashed, nil
+}
+
+func hashConfigMaps(ctx context.Context, client client.Client, namespace string, configMaps *objectSet[string]) (*string, error) {
+	// Collect the JSON of all config map data, sorted by object name
+	combinedJSON := []byte{}
+	for _, name := range configMaps.toSortedSlice() {
+		cm := &corev1.ConfigMap{}
+		err := client.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, cm)
+		if err != nil {
+			return nil, err
+		}
+		// Marshal config map data as JSON. Keys are sorted, see: [json.Marshal]
+		buf, err := json.Marshal(cm.Data)
+		if err != nil {
+			return nil, err
+		}
+		combinedJSON = append(combinedJSON, buf...)
+	}
+	// Hash the JSON with FNV-1
+	hash := fnv.New128()
+	hash.Write([]byte(combinedJSON))
+	hashed := fmt.Sprintf("%x", hash.Sum([]byte{}))
+	return &hashed, nil
+}
+
 // SeccompProfile returns a SeccompProfile for the restricted
 // Pod Security Standard that, on OpenShift, is backwards-compatible
 // with OpenShift < 4.11.
@@ -217,3 +340,28 @@ func SeccompProfile(openshift bool) *corev1.SeccompProfile {
 		Type: corev1.SeccompProfileTypeRuntimeDefault,
 	}
 }
+
+// Set abstraction for collecting names of secrets and config maps used by a pod
+type objectSet[T cmp.Ordered] struct {
+	impl map[T]struct{}
+}
+
+func newObjectSet[T cmp.Ordered]() *objectSet[T] {
+	return &objectSet[T]{
+		impl: map[T]struct{}{},
+	}
+}
+
+func (s *objectSet[T]) add(obj T) {
+	s.impl[obj] = struct{}{}
+}
+
+func (s *objectSet[T]) toSortedSlice() []T {
+	// Convert set to a sorted slice
+	slice := make([]T, 0, len(s.impl))
+	for k := range s.impl {
+		slice = append(slice, k)
+	}
+	slices.Sort(slice)
+	return slice
+}

internal/controllers/configmaps.go

@@ -136,7 +136,7 @@ func (r *Reconciler) reconcileOAuth2ProxyConfig(ctx context.Context, cr *model.C
 	if r.IsOpenShift {
 		return r.deleteConfigMap(ctx, cm)
 	} else {
-		json, err := json.Marshal(cfg)
+		json, err := json.MarshalIndent(cfg, "", "  ")
 		if err != nil {
 			return err
 		}

internal/controllers/reconciler.go

@@ -593,6 +593,12 @@ func (r *Reconciler) updateConditionsFromDeployment(ctx context.Context, cr *mod
 var errSelectorModified error = errors.New("deployment selector has been modified")
 
 func (r *Reconciler) createOrUpdateDeployment(ctx context.Context, deploy *appsv1.Deployment, owner metav1.Object) error {
+	// Annotate the deployment with hashes for any referenced secrets/config maps
+	err := common.AnnotateWithObjRefHashes(ctx, r.Client, deploy.Namespace, &deploy.Spec.Template)
+	if err != nil {
+		return err
+	}
+	// Make a copy of the new desired deployment
 	deployCopy := deploy.DeepCopy()
 	op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deploy, func() error {
 		// Merge any required labels and annotations

internal/controllers/reconciler_test.go

@@ -170,6 +170,7 @@ func resourceChecks() []resourceCheck {
 		{(*cryostatTestInput).expectAgentProxyConfigMap, "agent proxy config map"},
 		{(*cryostatTestInput).expectAgentGatewayService, "agent gateway service"},
 		{(*cryostatTestInput).expectAgentCallbackService, "agent callback service"},
+		{(*cryostatTestInput).expectOAuthCookieSecret, "OAuth2 cookie secret"},
 	}
 }
 
@@ -517,7 +518,7 @@ func (c *controllerTest) commonTests() {
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(metav1.IsControlledBy(secret, cr.Object)).To(BeTrue())
-				Expect(secret.StringData["CONNECTION_KEY"]).To(Equal(oldSecret.StringData["CONNECTION_KEY"]))
+				Expect(secret.Data["CONNECTION_KEY"]).To(Equal(oldSecret.Data["CONNECTION_KEY"]))
 			})
 		})
 		Context("with existing Routes", func() {
@@ -1839,20 +1840,21 @@ func (c *controllerTest) commonTests() {
 			})
 		})
 		Context("with secret provided for database", func() {
-			var customSecret *corev1.Secret
 			BeforeEach(func() {
-				customSecret = t.NewCustomDatabaseSecret()
-				t.objs = append(t.objs, t.NewCryostatWithDatabaseSecretProvided().Object, customSecret)
+				t.GeneratedPasswords = []string{"auth_cookie_secret", "object_storage", "keystore"}
+				t.DatabaseSecret = t.NewCustomDatabaseSecret()
+				t.objs = append(t.objs, t.NewCryostatWithDatabaseSecretProvided().Object, t.DatabaseSecret)
 			})
 			JustBeforeEach(func() {
 				t.reconcileCryostatFully()
 			})
 			It("should configure deployment appropriately", func() {
 				t.expectMainDeployment()
+				t.expectDatabaseDeployment()
 			})
 			It("should set Database Secret in CR Status", func() {
 				instance := t.getCryostatInstance()
-				Expect(instance.Status.DatabaseSecret).To(Equal(customSecret.Name))
+				Expect(instance.Status.DatabaseSecret).To(Equal(t.DatabaseSecret.Name))
 			})
 			It("should not generate default secret", func() {
 				secret := &corev1.Secret{}
@@ -2558,7 +2560,7 @@ func (c *controllerTest) commonTests() {
 				Expect(binding.RoleRef).To(Equal(expected.RoleRef))
 			})
 		})
-		Context("with additionnal label and annotation", func() {
+		Context("with additional label and annotation", func() {
 			BeforeEach(func() {
 				t.ReportReplicas = 1
 				t.objs = append(t.objs, t.NewCryostatWithAdditionalMetadata().Object)
@@ -2897,7 +2899,7 @@ func (t *cryostatTestInput) expectWaitingForCertificate() {
 
 func (t *cryostatTestInput) expectCertificates() {
 	// Check certificates
-	certs := []*certv1.Certificate{t.NewCryostatCert(), t.NewCACert(), t.NewReportsCert(), t.NewAgentProxyCert()}
+	certs := []*certv1.Certificate{t.NewCryostatCert(), t.NewCACert(), t.NewReportsCert(), t.NewAgentProxyCert(), t.NewDatabaseCert(), t.NewStorageCert()}
 	for _, expected := range certs {
 		actual := &certv1.Certificate{}
 		err := t.Client.Get(context.Background(), types.NamespacedName{Name: expected.Name, Namespace: expected.Namespace}, actual)
@@ -2920,7 +2922,7 @@ func (t *cryostatTestInput) expectCertificates() {
 	err := t.Client.Get(context.Background(), types.NamespacedName{Name: expectedSecret.Name, Namespace: expectedSecret.Namespace}, secret)
 	Expect(err).ToNot(HaveOccurred())
 	t.checkMetadata(secret, expectedSecret)
-	Expect(secret.StringData).To(Equal(expectedSecret.StringData))
+	Expect(secret.Data).To(Equal(expectedSecret.Data))
 
 	// Check CA Cert secrets in each target namespace
 	Expect(t.TargetNamespaces).ToNot(BeEmpty())
@@ -3099,7 +3101,7 @@ func (t *cryostatTestInput) expectOAuth2ConfigMap() {
 	t.checkMetadata(cm, expected)
 	Expect(cm.Data).To(HaveLen(1))
 	Expect(cm.Data).To(HaveKey("alpha_config.json"))
-	Expect(cm.Data["alpha_config.json"]).To(MatchJSON(expected.Data["alpha_config.json"]))
+	Expect(cm.Data["alpha_config.json"]).To(Equal(expected.Data["alpha_config.json"]))
 	Expect(cm.Immutable).To(Equal(expected.Immutable))
 }
 
@@ -3157,7 +3159,7 @@ func (t *cryostatTestInput) expectDatabaseSecret() {
 	// Compare to desired spec
 	expectedSecret := t.NewDatabaseSecret()
 	t.checkMetadata(secret, expectedSecret)
-	Expect(secret.StringData).To(Equal(expectedSecret.StringData))
+	Expect(secret.Data).To(Equal(expectedSecret.Data))
 	Expect(secret.Immutable).To(Equal(expectedSecret.Immutable))
 }
 
@@ -3169,7 +3171,18 @@ func (t *cryostatTestInput) expectStorageSecret() {
 	// Compare to desired spec
 	expectedSecret := t.NewStorageSecret()
 	t.checkMetadata(secret, expectedSecret)
-	Expect(secret.StringData).To(Equal(expectedSecret.StringData))
+	Expect(secret.Data).To(Equal(expectedSecret.Data))
+}
+
+func (t *cryostatTestInput) expectOAuthCookieSecret() {
+	expectedSecret := t.NewAuthProxyCookieSecret()
+	secret := &corev1.Secret{}
+	err := t.Client.Get(context.Background(), types.NamespacedName{Name: expectedSecret.Name, Namespace: expectedSecret.Namespace}, secret)
+	Expect(err).ToNot(HaveOccurred())
+
+	// Compare to desired spec
+	t.checkMetadata(secret, expectedSecret)
+	Expect(secret.Data).To(Equal(expectedSecret.Data))
 }
 
 func (t *cryostatTestInput) expectCoreService() {
@@ -3421,6 +3434,7 @@ func (t *cryostatTestInput) checkMainPodTemplate(deployment *appsv1.Deployment,
 		"kind":      "cryostat",
 		"component": "cryostat",
 	}))
+	Expect(template.Annotations).To(Equal(t.NewMainPodAnnotations()))
 	Expect(template.Spec.Volumes).To(ConsistOf(t.NewVolumes()))
 	Expect(template.Spec.SecurityContext).To(Equal(t.NewPodSecurityContext(cr)))
 
@@ -3499,10 +3513,10 @@ func (t *cryostatTestInput) expectMainPodTemplateHasExtraMetadata(deployment *ap
 		"myPodExtraLabel":       "myPodLabel",
 		"myPodSecondExtraLabel": "myPodSecondLabel",
 	}))
-	Expect(template.Annotations).To(Equal(map[string]string{
-		"myPodExtraAnnotation":       "myPodAnnotation",
-		"mySecondPodExtraAnnotation": "mySecondPodAnnotation",
-	}))
+	annotations := t.NewMainPodAnnotations()
+	annotations["myPodExtraAnnotation"] = "myPodAnnotation"
+	annotations["mySecondPodExtraAnnotation"] = "mySecondPodAnnotation"
+	Expect(template.Annotations).To(Equal(annotations))
 }
 
 func (t *cryostatTestInput) expectDatabaseDeployment() {
@@ -3534,6 +3548,7 @@ func (t *cryostatTestInput) expectDatabaseDeployment() {
 		"kind":      "cryostat",
 		"component": "database",
 	}))
+	Expect(template.Annotations).To(Equal(t.NewDatabasePodAnnotations()))
 	Expect(template.Spec.Volumes).To(ConsistOf(t.NewDatabaseVolumes()))
 	Expect(template.Spec.SecurityContext).To(Equal(t.NewPodSecurityContext(cr)))
 
@@ -3587,6 +3602,7 @@ func (t *cryostatTestInput) expectStorageDeployment() {
 		"kind":      "cryostat",
 		"component": "storage",
 	}))
+	Expect(template.Annotations).To(Equal(t.NewStoragePodAnnotations()))
 	Expect(template.Spec.Volumes).To(ConsistOf(t.NewStorageVolumes()))
 	Expect(template.Spec.SecurityContext).To(Equal(t.NewPodSecurityContext(cr)))
 
@@ -3643,6 +3659,7 @@ func (t *cryostatTestInput) checkReportsDeployment() {
 		"kind":      "cryostat",
 		"component": "reports",
 	}))
+	Expect(template.Annotations).To(Equal(t.NewReportsPodAnnotations()))
 	Expect(template.Spec.Volumes).To(ConsistOf(t.NewReportsVolumes()))
 	Expect(template.Spec.SecurityContext).To(Equal(t.NewReportPodSecurityContext(cr)))
 
@@ -3689,10 +3706,10 @@ func (t *cryostatTestInput) expectReportsDeploymentHasExtraMetadata() {
 		"myPodExtraLabel":       "myPodLabel",
 		"myPodSecondExtraLabel": "myPodSecondLabel",
 	}))
-	Expect(template.Annotations).To(Equal(map[string]string{
-		"myPodExtraAnnotation":       "myPodAnnotation",
-		"mySecondPodExtraAnnotation": "mySecondPodAnnotation",
-	}))
+	annotations := t.NewReportsPodAnnotations()
+	annotations["myPodExtraAnnotation"] = "myPodAnnotation"
+	annotations["mySecondPodExtraAnnotation"] = "mySecondPodAnnotation"
+	Expect(template.Annotations).To(Equal(annotations))
 }
 
 func (t *cryostatTestInput) checkDeploymentHasTemplates() {