Skip to content

Commit b6d2524

Browse files
ebaronebaron
authored
feat(deploy): add secret/config map hash to pod templates (#1112)
1 parent f453b3f commit b6d2524

File tree

6 files changed

+455
-126
lines changed

6 files changed

+455
-126
lines changed

‎internal/controllers/common/common_utils.go

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,17 @@
1515
package common
1616

1717
import (
18+
"cmp"
19+
"context"
1820
"crypto/sha256"
21+
"encoding/json"
1922
"fmt"
2023
"hash/fnv"
2124
"io/ioutil"
2225
"math/rand"
2326
"os"
2427
"regexp"
28+
"slices"
2529
"strings"
2630
"time"
2731

@@ -31,6 +35,8 @@ import (
3135
"k8s.io/apimachinery/pkg/api/resource"
3236
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3337
"k8s.io/apimachinery/pkg/runtime/schema"
38+
"k8s.io/apimachinery/pkg/types"
39+
"sigs.k8s.io/controller-runtime/pkg/client"
3440
logf "sigs.k8s.io/controller-runtime/pkg/log"
3541
)
3642

@@ -202,6 +208,123 @@ func checkResourceRequestWithLimit(requests, limits corev1.ResourceList) {
202208
}
203209
}
204210

211+
const annotationSecretHash = "io.cryostat/secret-hash"
212+
const annotationConfigMapHash = "io.cryostat/config-map-hash"
213+
214+
// AnnotateWithObjRefHashes annotates the provided pod template with hashes of the secret and config map data used
215+
// by this pod template. This allows the pod template parent to automatically roll out a new revision when
216+
// the hashed data changes.
217+
func AnnotateWithObjRefHashes(ctx context.Context, client client.Client, namespace string, template *corev1.PodTemplateSpec) error {
218+
if template.Annotations == nil {
219+
template.Annotations = map[string]string{}
220+
}
221+
222+
// Collect names of secrets and config maps used by this pod template
223+
secrets := newObjectSet[string]()
224+
configMaps := newObjectSet[string]()
225+
226+
// Look for secrets and config maps references in environment variables
227+
for _, container := range template.Spec.Containers {
228+
// Look through Env[].ValueFrom for secret/config map refs
229+
for _, env := range container.Env {
230+
if env.ValueFrom != nil {
231+
if env.ValueFrom.SecretKeyRef != nil {
232+
secrets.add(env.ValueFrom.SecretKeyRef.Name)
233+
} else if env.ValueFrom.ConfigMapKeyRef != nil {
234+
configMaps.add(env.ValueFrom.ConfigMapKeyRef.Name)
235+
}
236+
}
237+
}
238+
// Look through EnvFrom for secret/config map refs
239+
for _, envFrom := range container.EnvFrom {
240+
if envFrom.SecretRef != nil {
241+
secrets.add(envFrom.SecretRef.Name)
242+
} else if envFrom.ConfigMapRef != nil {
243+
configMaps.add(envFrom.ConfigMapRef.Name)
244+
}
245+
}
246+
}
247+
248+
// Look for secrets and config maps references in volumes
249+
for _, vol := range template.Spec.Volumes {
250+
if vol.Secret != nil {
251+
// Look for secret volumes
252+
secrets.add(vol.Secret.SecretName)
253+
} else if vol.ConfigMap != nil {
254+
// Look for config map volumes
255+
configMaps.add(vol.ConfigMap.Name)
256+
} else if vol.Projected != nil {
257+
// Also look for secret/config map sources in projected volumes
258+
for _, source := range vol.Projected.Sources {
259+
if source.Secret != nil {
260+
secrets.add(source.Secret.Name)
261+
} else if source.ConfigMap != nil {
262+
configMaps.add(source.ConfigMap.Name)
263+
}
264+
}
265+
}
266+
}
267+
268+
// Hash the discovered secrets and config maps
269+
secretHash, err := hashSecrets(ctx, client, namespace, secrets)
270+
if err != nil {
271+
return err
272+
}
273+
configMapHash, err := hashConfigMaps(ctx, client, namespace, configMaps)
274+
if err != nil {
275+
return err
276+
}
277+
278+
// Apply the hashes as annotations to the pod template
279+
template.Annotations[annotationSecretHash] = *secretHash
280+
template.Annotations[annotationConfigMapHash] = *configMapHash
281+
return nil
282+
}
283+
284+
func hashSecrets(ctx context.Context, client client.Client, namespace string, secrets *objectSet[string]) (*string, error) {
285+
// Collect the JSON of all secret data, sorted by object name
286+
combinedJSON := []byte{}
287+
for _, name := range secrets.toSortedSlice() {
288+
secret := &corev1.Secret{}
289+
err := client.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, secret)
290+
if err != nil {
291+
return nil, err
292+
}
293+
// Marshal secret data as JSON. Keys are sorted, see: [json.Marshal]
294+
buf, err := json.Marshal(secret.Data)
295+
if err != nil {
296+
return nil, err
297+
}
298+
combinedJSON = append(combinedJSON, buf...)
299+
}
300+
// Hash the JSON with SHA256
301+
hashed := fmt.Sprintf("%x", sha256.Sum256(combinedJSON))
302+
return &hashed, nil
303+
}
304+
305+
func hashConfigMaps(ctx context.Context, client client.Client, namespace string, configMaps *objectSet[string]) (*string, error) {
306+
// Collect the JSON of all config map data, sorted by object name
307+
combinedJSON := []byte{}
308+
for _, name := range configMaps.toSortedSlice() {
309+
cm := &corev1.ConfigMap{}
310+
err := client.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, cm)
311+
if err != nil {
312+
return nil, err
313+
}
314+
// Marshal config map data as JSON. Keys are sorted, see: [json.Marshal]
315+
buf, err := json.Marshal(cm.Data)
316+
if err != nil {
317+
return nil, err
318+
}
319+
combinedJSON = append(combinedJSON, buf...)
320+
}
321+
// Hash the JSON with FNV-1
322+
hash := fnv.New128()
323+
hash.Write([]byte(combinedJSON))
324+
hashed := fmt.Sprintf("%x", hash.Sum([]byte{}))
325+
return &hashed, nil
326+
}
327+
205328
// SeccompProfile returns a SeccompProfile for the restricted
206329
// Pod Security Standard that, on OpenShift, is backwards-compatible
207330
// with OpenShift < 4.11.
@@ -217,3 +340,28 @@ func SeccompProfile(openshift bool) *corev1.SeccompProfile {
217340
Type: corev1.SeccompProfileTypeRuntimeDefault,
218341
}
219342
}
343+
344+
// Set abstraction for collecting names of secrets and config maps used by a pod
345+
type objectSet[T cmp.Ordered] struct {
346+
impl map[T]struct{}
347+
}
348+
349+
func newObjectSet[T cmp.Ordered]() *objectSet[T] {
350+
return &objectSet[T]{
351+
impl: map[T]struct{}{},
352+
}
353+
}
354+
355+
func (s *objectSet[T]) add(obj T) {
356+
s.impl[obj] = struct{}{}
357+
}
358+
359+
func (s *objectSet[T]) toSortedSlice() []T {
360+
// Convert set to a sorted slice
361+
slice := make([]T, 0, len(s.impl))
362+
for k := range s.impl {
363+
slice = append(slice, k)
364+
}
365+
slices.Sort(slice)
366+
return slice
367+
}

‎internal/controllers/configmaps.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ func (r *Reconciler) reconcileOAuth2ProxyConfig(ctx context.Context, cr *model.C
136136
if r.IsOpenShift {
137137
return r.deleteConfigMap(ctx, cm)
138138
} else {
139-
json, err := json.Marshal(cfg)
139+
json, err := json.MarshalIndent(cfg, "", " ")
140140
if err != nil {
141141
return err
142142
}

‎internal/controllers/reconciler.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -593,6 +593,12 @@ func (r *Reconciler) updateConditionsFromDeployment(ctx context.Context, cr *mod
593593
var errSelectorModified error = errors.New("deployment selector has been modified")
594594

595595
func (r *Reconciler) createOrUpdateDeployment(ctx context.Context, deploy *appsv1.Deployment, owner metav1.Object) error {
596+
// Annotate the deployment with hashes for any referenced secrets/config maps
597+
err := common.AnnotateWithObjRefHashes(ctx, r.Client, deploy.Namespace, &deploy.Spec.Template)
598+
if err != nil {
599+
return err
600+
}
601+
// Make a copy of the new desired deployment
596602
deployCopy := deploy.DeepCopy()
597603
op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deploy, func() error {
598604
// Merge any required labels and annotations

‎internal/controllers/reconciler_test.go

Lines changed: 36 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ func resourceChecks() []resourceCheck {
170170
{(*cryostatTestInput).expectAgentProxyConfigMap, "agent proxy config map"},
171171
{(*cryostatTestInput).expectAgentGatewayService, "agent gateway service"},
172172
{(*cryostatTestInput).expectAgentCallbackService, "agent callback service"},
173+
{(*cryostatTestInput).expectOAuthCookieSecret, "OAuth2 cookie secret"},
173174
}
174175
}
175176

@@ -517,7 +518,7 @@ func (c *controllerTest) commonTests() {
517518
Expect(err).ToNot(HaveOccurred())
518519

519520
Expect(metav1.IsControlledBy(secret, cr.Object)).To(BeTrue())
520-
Expect(secret.StringData["CONNECTION_KEY"]).To(Equal(oldSecret.StringData["CONNECTION_KEY"]))
521+
Expect(secret.Data["CONNECTION_KEY"]).To(Equal(oldSecret.Data["CONNECTION_KEY"]))
521522
})
522523
})
523524
Context("with existing Routes", func() {
@@ -1839,20 +1840,21 @@ func (c *controllerTest) commonTests() {
18391840
})
18401841
})
18411842
Context("with secret provided for database", func() {
1842-
var customSecret *corev1.Secret
18431843
BeforeEach(func() {
1844-
customSecret = t.NewCustomDatabaseSecret()
1845-
t.objs = append(t.objs, t.NewCryostatWithDatabaseSecretProvided().Object, customSecret)
1844+
t.GeneratedPasswords = []string{"auth_cookie_secret", "object_storage", "keystore"}
1845+
t.DatabaseSecret = t.NewCustomDatabaseSecret()
1846+
t.objs = append(t.objs, t.NewCryostatWithDatabaseSecretProvided().Object, t.DatabaseSecret)
18461847
})
18471848
JustBeforeEach(func() {
18481849
t.reconcileCryostatFully()
18491850
})
18501851
It("should configure deployment appropriately", func() {
18511852
t.expectMainDeployment()
1853+
t.expectDatabaseDeployment()
18521854
})
18531855
It("should set Database Secret in CR Status", func() {
18541856
instance := t.getCryostatInstance()
1855-
Expect(instance.Status.DatabaseSecret).To(Equal(customSecret.Name))
1857+
Expect(instance.Status.DatabaseSecret).To(Equal(t.DatabaseSecret.Name))
18561858
})
18571859
It("should not generate default secret", func() {
18581860
secret := &corev1.Secret{}
@@ -2558,7 +2560,7 @@ func (c *controllerTest) commonTests() {
25582560
Expect(binding.RoleRef).To(Equal(expected.RoleRef))
25592561
})
25602562
})
2561-
Context("with additionnal label and annotation", func() {
2563+
Context("with additional label and annotation", func() {
25622564
BeforeEach(func() {
25632565
t.ReportReplicas = 1
25642566
t.objs = append(t.objs, t.NewCryostatWithAdditionalMetadata().Object)
@@ -2897,7 +2899,7 @@ func (t *cryostatTestInput) expectWaitingForCertificate() {
28972899

28982900
func (t *cryostatTestInput) expectCertificates() {
28992901
// Check certificates
2900-
certs := []*certv1.Certificate{t.NewCryostatCert(), t.NewCACert(), t.NewReportsCert(), t.NewAgentProxyCert()}
2902+
certs := []*certv1.Certificate{t.NewCryostatCert(), t.NewCACert(), t.NewReportsCert(), t.NewAgentProxyCert(), t.NewDatabaseCert(), t.NewStorageCert()}
29012903
for _, expected := range certs {
29022904
actual := &certv1.Certificate{}
29032905
err := t.Client.Get(context.Background(), types.NamespacedName{Name: expected.Name, Namespace: expected.Namespace}, actual)
@@ -2920,7 +2922,7 @@ func (t *cryostatTestInput) expectCertificates() {
29202922
err := t.Client.Get(context.Background(), types.NamespacedName{Name: expectedSecret.Name, Namespace: expectedSecret.Namespace}, secret)
29212923
Expect(err).ToNot(HaveOccurred())
29222924
t.checkMetadata(secret, expectedSecret)
2923-
Expect(secret.StringData).To(Equal(expectedSecret.StringData))
2925+
Expect(secret.Data).To(Equal(expectedSecret.Data))
29242926

29252927
// Check CA Cert secrets in each target namespace
29262928
Expect(t.TargetNamespaces).ToNot(BeEmpty())
@@ -3099,7 +3101,7 @@ func (t *cryostatTestInput) expectOAuth2ConfigMap() {
30993101
t.checkMetadata(cm, expected)
31003102
Expect(cm.Data).To(HaveLen(1))
31013103
Expect(cm.Data).To(HaveKey("alpha_config.json"))
3102-
Expect(cm.Data["alpha_config.json"]).To(MatchJSON(expected.Data["alpha_config.json"]))
3104+
Expect(cm.Data["alpha_config.json"]).To(Equal(expected.Data["alpha_config.json"]))
31033105
Expect(cm.Immutable).To(Equal(expected.Immutable))
31043106
}
31053107

@@ -3157,7 +3159,7 @@ func (t *cryostatTestInput) expectDatabaseSecret() {
31573159
// Compare to desired spec
31583160
expectedSecret := t.NewDatabaseSecret()
31593161
t.checkMetadata(secret, expectedSecret)
3160-
Expect(secret.StringData).To(Equal(expectedSecret.StringData))
3162+
Expect(secret.Data).To(Equal(expectedSecret.Data))
31613163
Expect(secret.Immutable).To(Equal(expectedSecret.Immutable))
31623164
}
31633165

@@ -3169,7 +3171,18 @@ func (t *cryostatTestInput) expectStorageSecret() {
31693171
// Compare to desired spec
31703172
expectedSecret := t.NewStorageSecret()
31713173
t.checkMetadata(secret, expectedSecret)
3172-
Expect(secret.StringData).To(Equal(expectedSecret.StringData))
3174+
Expect(secret.Data).To(Equal(expectedSecret.Data))
3175+
}
3176+
3177+
func (t *cryostatTestInput) expectOAuthCookieSecret() {
3178+
expectedSecret := t.NewAuthProxyCookieSecret()
3179+
secret := &corev1.Secret{}
3180+
err := t.Client.Get(context.Background(), types.NamespacedName{Name: expectedSecret.Name, Namespace: expectedSecret.Namespace}, secret)
3181+
Expect(err).ToNot(HaveOccurred())
3182+
3183+
// Compare to desired spec
3184+
t.checkMetadata(secret, expectedSecret)
3185+
Expect(secret.Data).To(Equal(expectedSecret.Data))
31733186
}
31743187

31753188
func (t *cryostatTestInput) expectCoreService() {
@@ -3421,6 +3434,7 @@ func (t *cryostatTestInput) checkMainPodTemplate(deployment *appsv1.Deployment,
34213434
"kind": "cryostat",
34223435
"component": "cryostat",
34233436
}))
3437+
Expect(template.Annotations).To(Equal(t.NewMainPodAnnotations()))
34243438
Expect(template.Spec.Volumes).To(ConsistOf(t.NewVolumes()))
34253439
Expect(template.Spec.SecurityContext).To(Equal(t.NewPodSecurityContext(cr)))
34263440

@@ -3499,10 +3513,10 @@ func (t *cryostatTestInput) expectMainPodTemplateHasExtraMetadata(deployment *ap
34993513
"myPodExtraLabel": "myPodLabel",
35003514
"myPodSecondExtraLabel": "myPodSecondLabel",
35013515
}))
3502-
Expect(template.Annotations).To(Equal(map[string]string{
3503-
"myPodExtraAnnotation": "myPodAnnotation",
3504-
"mySecondPodExtraAnnotation": "mySecondPodAnnotation",
3505-
}))
3516+
annotations := t.NewMainPodAnnotations()
3517+
annotations["myPodExtraAnnotation"] = "myPodAnnotation"
3518+
annotations["mySecondPodExtraAnnotation"] = "mySecondPodAnnotation"
3519+
Expect(template.Annotations).To(Equal(annotations))
35063520
}
35073521

35083522
func (t *cryostatTestInput) expectDatabaseDeployment() {
@@ -3534,6 +3548,7 @@ func (t *cryostatTestInput) expectDatabaseDeployment() {
35343548
"kind": "cryostat",
35353549
"component": "database",
35363550
}))
3551+
Expect(template.Annotations).To(Equal(t.NewDatabasePodAnnotations()))
35373552
Expect(template.Spec.Volumes).To(ConsistOf(t.NewDatabaseVolumes()))
35383553
Expect(template.Spec.SecurityContext).To(Equal(t.NewPodSecurityContext(cr)))
35393554

@@ -3587,6 +3602,7 @@ func (t *cryostatTestInput) expectStorageDeployment() {
35873602
"kind": "cryostat",
35883603
"component": "storage",
35893604
}))
3605+
Expect(template.Annotations).To(Equal(t.NewStoragePodAnnotations()))
35903606
Expect(template.Spec.Volumes).To(ConsistOf(t.NewStorageVolumes()))
35913607
Expect(template.Spec.SecurityContext).To(Equal(t.NewPodSecurityContext(cr)))
35923608

@@ -3643,6 +3659,7 @@ func (t *cryostatTestInput) checkReportsDeployment() {
36433659
"kind": "cryostat",
36443660
"component": "reports",
36453661
}))
3662+
Expect(template.Annotations).To(Equal(t.NewReportsPodAnnotations()))
36463663
Expect(template.Spec.Volumes).To(ConsistOf(t.NewReportsVolumes()))
36473664
Expect(template.Spec.SecurityContext).To(Equal(t.NewReportPodSecurityContext(cr)))
36483665

@@ -3689,10 +3706,10 @@ func (t *cryostatTestInput) expectReportsDeploymentHasExtraMetadata() {
36893706
"myPodExtraLabel": "myPodLabel",
36903707
"myPodSecondExtraLabel": "myPodSecondLabel",
36913708
}))
3692-
Expect(template.Annotations).To(Equal(map[string]string{
3693-
"myPodExtraAnnotation": "myPodAnnotation",
3694-
"mySecondPodExtraAnnotation": "mySecondPodAnnotation",
3695-
}))
3709+
annotations := t.NewReportsPodAnnotations()
3710+
annotations["myPodExtraAnnotation"] = "myPodAnnotation"
3711+
annotations["mySecondPodExtraAnnotation"] = "mySecondPodAnnotation"
3712+
Expect(template.Annotations).To(Equal(annotations))
36963713
}
36973714

36983715
func (t *cryostatTestInput) checkDeploymentHasTemplates() {

0 commit comments

Comments
 (0)