feat(poller): lodash template preprocess for externalsecret.spec.template field (#626)

Author: intpp

.gitignore

@@ -60,4 +60,5 @@ typings/
 # e2e test stuff
 e2e/**/.kubeconfig
 
+# IDE
 .idea/

README.md

@@ -236,6 +236,133 @@ spec:
       name: .dockerconfigjson
 ```
 
+### Templating
+
+Kubernetes External Secrets supports templating in `ExternalSecret` using [lodash.template](https://lodash.com/docs/4.17.15#template).
+
+Template is applied to all `ExternalSecret.template` section of manifest.
+Data retrieved from secure backend is available via `data` variable.
+Additonal object `yaml` of instance of [js-yaml](https://github.com/nodeca/js-yaml) is available in `lodash` templates.
+It can be leveraged for easier YAML content manipulation.
+
+Templating can be used for:
+
+* Generating K8S `Secret` keys:
+  * upserting plain text via `ExternalSecret.template.stringData`
+  * upserting base64 encoded content `ExternalSecret.template.data`
+* For creating dynamic labels, annotations and other fields available in K8S `Secret` object.
+
+To demonstrate templating functionality let's assume the secure backend, e.g. Hashicorp Vaule, contains following data
+
+<table>
+<tr>
+<th>kv/extsec/secret1</th>
+<th>kv/extsec/secret2</th
+</tr>
+<tr>
+<td>
+
+```json
+{
+  "intKey": 11,
+  "objKey": {
+    "strKey": "hello world"
+  }
+}
+```
+
+</td>
+<td>
+
+```json
+{
+  "arrKey": [
+    1,
+    2,
+    3
+  ]
+}
+```
+
+</td>
+</tr>
+</table>
+
+Then, one could create following `ExternalSecret`
+
+````yaml
+apiVersion: kubernetes-client.io/v1
+kind: ExternalSecret
+metadata:
+  name: tmpl-ext-sec
+spec:
+  backendType: vault
+  data:
+  - key: kv/data/extsec/secret1
+    name: s1
+  - key: kv/data/extsec/secret2
+    name: s2
+  kvVersion: 2
+  template:
+    data:
+      file.txt: |
+        <%= Buffer.from(JSON.stringify(JSON.parse(data.s1).objKey)).toString("base64") %>
+    metadata:
+      labels:
+        label1: <%= JSON.parse(data.s1).intKey %>
+        label2: <%= JSON.parse(data.s1).objKey.strKey.replace(" ", "-") %>
+    stringData:
+      file.yaml: |
+        <%= yaml.dump(JSON.parse(data.s1)) %>
+        <% let s2 = JSON.parse(data.s2) %><% s2.arrKey.forEach((e, i) => { %>arr_<%= i %>: <%= e %>
+        <% }) %>`
+  vaultMountPoint: kubernetes
+  vaultRole: demo
+````
+
+After applying this `ExternalSecret` to K8S cluster, the operator will generate following `Secret`
+
+````yaml
+apiVersion: v1
+data:
+  file.txt: eyJzdHJLZXkiOiJoZWxsbyB3b3JsZCJ9
+  file.yaml: aW50S2V5OiAxMQpvYmpLZXk6CiAgc3RyS2V5OiBoZWxsbyB3b3JsZAoKYXJyXzA6IDEKYXJyXzE6IDIKYXJyXzI6IDMKYAo=
+  s1: eyJpbnRLZXkiOjExLCJvYmpLZXkiOnsic3RyS2V5IjoiaGVsbG8gd29ybGQifX0=
+  s2: eyJhcnJLZXkiOlsxLDIsM119
+kind: Secret
+metadata:
+  name: tmpl-ext-sec
+  labels:
+    label1: "11"
+    label2: hello-world
+type: Opaque
+````
+
+Resulting `Secret` could be inspected to see that result is generated by `lodash` templating engine
+
+````bash
+$ kubectl get secret/tmpl-ext-sec -ogo-template='{{ index .data "s1" | base64decode }}'
+{"intKey":11,"objKey":{"strKey":"hello world"}}
+
+$ kubectl get secret/tmpl-ext-sec -ogo-template='{{ index .data "s2" | base64decode }}'
+{"arrKey":[1,2,3]}
+
+$ kubectl get secret/tmpl-ext-sec -ogo-template='{{ index .data "file.txt" | base64decode }}'
+{"strKey":"hello world"}
+
+$ kubectl get secret/tmpl-ext-sec -ogo-template='{{ index .data "file.yaml" | base64decode }}'
+intKey: 11
+objKey:
+  strKey: hello world
+
+arr_0: 1
+arr_1: 2
+arr_2: 3
+
+$ kubectl get secret/tmpl-ext-sec -ogo-template='{{ .metadata.labels }}'
+map[label1:11 label2:hello-world]
+````
+
 ## Scoping access
 
 ### Using Namespace annotation

e2e/tests/secrets-manager.test.js

@@ -77,6 +77,53 @@ describe('secretsmanager', async () => {
     expect(secret.body.data.password).to.equal('MTIzNA==')
   })
 
+  it('should pull existing secret from secretsmanager and create a secret using templating', async () => {
+    let result = await createSecret({
+      Name: `e2e/${uuid}/template`,
+      SecretString: '{"secretData":"foo123"}'
+    }).catch(err => {
+      expect(err).to.equal(null)
+    })
+
+    result = await kubeClient
+      .apis[customResourceManifest.spec.group]
+      .v1.namespaces('default')[customResourceManifest.spec.names.plural]
+      .post({
+        body: {
+          apiVersion: 'kubernetes-client.io/v1',
+          kind: 'ExternalSecret',
+          metadata: {
+            name: `e2e-secretmanager-template-${uuid}`
+          },
+          spec: {
+            template: {
+              metadata: {
+                labels: {
+                  secretLabel: '<%= "Hello".concat(data.secretData) %>'
+                }
+              }
+            },
+            backendType: 'secretsManager',
+            data: [
+              {
+                key: `e2e/${uuid}/template`,
+                property: 'secretData',
+                name: 'secretData'
+              }
+            ]
+          }
+        }
+      })
+
+    expect(result).to.not.equal(undefined)
+    expect(result.statusCode).to.equal(201)
+
+    const secret = await waitForSecret('default', `e2e-secretmanager-template-${uuid}`)
+    expect(secret).to.not.equal(undefined)
+    expect(secret.body.data.secretData).to.equal('Zm9vMTIz') // foo123 is base64 Zm9vMTIz
+    expect(secret.body.metadata.labels.secretLabel).to.equal('Hellofoo123')
+  })
+
   it('should pull TLS secret from secretsmanager', async () => {
     let result = await createSecret({
       Name: `e2e/${uuid}/tls/cert`,

examples/hello-service-external-secret-vault-template.yml

@@ -0,0 +1,23 @@
+apiVersion: 'kubernetes-client.io/v1'
+kind: ExternalSecret
+metadata:
+  name: hello-service
+spec:
+  backendType: vault
+  vaultMountPoint: my-kubernetes-vault-mount-point
+  vaultRole: my-vault-role
+  kvVersion: 2
+  data:
+    - key: kv/data/test/secret1
+      name: s1
+    - key: kv/data/test/secret2
+      name: s2
+  template:
+    metadata:
+      labels:
+        world: <% let content = JSON.parse(data.s1) %><%= content.f2.f22 %>
+    stringData:
+      file.yaml: |
+        <%= yaml.dump(JSON.parse(data.s1)) %>
+        <% let s2 = JSON.parse(data.s2) %><% s2.arr.forEach((e, i) => { %>arr_<%= i %>: <%= e %>
+        <% }) %>

lib/poller.js

@@ -2,6 +2,8 @@
 
 const clonedeep = require('lodash.clonedeep')
 const merge = require('lodash.merge')
+const mapValues = require('lodash.mapvalues')
+const { compileObjectTemplateKeys } = require('./utils')
 
 /**
  * Kubernetes secret descriptor.
@@ -85,14 +87,20 @@ class Poller {
    */
   async _createSecretManifest () {
     const spec = this._spec
-    const template = spec.template || {}
+    let template = spec.template || {}
 
     // spec.type for backwards compat
     const type = template.type || spec.type || 'Opaque'
 
     const data = await this._backends[spec.backendType]
       .getSecretManifestData({ spec })
 
+    if (template && typeof template === 'object' && !Array.isArray(template)) {
+      const decodedData = mapValues(data, value => Buffer.from(value, 'base64').toString('ascii'))
+
+      template = compileObjectTemplateKeys(template, decodedData)
+    }
+
     const secretManifest = {
       apiVersion: 'v1',
       kind: 'Secret',

lib/poller.test.js

@@ -328,6 +328,99 @@ describe('Poller', () => {
         }
       })
     })
+
+    it('creates secret manifest - with lodash template', async () => {
+      const poller = pollerFactory({
+        type: 'dummy-test-type',
+        backendType: 'fakeBackendType',
+        name: 'fakeSecretName',
+        template: {
+          metadata: {
+            labels: {
+              world: '<% let content = JSON.parse(data.s1) %><%= content.f2.f22 %>'
+            }
+          },
+          stringData: {
+            'test.yaml': `
+              <%= yaml.dump(JSON.parse(data.s1)) %>
+              <% let s2 = JSON.parse(data.s2) %><% s2.arr.forEach((e, i) => { %>arr_<%= i %>: <%= e %>
+              <% }) %>
+            `
+          }
+        },
+        data: [
+          { key: 'kv/data/test/secret1', name: 's1' },
+          { key: 'kv/data/test/secret2', name: 's2' }
+        ]
+      })
+
+      backendMock.getSecretManifestData.resolves({
+        s1: 'eyJmMSI6MTEsImYyIjp7ImYyMiI6ImhlbGxvIn19Cg==', // base 64 value
+        s2: 'eyJhcnIiOlsxLDIsM119' // base 64 value
+      })
+
+      const secretManifest = await poller._createSecretManifest()
+
+      expect(secretManifest).deep.equals({
+        apiVersion: 'v1',
+        kind: 'Secret',
+        metadata: {
+          ownerReferences: [getOwnerReference()],
+          labels: {
+            world: 'hello'
+          },
+          name: 'fakeSecretName'
+        },
+        type: 'dummy-test-type',
+        data: {
+          s1: 'eyJmMSI6MTEsImYyIjp7ImYyMiI6ImhlbGxvIn19Cg==', // base 64 value
+          s2: 'eyJhcnIiOlsxLDIsM119' // base 64 value
+        },
+        stringData: {
+          'test.yaml': '\n              f1: 11\nf2:\n  f22: hello\n\n              arr_0: 1\n              arr_1: 2\n              arr_2: 3\n              \n            '
+        }
+      })
+    })
+
+    it('creates secret manifest - with lodash template (without stringData)', async () => {
+      const poller = pollerFactory({
+        type: 'dummy-test-type',
+        backendType: 'fakeBackendType',
+        name: 'fakeSecretName',
+        template: {
+          metadata: {
+            labels: {
+              world: '<% let content = JSON.parse(data.s1) %><%= content.f2.f22 %>'
+            }
+          }
+        },
+        data: [
+          { key: 'kv/data/test/secret1', name: 's1' }
+        ]
+      })
+
+      backendMock.getSecretManifestData.resolves({
+        s1: 'eyJmMSI6MTEsImYyIjp7ImYyMiI6ImhlbGxvIn19Cg==' // base 64 value
+      })
+
+      const secretManifest = await poller._createSecretManifest()
+
+      expect(secretManifest).deep.equals({
+        apiVersion: 'v1',
+        kind: 'Secret',
+        metadata: {
+          ownerReferences: [getOwnerReference()],
+          labels: {
+            world: 'hello'
+          },
+          name: 'fakeSecretName'
+        },
+        type: 'dummy-test-type',
+        data: {
+          s1: 'eyJmMSI6MTEsImYyIjp7ImYyMiI6ImhlbGxvIn19Cg==' // base 64 value
+        }
+      })
+    })
   })
 
   describe('_poll', () => {

lib/utils.js

@@ -0,0 +1,26 @@
+const yaml = require('js-yaml')
+const parseTemplate = require('lodash.template')
+const mapValues = require('lodash.mapvalues')
+
+const compileTemplate = (template, data) => parseTemplate(template, { imports: { yaml }, variable: 'data' })(data)
+
+const compileObjectTemplateKeys = (object, data) => {
+  return mapValues(object, (value) => {
+    if (value) {
+      const valueType = typeof value
+
+      if (valueType === 'string') {
+        return compileTemplate(value, data)
+      } else if (valueType === 'object' && !Array.isArray(value)) {
+        return compileObjectTemplateKeys(value, data)
+      }
+    }
+
+    return value
+  })
+}
+
+module.exports = {
+  compileTemplate,
+  compileObjectTemplateKeys
+}