feat(aws-ssm): Add support to get parameters by path (#603)

* Adding support to scrape full paths instead of each individual key from SSM

* Multiple changes related with improvements
Updating Readme
Improving code with users suggestions
Lint fixes
Adding tests for ssm path feature

* Fixing additional lint issues with tests

Author: rjmsilveira

README.md

@@ -391,6 +391,33 @@ spec:
       property: password
 ```
 
+# AWS SSM Parameter Store
+
+You can scrape values from SSM Parameter Store individually or by providing a path to fetch all keys inside.
+
+Additionally you can also scrape all sub paths (child paths) if you need to. The default is not to scrape child paths
+
+```yml
+apiVersion: kubernetes-client.io/v1
+kind: ExternalSecret
+metadata:
+  name: hello-service
+spec:
+  backendType: secretsManager
+  # optional: specify role to assume when retrieving the data
+  roleArn: arn:aws:iam::123456789012:role/test-role
+  # optional: specify region
+  region: us-east-1
+  data:
+    - key: /foo/name
+      name: fooName
+    - path: /extra-people/
+      recursive: false
+```
+
+
+
+
 ### Hashicorp Vault
 
 kubernetes-external-secrets supports fetching secrets from [Hashicorp Vault](https://www.vaultproject.io/), using the [Kubernetes authentication method](https://www.vaultproject.io/docs/auth/kubernetes).

charts/kubernetes-external-secrets/crds/kubernetes-client.io_externalsecrets_crd.yaml

@@ -72,25 +72,36 @@ spec:
               type: array
               items:
                 type: object
-                properties:
-                  key:
-                    description: Secret key in backend
-                    type: string
-                  name:
-                    description: Name set for this key in the generated secret
-                    type: string
-                  property:
-                    description: Property to extract if secret in backend is a JSON object
-                  isBinary:
-                    description: >-
-                      Whether the backend secret shall be treated as binary data
-                      represented by a base64-encoded string. You must set this to true
-                      for any base64-encoded binary data in the backend - to ensure it
-                      is not encoded in base64 again. Default is false.
-                    type: boolean
-                required:
-                  - name
-                  - key
+                anyOf:
+                  - properties:
+                      key:
+                        description: Secret key in backend
+                        type: string
+                      name:
+                        description: Name set for this key in the generated secret
+                        type: string
+                      property:
+                        description: Property to extract if secret in backend is a JSON object
+                      isBinary:
+                        description: >-
+                          Whether the backend secret shall be treated as binary data
+                          represented by a base64-encoded string. You must set this to true
+                          for any base64-encoded binary data in the backend - to ensure it
+                          is not encoded in base64 again. Default is false.
+                        type: boolean
+                    required:
+                      - key
+                      - name
+                  - properties:
+                      path:
+                        description: >-
+                          Path from SSM to scrape secrets
+                          This will fetch all secrets and use the key from the secret as variable name
+                      recursive:
+                        description: Allow to recurse thru all child keys on a given path
+                        type: boolean
+                    required:
+                      - path
             roleArn:
               type: string
           oneOf:

config/aws-config.js

@@ -43,15 +43,15 @@ if (stsEndpoint) {
 
 if (localstack) {
   secretsManagerConfig = {
-    endpoint: process.env.LOCALSTACK_SM_URL || 'http://localhost:4584',
+    endpoint: process.env.LOCALSTACK_SM_URL || 'http://localhost:4566',
     region: process.env.AWS_REGION || 'us-west-2'
   }
   systemManagerConfig = {
-    endpoint: process.env.LOCALSTACK_SSM_URL || 'http://localhost:4583',
+    endpoint: process.env.LOCALSTACK_SSM_URL || 'http://localhost:4566',
     region: process.env.AWS_REGION || 'us-west-2'
   }
   stsConfig = {
-    endpoint: process.env.LOCALSTACK_STS_URL || 'http://localhost:4592',
+    endpoint: process.env.LOCALSTACK_STS_URL || 'http://localhost:4566',
     region: process.env.AWS_REGION || 'us-west-2'
   }
 }

e2e/tests/ssm.test.js

@@ -53,6 +53,54 @@ describe('ssm', async () => {
     expect(secret.body.data.name).to.equal('Zm9v')
   })
 
+  it('should pull existing secrets from ssm path and create a secret from it', async () => {
+    const name1 = await putParameter({
+      Name: `/e2e/${uuid}-names/name1`,
+      Type: 'String',
+      Value: 'foo'
+    }).catch(err => {
+      expect(err).to.equal(null)
+    })
+
+    const name2 = await putParameter({
+      Name: `/e2e/${uuid}-names/name2`,
+      Type: 'String',
+      Value: 'bar'
+    }).catch(err => {
+      expect(err).to.equal(null)
+    })
+
+    const result = await kubeClient
+      .apis[customResourceManifest.spec.group]
+      .v1.namespaces('default')[customResourceManifest.spec.names.plural]
+      .post({
+        body: {
+          apiVersion: 'kubernetes-client.io/v1',
+          kind: 'ExternalSecret',
+          metadata: {
+            name: `e2e-ssm-${uuid}-names`
+          },
+          spec: {
+            backendType: 'systemManager',
+            data: [
+              {
+                path: `/e2e/${uuid}-names`
+              }
+            ]
+          }
+        }
+      })
+
+    expect(name1).to.not.equal(undefined)
+    expect(name2).to.not.equal(undefined)
+    expect(result).to.not.equal(undefined)
+    expect(result.statusCode).to.equal(201)
+
+    const secret = await waitForSecret('default', `e2e-ssm-${uuid}-names`)
+    expect(secret.body.data.name1).to.equal('Zm9v') // Expect base64 foo
+    expect(secret.body.data.name2).to.equal('YmFy') // Expect base64 bar
+  })
+
   it('should pull existing secret from ssm in a different region', async () => {
     const ssmEU = awsConfig.systemManagerFactory({
       region: 'eu-west-1'

examples/ssm-example.yaml

@@ -7,7 +7,13 @@ spec:
   # optional: specify role to assume when retrieving the data
   roleArn: arn:aws:iam::123456789012:role/test-role
   # optional: specify region
-  region: eu-west-1
+  region: us-west-2
   data:
-    - key: /foo/name1
+    # Can either be key+name or all keys from a given path or even both
+    # Order below is important. Values are fetched from SSM in the same order you put them here (top to bottom)
+    # This means that if a given key is found duplicate, the last value found has precedence
+    - key: /foo/name
       name: variable-name
+    - path: /bar/
+      # optional: choose whether to scrape all child paths or not. Default is false
+      recursive: false

lib/backends/kv-backend.js

@@ -18,6 +18,7 @@ class KVBackend extends AbstractBackend {
    * @param {Object[]} data - Kubernetes secret properties.
    * @param {string} data[].key - Secret key in the backend.
    * @param {string} data[].name - Kubernetes Secret property name.
+   * @param {string} data[].path - Kubernetes Secret path to fetch keys from.
    * @param {string} data[].property - If the backend secret is an
    *   object, this is the property name of the value to use.
    * @param {string} data[].isBinary - If the backend secret shall be treated
@@ -27,8 +28,24 @@ class KVBackend extends AbstractBackend {
    */
   _fetchDataValues ({ data, specOptions }) {
     return Promise.all(data.map(async dataItem => {
-      const { name, property = null, key, ...keyOptions } = dataItem
-      const plainOrObjValue = await this._get({ key, keyOptions, specOptions })
+      const { name, property = null, key, path, ...keyOptions } = dataItem
+
+      let response = {}
+      let plainOrObjValue
+
+      // Supporting fetching by key or by path
+      // If 'path' is not defined, we can assume 'key' will exist due to CRD validation
+      let singleParameterKey = true
+      if (path) { singleParameterKey = false }
+
+      if (singleParameterKey) {
+        // Single secret
+        plainOrObjValue = await this._get({ key, keyOptions, specOptions })
+      } else {
+        // All secrets inside the specified path
+        plainOrObjValue = await this._getByPath({ path, keyOptions, specOptions })
+      }
+
       const shouldParseValue = 'property' in dataItem
       const isBinary = 'isBinary' in dataItem && dataItem.isBinary === true
 
@@ -39,8 +56,8 @@ class KVBackend extends AbstractBackend {
           parsedValue = JSON.parse(value)
         } catch (err) {
           this._logger.warn(`Failed to JSON.parse value for '${key}',` +
-           ' please verify that your secret value is correctly formatted as JSON.' +
-           ` To use plain text secret remove the 'property: ${property}'`)
+            ' please verify that your secret value is correctly formatted as JSON.' +
+            ` To use plain text secret remove the 'property: ${property}'`)
           return
         }
 
@@ -61,7 +78,17 @@ class KVBackend extends AbstractBackend {
         }
       }
 
-      return { [name]: value }
+      if (singleParameterKey) {
+        // Not path, return as is
+        response = { [name]: value }
+      } else {
+        // Returning dict with path keys and values
+        for (const records in value) {
+          response[records] = value[records]
+        }
+      }
+
+      return response
     }))
   }
 
@@ -79,7 +106,7 @@ class KVBackend extends AbstractBackend {
         return JSON.parse(value)
       } catch (err) {
         this._logger.warn(`Failed to JSON.parse value for '${key}',` +
-           ' please verify that your secret value is correctly formatted as JSON.')
+          ' please verify that your secret value is correctly formatted as JSON.')
       }
     }))
   }
@@ -95,6 +122,17 @@ class KVBackend extends AbstractBackend {
     throw new Error('_get not implemented')
   }
 
+  /**
+   * Get a secret property value from Key Value backend.
+   * @param {string} path - Path from where to fetch secrets on the backend.
+   * @param {string} keyOptions - Options for this specific key, eg version etc.
+   * @param {string} specOptions - Options for this external secret, eg role
+   * @returns {Promise} Promise object representing secret property values.
+   */
+  _getByPath ({ path, keyOptions, specOptions }) {
+    throw new Error('_getByPath not implemented')
+  }
+
   /**
    * Convert secret value to buffer
    * @param {(string|Buffer|object)} plainValue - plain secret value

lib/backends/system-manager-backend.js

@@ -24,11 +24,10 @@ class SystemManagerBackend extends KVBackend {
    * @returns {Promise} Promise object representing secret property value.
    */
   async _get ({ key, specOptions: { roleArn, region } }) {
-    this._logger.info(`fetching secret property ${key} with role: ${roleArn || 'pods role'} in region ${region}`)
-
     let client = this._client
     let factoryArgs = null
     if (roleArn) {
+      this._logger.info(`fetching secret property ${key} with role: ${roleArn} in region ${region}`)
       const credentials = this._assumeRole({
         RoleArn: roleArn,
         RoleSessionName: 'k8s-external-secrets'
@@ -37,6 +36,8 @@ class SystemManagerBackend extends KVBackend {
         ...factoryArgs,
         credentials
       }
+    } else {
+      this._logger.info(`fetching secret property ${key} with pod role in region ${region}`)
     }
     if (region) {
       factoryArgs = {
@@ -63,6 +64,80 @@ class SystemManagerBackend extends KVBackend {
       throw err
     }
   }
+
+  /**
+   * Get secret property value from System Manager.
+   * @param {string} path - Key used to store secret property value in System Manager.
+   * @param {object} specOptions - Options for this external secret, eg role
+   * @param {string} specOptions.roleArn - IAM role arn to assume
+   * @returns {Promise} Promise object representing secret property value.
+   */
+  async _getByPath ({ path, keyOptions, specOptions: { roleArn, region } }) {
+    let client = this._client
+    let factoryArgs = null
+    const recursive = keyOptions.recursive || false
+
+    this._logger.info(`fetching all secrets ${recursive ? '(recursively)' : ''} inside path ${path} with role ${roleArn !== ' from pod'} in region ${region}`)
+
+    if (roleArn) {
+      const credentials = this._assumeRole({
+        RoleArn: roleArn,
+        RoleSessionName: 'k8s-external-secrets'
+      })
+      factoryArgs = {
+        ...factoryArgs,
+        credentials
+      }
+    }
+    if (region) {
+      factoryArgs = {
+        ...factoryArgs,
+        region
+      }
+    }
+    if (factoryArgs) {
+      client = this._clientFactory(factoryArgs)
+    }
+    try {
+      const getAllParameters = async () => {
+        const EMPTY = Symbol('empty')
+        this._logger.info(`fetching parameters for path ${path}`)
+        const res = []
+        for await (const lf of (async function * () {
+          let NextToken = EMPTY
+          while (NextToken || NextToken === EMPTY) {
+            const parameters = await client.getParametersByPath({
+              Path: path,
+              WithDecryption: true,
+              Recursive: recursive,
+              NextToken: NextToken !== EMPTY ? NextToken : undefined
+            }).promise()
+            yield * parameters.Parameters
+            NextToken = parameters.NextToken
+          }
+        })()) {
+          res.push(lf)
+        }
+        return res
+      }
+
+      const parameters = {}
+      const ssmData = await getAllParameters()
+      for (const ssmRecord in ssmData) {
+        const paramName = require('path').basename(ssmData[String(ssmRecord)].Name)
+        const paramValue = ssmData[ssmRecord].Value
+        parameters[paramName] = paramValue
+      }
+
+      return parameters
+    } catch (err) {
+      if (err.code === 'ParameterNotFound' && (!err.message || err.message === 'null')) {
+        err.message = `ParameterNotFound: ${path} could not be found.`
+      }
+
+      throw err
+    }
+  }
 }
 
 module.exports = SystemManagerBackend

package.json

@@ -7,7 +7,7 @@
     "coverage": "nyc ./node_modules/mocha/bin/_mocha --recursive lib",
     "lint": "eslint --fix --ignore-pattern /coverage/ ./",
     "local": "LOCALSTACK=1 AWS_ACCESS_KEY_ID=foobar AWS_SECRET_ACCESS_KEY=foobar nodemon",
-    "localstack": "docker run -it -p 4583:4583 -p 4584:4584 -p 4592:4592 -p 9999:8080 -e SERVICES=ssm,secretsmanager,sts -e DEBUG=1 --rm localstack/localstack:0.10.5",
+    "localstack": "docker run -it -p 4566:4566 -p 9999:8080 -e SERVICES=ssm,secretsmanager,sts -e DEBUG=1 --rm localstack/localstack:latest",
     "release": "standard-version --tag-prefix='' && ./release.sh",
     "start": "./bin/daemon.js",
     "nodemon": "nodemon ./bin/daemon.js",