feat: add support for dataFrom & fix: encoding of non-string values (#196)

* feat: add support for using dataFrom

Adds support for using `dataFrom` to get all values from a secret backend

```yaml
apiVersion: kubernetes-client.io/v1
kind: ExternalSecret
metadata:
  name: postgres-credentials
secretDescriptor:
  backendType: secretsManager
  dataFrom:
  - /development/postgres/credentials
  - /development/api/credentials
  data:
  - key: /development/auth/secret
    name: authSecret
```

Author: Flydiverny

.travis.yml

@@ -3,7 +3,7 @@ language: node_js
 matrix:
   fast_finish: true
   include:
-    - node_js: '10'
+    - node_js: '12'
 # https://github.com/greenkeeperio/greenkeeper-lockfile#npm
 before_install:
 # package-lock.json was introduced in npm@5

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:10.15.1-alpine
+FROM node:12.13.0-alpine
 
 ENV NODE_ENV production
 ENV NPM_CONFIG_LOGLEVEL info

README.md

@@ -243,6 +243,40 @@ secretDescriptor:
       property: username
 ```
 
+alternatively you can use `dataFrom` and get all the values from hello-service/credentials:
+
+```yml
+apiVersion: 'kubernetes-client.io/v1'
+kind: ExternalSecret
+metadata:
+  name: hello-service
+secretDescriptor:
+  backendType: secretsManager
+  # optional: specify role to assume when retrieving the data
+  roleArn: arn:aws:iam::123456789012:role/test-role
+  dataFrom:
+    - hello-service/credentials
+```
+
+`data` and `dataFrom` can of course be combined, any naming conflicts will use the last defined, with `data` overriding `dataFrom`
+
+```yml
+apiVersion: 'kubernetes-client.io/v1'
+kind: ExternalSecret
+metadata:
+  name: hello-service
+secretDescriptor:
+  backendType: secretsManager
+  # optional: specify role to assume when retrieving the data
+  roleArn: arn:aws:iam::123456789012:role/test-role
+  dataFrom:
+    - hello-service/credentials
+  data:
+    - key: hello-service/migration-credentials
+      name: password
+      property: password
+```
+
 ## Metrics
 
 kubernetes-external-secrets exposes the following metrics over a prometheus endpoint:

lib/backends/kv-backend.js

@@ -15,20 +15,22 @@ class KVBackend extends AbstractBackend {
 
   /**
    * Fetch Kubernetes secret property values.
-   * @param {Object[]} secretProperties - Kubernetes secret properties.
-   * @param {string} secretProperties[].key - Secret key in the backend.
-   * @param {string} secretProperties[].name - Kubernetes Secret property name.
-   * @param {string} secretProperties[].property - If the backend secret is an
+   * @param {Object[]} data - Kubernetes secret properties.
+   * @param {string} data[].key - Secret key in the backend.
+   * @param {string} data[].name - Kubernetes Secret property name.
+   * @param {string} data[].property - If the backend secret is an
    *   object, this is the property name of the value to use.
-   * @param {string} secretProperties[].roleArn - If the client should assume a role before fetching the secret
+   * @param {string} roleArn - If the client should assume a role before fetching the secret
    * @returns {Promise} Promise object representing secret property values.
    */
-  _fetchSecretPropertyValues ({ externalData, roleArn }) {
-    return Promise.all(externalData.map(async secretProperty => {
+  _fetchDataValues ({ data, roleArn }) {
+    return Promise.all(data.map(async secretProperty => {
       this._logger.info(`fetching secret property ${secretProperty.name} with role: ${roleArn || 'no role set'}`)
-      const value = await this._get({ secretKey: secretProperty.key, roleArn })
+      const plainOrObjValue = await this._get({ secretKey: secretProperty.key, roleArn })
+      const shouldParseValue = 'property' in secretProperty
 
-      if ('property' in secretProperty) {
+      let value = plainOrObjValue
+      if (shouldParseValue) {
         let parsedValue
         try {
           parsedValue = JSON.parse(value)
@@ -43,10 +45,30 @@ class KVBackend extends AbstractBackend {
           throw new Error(`Could not find property ${secretProperty.property} in ${secretProperty.key}`)
         }
 
-        return parsedValue[secretProperty.property]
+        value = parsedValue[secretProperty.property]
       }
 
-      return value
+      return { [secretProperty.name]: value }
+    }))
+  }
+
+  /**
+   * Fetch Kubernetes secret property values.
+   * @param {string[]} dataFrom - Array of secret keys in the backend
+   * @param {string} roleArn - If the client should assume a role before fetching the secret
+   * @returns {Promise} Promise object representing secret property values.
+   */
+  _fetchDataFromValues ({ dataFrom, roleArn }) {
+    return Promise.all(dataFrom.map(async secretKey => {
+      this._logger.info(`fetching secret ${secretKey} with role: ${roleArn || 'no role set'}`)
+      const value = await this._get({ secretKey, roleArn })
+
+      try {
+        return JSON.parse(value)
+      } catch (err) {
+        this._logger.warn(`Failed to JSON.parse value for '${secretKey}',` +
+           ` please verify that your secret value is correctly formatted as JSON.`)
+      }
     }))
   }
 
@@ -62,18 +84,33 @@ class KVBackend extends AbstractBackend {
    * @param {SecretDescriptor} secretDescriptor - Kubernetes secret descriptor.
    * @returns {Promise} Promise object representing Kubernetes secret manifest data.
    */
-  async getSecretManifestData ({ secretDescriptor }) {
-    const data = {}
-    // Use secretDescriptor.properties to be backwards compatible.
-    const externalData = secretDescriptor.data || secretDescriptor.properties
-    const secretPropertyValues = await this._fetchSecretPropertyValues({
-      externalData,
-      roleArn: secretDescriptor.roleArn
-    })
-    externalData.forEach((secretProperty, index) => {
-      data[secretProperty.name] = (Buffer.from(secretPropertyValues[index], 'utf8')).toString('base64')
-    })
-    return data
+  async getSecretManifestData ({
+    secretDescriptor: {
+      // Use secretDescriptor.properties to be backwards compatible.
+      properties = [],
+      data = properties,
+      dataFrom = [],
+      roleArn
+    }
+  }) {
+    const [dataFromValues, dataValues] = await Promise.all([
+      this._fetchDataFromValues({ dataFrom, roleArn }),
+      this._fetchDataValues({ data, roleArn })
+    ])
+
+    const plainValues = dataFromValues.concat(dataValues)
+      .reduce((acc, parsedValue) => ({
+        ...acc,
+        ...parsedValue
+      }), {})
+
+    const encodedEntries = Object.entries(plainValues)
+      .map(([name, plainValue]) => [
+        name,
+        (Buffer.from(`${plainValue}`, 'utf8')).toString('base64')
+      ])
+
+    return Object.fromEntries(encodedEntries)
   }
 }