feat(multitenancy): Allow to watch ExternalSecrets in specific namespaces (#548)

* feat: allow to watch externalsecrets in specified namespaces

* enable debug for e2e test

* refactor: call startWatcher for each namespace instead of handling multi namespaces inside the watcher method

* chore: update e2e/README.md

* chore: remove DEBUG log for e2e workflow

* chore: update readme, text + add env to chart readme

* chore: update chart readme add format for watched namespaces env

* fix: add WATCHED_NAMESPACES default value to helm chart values.yaml

Co-authored-by: Markus Maga <[email protected]>

Author: aabouzaid

.github/workflows/workflow.yml

@@ -36,3 +36,4 @@ jobs:
           helm init --client-only
         if: matrix.helmVersion == 'V2'
       - run: ./e2e/run-e2e-suite.sh ${{ matrix.disableCustomResourceManager }} ${{ matrix.helmVersion }}
+

README.md

@@ -230,9 +230,12 @@ spec:
       name: .dockerconfigjson
 ```
 
-## Enforcing naming conventions for backend keys
+## Scoping access
 
-by default an `ExternalSecret` may access arbitrary keys from the backend e.g.
+### Using Namespace annotation
+
+Enforcing naming conventions for backend keys could be done by using namespace annotations.
+By default an `ExternalSecret` may access arbitrary keys from the backend e.g.
 
 ```yml
   data:
@@ -256,6 +259,31 @@ metadata:
     externalsecrets.kubernetes-client.io/permitted-key-name: "/dev/cluster1/core-namespace/.*"
 ```
 
+### Using ExternalSecret controller config
+
+ExternalSecret config allows scoping the access of kubernetes-external-secrets controller.
+This allows to deploy multi kubernetes-external-secrets instances in the same cluster
+and each instance can access a set of predefined namespaces.
+
+To enable this option, set the env var in the controller side with a list of namespaces:
+```yaml
+env:
+  WATCHED_NAMESPACES: "default,qa,dev"
+```
+
+Finally, in case more than one kubernetes-external-secrets is deployed,
+it's recommended to make only one deployment manage the CRDs.
+To disable CRD management in the other deployments,
+to avoid having them fighting over the CRD.
+
+That can be done in the controller side by setting the env var:
+```yaml
+env:
+  DISABLE_CUSTOM_RESOURCE_MANAGER: true
+```
+
+Or in Helm, by setting `customResourceManagerDisabled=true`.
+
 ## Deprecations
 
 A few properties has changed name overtime, we still maintain backwards compatbility with these but they will eventually be removed, and they are not validated using the CRD validation.

bin/daemon.js

@@ -26,7 +26,8 @@ const {
   rolePermittedAnnotation,
   namingPermittedAnnotation,
   enforceNamespaceAnnotation,
-  watchTimeout
+  watchTimeout,
+  watchedNamespaces
 } = require('../config')
 
 async function main () {
@@ -37,6 +38,7 @@ async function main () {
 
   const externalSecretEvents = getExternalSecretEvents({
     kubeClient,
+    watchedNamespaces,
     customResourceManifest,
     logger,
     watchTimeout

charts/kubernetes-external-secrets/README.md

@@ -84,6 +84,7 @@ The following table lists the configurable parameters of the `kubernetes-externa
 | `env.VAULT_ADDR`                          | Endpoint for the Vault backend, if using Vault                                                                                    | `http://127.0.0.1:8200`                |
 | `env.DISABLE_POLLING`                     | Disables backend polling and only updates secrets when ExternalSecret is modified, setting this to any value will disable polling | `nil`                                 |
 | `env.WATCH_TIMEOUT`                     | Restarts the external secrets resource watcher if no events have been seen in this time period (miliseconds) | `60000`                                 |
+| `env.WATCHED_NAMESPACES`                     | Limits which namespaces the controller will watch, by default all namespaces will be watched. Comma separated list `qa,stage` | `''`                                 |
 | `envVarsFromSecret.AWS_ACCESS_KEY_ID`     | Set AWS_ACCESS_KEY_ID (from a secret) in Deployment Pod                                                                           |                                       |
 | `envVarsFromSecret.AWS_SECRET_ACCESS_KEY` | Set AWS_SECRET_ACCESS_KEY (from a secret) in Deployment Pod                                                                       |                                       |
 | `envVarsFromSecret.AZURE_TENANT_ID`       | Set AZURE_TENANT_ID (from a secret) in Deployment Pod                                                                             |                                       |

charts/kubernetes-external-secrets/values.yaml

@@ -16,6 +16,7 @@ env:
   AWS_DEFAULT_REGION: us-west-2
   POLLER_INTERVAL_MILLISECONDS: 10000  # Caution, setting this frequency may incur additional charges on some platforms
   WATCH_TIMEOUT: 60000
+  WATCHED_NAMESPACES: '' # Comma separated list of namespaces, empty or unset means ALL namespaces.
   LOG_LEVEL: info
   LOG_MESSAGE_KEY: 'msg'
   # Print logs level as string ("info") rather than integer (30)

config/environment.js

@@ -41,6 +41,17 @@ const metricsPort = process.env.METRICS_PORT || 3001
 const customResourceManagerDisabled = 'DISABLE_CUSTOM_RESOURCE_MANAGER' in process.env
 const watchTimeout = process.env.WATCH_TIMEOUT ? parseInt(process.env.WATCH_TIMEOUT) : 60000
 
+// A comma-separated list of watched namespaces. If set, only ExternalSecrets in those namespaces will be handled.
+let watchedNamespaces = process.env.WATCHED_NAMESPACES || ''
+
+// Return an array after splitting the watched namespaces string and clean up user input.
+watchedNamespaces = watchedNamespaces
+  .split(',')
+  // Remove any extra spaces.
+  .map(namespace => { return namespace.trim() })
+  // Remove empty values (in case there is a tailing comma).
+  .filter(namespace => namespace)
+
 module.exports = {
   vaultEndpoint,
   vaultNamespace,
@@ -58,5 +69,6 @@ module.exports = {
   customResourceManagerDisabled,
   useHumanReadableLogLevels,
   logMessageKey,
-  watchTimeout
+  watchTimeout,
+  watchedNamespaces
 }

e2e/README.md

@@ -33,7 +33,7 @@ kind load docker-image --name="es-dev-cluster" external-secrets:test
 kubectl apply -f ./localstack.deployment.yaml
 
 # deploy external secrets
-helm template ../charts/kubernetes-external-secrets \
+helm template e2e ../charts/kubernetes-external-secrets \
   --set image.repository=external-secrets \
   --set image.tag=test \
   --set env.LOG_LEVEL=debug \

lib/external-secret.js

@@ -24,20 +24,23 @@ function createEventQueue () {
 
 async function startWatcher ({
   kubeClient,
+  namespace,
   customResourceManifest,
   logger,
   eventQueue,
   watchTimeout
 }) {
   const deathQueue = createEventQueue()
+  const loggedNamespaceName = namespace || '*'
 
   try {
     while (true) {
-      logger.debug('Starting watch stream')
+      logger.debug('Starting watch stream for namespace %s', loggedNamespaceName)
 
       const stream = kubeClient
         .apis[customResourceManifest.spec.group]
-        .v1.watch[customResourceManifest.spec.names.plural]
+        .v1.watch
+        .namespaces(namespace)[customResourceManifest.spec.names.plural]
         .getStream()
 
       const jsonStream = new JSONStream()
@@ -51,7 +54,7 @@ async function startWatcher ({
 
         const timeMs = watchTimeout
         timeout = setTimeout(() => {
-          logger.info(`No watch event for ${timeMs} ms, restarting watcher`)
+          logger.info(`No watch event for ${timeMs} ms, restarting watcher for ${loggedNamespaceName}`)
           stream.abort()
         }, timeMs)
         timeout.unref()
@@ -63,7 +66,7 @@ async function startWatcher ({
       })
 
       jsonStream.on('error', (err) => {
-        logger.warn(err, 'Got error on stream')
+        logger.warn(err, 'Got error on stream for namespace %s', loggedNamespaceName)
         deathQueue.put('ERROR')
         clearTimeout(timeout)
       })
@@ -75,38 +78,49 @@ async function startWatcher ({
 
       const deathEvent = await deathQueue.take()
 
-      logger.info('Stopping watch stream due to event: %s', deathEvent)
+      logger.info('Stopping watch stream for namespace %s due to event: %s', loggedNamespaceName, deathEvent)
       eventQueue.put({ type: 'DELETED_ALL' })
 
       stream.abort()
     }
   } catch (err) {
-    logger.error(err, 'Watcher crashed')
+    logger.error(err, 'Watcher for namespace %s crashed', loggedNamespaceName)
+    throw err
   }
 }
 
 /**
  * Get a stream of external secret events. This implementation uses
  * watch and yields as a stream of events.
  * @param {Object} kubeClient - Client for interacting with kubernetes cluster.
+ * @param {Array} watchedNamespaces - List of scoped namespaces.
  * @param {Object} customResourceManifest - Custom resource manifest.
  * @returns {Object} An async generator that yields externalsecret events.
  */
 function getExternalSecretEvents ({
   kubeClient,
+  watchedNamespaces,
   customResourceManifest,
   logger,
   watchTimeout
 }) {
   return (async function * () {
     const eventQueue = createEventQueue()
 
-    startWatcher({
-      kubeClient,
-      customResourceManifest,
-      logger,
-      eventQueue,
-      watchTimeout
+    // If the watchedNamespaces is an empty array (i.e. no scoped access),
+    // add an empty element so all ExternalSecret resources in all namespaces will be watched.
+    const namespaceToWatch = watchedNamespaces.length ? watchedNamespaces : ['']
+
+    // Create watcher for each namespace
+    namespaceToWatch.forEach((namespace) => {
+      startWatcher({
+        namespace,
+        kubeClient,
+        customResourceManifest,
+        logger,
+        eventQueue,
+        watchTimeout
+      })
     })
 
     while (true) {

lib/external-secret.test.js

@@ -9,6 +9,7 @@ const { getExternalSecretEvents } = require('./external-secret')
 
 describe('getExternalSecretEvents', () => {
   let kubeClientMock
+  let watchedNamespaces
   let externalSecretsApiMock
   let fakeCustomResourceManifest
   let loggerMock
@@ -26,19 +27,29 @@ describe('getExternalSecretEvents', () => {
     externalSecretsApiMock = sinon.mock()
 
     mockedStream = new Readable()
-    mockedStream._read = () => {}
-    mockedStream.abort = () => {}
+    mockedStream._read = () => { }
 
     externalSecretsApiMock.get = sinon.stub()
-    kubeClientMock = sinon.mock()
-    kubeClientMock.apis = sinon.mock()
-    kubeClientMock.apis['kubernetes-client.io'] = sinon.mock()
-    kubeClientMock.apis['kubernetes-client.io'].v1 = sinon.mock()
-    kubeClientMock.apis['kubernetes-client.io'].v1.watch = sinon.mock()
-    kubeClientMock.apis['kubernetes-client.io']
-      .v1.watch.externalsecrets = sinon.mock()
-    kubeClientMock.apis['kubernetes-client.io']
-      .v1.watch.externalsecrets.getStream = () => mockedStream
+
+    kubeClientMock = {
+      apis: {
+        'kubernetes-client.io': {
+          v1: {
+            watch: {
+              namespaces: () => {
+                return {
+                  externalsecrets: {
+                    getStream: () => mockedStream
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    watchedNamespaces = []
 
     loggerMock = sinon.mock()
     loggerMock.info = sinon.stub()
@@ -60,6 +71,7 @@ describe('getExternalSecretEvents', () => {
 
     const events = getExternalSecretEvents({
       kubeClient: kubeClientMock,
+      watchedNamespaces: watchedNamespaces,
       customResourceManifest: fakeCustomResourceManifest,
       logger: loggerMock,
       watchTimeout: 5000