fix(vault): token ttl conditional renew (#457)

* fix(vault-backend): token ttl conditional renew

Author: EricHorst

README.md

@@ -352,6 +352,8 @@ spec:
 
 If you use Vault Namespaces (a Vault Enterprise feature) you can set the namespace to interact with via the `VAULT_NAMESPACE` environment variable.
 
+The Vault token obtained by Kubernetes authentication will be renewed as needed. By default the token will be renewed three poller intervals (POLLER_INTERVAL_MILLISECONDS) before the token TTL expires. The default should be acceptable in most cases but the token renew threshold can also be customized by setting the `VAULT_TOKEN_RENEW_THRESHOLD` environment variable. The token renew threshold value is specified in seconds and tokens with remaining TTL less than this number of seconds will be renewed. In order to minimize token renewal load on the Vault server it is suggested that Kubernetes auth tokens issued by Vault have a TTL of at least ten times the poller interval so that they are renewed less frequently. A longer token TTL results in a lower  token renewal load on Vault.
+
 If Vault uses a certificate issued by a self-signed CA you will need to provide that certificate:
 
 ```sh

config/environment.js

@@ -19,6 +19,7 @@ if (environment === 'development') {
 const vaultEndpoint = process.env.VAULT_ADDR || 'http://127.0.0.1:8200'
 // Grab the vault namespace from the environment
 const vaultNamespace = process.env.VAULT_NAMESPACE || null
+const vaultTokenRenewThreshold = process.env.VAULT_TOKEN_RENEW_THRESHOLD || null
 
 const pollerIntervalMilliseconds = process.env.POLLER_INTERVAL_MILLISECONDS
   ? Number(process.env.POLLER_INTERVAL_MILLISECONDS) : 10000
@@ -40,6 +41,7 @@ const customResourceManagerDisabled = 'DISABLE_CUSTOM_RESOURCE_MANAGER' in proce
 module.exports = {
   vaultEndpoint,
   vaultNamespace,
+  vaultTokenRenewThreshold,
   environment,
   pollerIntervalMilliseconds,
   metricsPort,

config/index.js

@@ -77,7 +77,12 @@ if (envConfig.vaultNamespace) {
   }
 }
 const vaultClient = vault(vaultOptions)
-const vaultBackend = new VaultBackend({ client: vaultClient, logger })
+// The Vault token is renewed only during polling, not asynchronously. The default tokenRenewThreshold
+// is three times larger than the pollerInterval so that the token is renewed before it
+// expires and with at least one remaining poll opportunty to retry renewal if it fails.
+const vaultTokenRenewThreshold = envConfig.vaultTokenRenewThreshold
+  ? Number(envConfig.vaultTokenRenewThreshold) : 3 * envConfig.pollerIntervalMilliseconds / 1000
+const vaultBackend = new VaultBackend({ client: vaultClient, tokenRenewThreshold: vaultTokenRenewThreshold, logger })
 const azureKeyVaultBackend = new AzureKeyVaultBackend({
   credential: azureConfig.azureKeyVault(),
   logger

lib/backends/vault-backend.js

@@ -7,11 +7,13 @@ class VaultBackend extends KVBackend {
   /**
    * Create Vault backend.
    * @param {Object} client - Client for interacting with Vault.
+   * @param {Number} tokenRenewThreshold - tokens are renewed when ttl reaches this threshold
    * @param {Object} logger - Logger for logging stuff.
    */
-  constructor ({ client, logger }) {
+  constructor ({ client, tokenRenewThreshold, logger }) {
     super({ logger })
     this._client = client
+    this._tokenRenewThreshold = tokenRenewThreshold
   }
 
   /**
@@ -40,15 +42,20 @@ class VaultBackend extends KVBackend {
     if (!this._client.token) {
       const jwt = this._fetchServiceAccountToken()
       this._logger.debug('fetching new token from vault')
-      const vault = await this._client.kubernetesLogin({
+      await this._client.kubernetesLogin({
         mount_point: vaultMountPoint,
         role: vaultRole,
         jwt: jwt
       })
-      this._client.token = vault.auth.client_token
     } else {
-      this._logger.debug('renewing existing token from vault')
-      this._client.tokenRenewSelf()
+      this._logger.debug('checking vault token expiry')
+      const tokenStatus = await this._client.tokenLookupSelf()
+      this._logger.debug(`vault token valid for ${tokenStatus.data.ttl} seconds, renews at ${this._tokenRenewThreshold}`)
+
+      if (Number(tokenStatus.data.ttl) <= this._tokenRenewThreshold) {
+        this._logger.debug('renewing vault token')
+        await this._client.tokenRenewSelf()
+      }
     }
 
     this._logger.debug(`reading secret key ${key} from vault`)

lib/backends/vault-backend.test.js

@@ -34,18 +34,32 @@ describe('VaultBackend', () => {
     }
   }
 
+  const vaultTokenRenewThreshold = 30
+  const mockTokenLookupResultMustRenew = {
+    data: {
+      ttl: 15
+    }
+  }
+  const mockTokenLookupResultNoRenew = {
+    data: {
+      ttl: 60
+    }
+  }
+
   beforeEach(() => {
     clientMock = sinon.mock()
 
     vaultBackend = new VaultBackend({
       client: clientMock,
+      tokenRenewThreshold: vaultTokenRenewThreshold,
       logger
     })
   })
 
   describe('_get', () => {
     beforeEach(() => {
       clientMock.read = sinon.stub().returns(kv2Secret)
+      clientMock.tokenLookupSelf = sinon.stub().returns(mockTokenLookupResultMustRenew)
       clientMock.tokenRenewSelf = sinon.stub().returns(true)
       clientMock.kubernetesLogin = sinon.stub().returns({
         auth: {
@@ -133,8 +147,9 @@ describe('VaultBackend', () => {
       expect(secretPropertyValue).equals(quotedSecretValue)
     })
 
-    it('returns secret property value after renewing token if a token exists', async () => {
+    it('returns secret property value after renewing token if a token exists that needs renewal', async () => {
       clientMock.token = 'an-existing-token'
+      clientMock.tokenLookupSelf = sinon.stub().returns(mockTokenLookupResultMustRenew)
 
       const secretPropertyValue = await vaultBackend._get({
         specOptions: {
@@ -147,7 +162,10 @@ describe('VaultBackend', () => {
       // No logging into Vault...
       sinon.assert.notCalled(clientMock.kubernetesLogin)
 
-      // ... but renew the token instead ...
+      // ... but check the token instead ...
+      sinon.assert.calledOnce(clientMock.tokenLookupSelf)
+
+      // ... then renew the token ...
       sinon.assert.calledOnce(clientMock.tokenRenewSelf)
 
       // ... then we fetch the secret ...
@@ -156,11 +174,40 @@ describe('VaultBackend', () => {
       // ... and expect to get its proper value
       expect(secretPropertyValue).equals(quotedSecretValue)
     })
+
+    it('returns secret property value if a token exists that does not need renewal', async () => {
+      clientMock.token = 'an-existing-token'
+      clientMock.tokenLookupSelf = sinon.stub().returns(mockTokenLookupResultNoRenew)
+
+      const secretPropertyValue = await vaultBackend._get({
+        specOptions: {
+          vaultMountPoint: mountPoint,
+          vaultRole: role
+        },
+        key: secretKey
+      })
+
+      // No logging into Vault...
+      sinon.assert.notCalled(clientMock.kubernetesLogin)
+
+      // ... but check the token instead ...
+      sinon.assert.calledOnce(clientMock.tokenLookupSelf)
+
+      // ... and token does not need renewal ...
+      sinon.assert.notCalled(clientMock.tokenRenewSelf)
+
+      // ... then we fetch the secret ...
+      sinon.assert.calledWith(clientMock.read, secretKey)
+
+      // ... and expect to get its proper value
+      expect(secretPropertyValue).equals(quotedSecretValue)
+    })
   })
 
   describe('getSecretManifestData', () => {
     beforeEach(() => {
       clientMock.read = sinon.stub().returns(kv2Secret)
+      clientMock.tokenLookupSelf = sinon.stub().returns(mockTokenLookupResultMustRenew)
       clientMock.tokenRenewSelf = sinon.stub().returns(true)
       clientMock.kubernetesLogin = sinon.stub().returns({ auth: { client_token: '1234' } })