fix: allow expired leases to be overwritten (#828)

Signed-off-by: Nathan Klick <[email protected]>

Author: nathanklick

.github/workflows/zxc-update-readme.yaml

@@ -86,17 +86,6 @@ jobs:
           verbosity: 3
           wait: 120s
 
-      - name: Install NVM & Dependencies
-        id: npm-deps
-        run: |
-          curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
-          export NVM_DIR="$HOME/.nvm"
-          [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
-          [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion
-          nvm install lts/hydrogen
-          nvm use lts/hydrogen
-          npm ci
-
       - name: Install gettext-base
         id: gettext-base
         run: |

README.md

@@ -7,10 +7,9 @@
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/83a423a3a1c942459127b3aec62ab0b5)](https://app.codacy.com/gh/hashgraph/solo/dashboard?utm_source=gh\&utm_medium=referral\&utm_content=\&utm_campaign=Badge_grade)
 [![codecov](https://codecov.io/gh/hashgraph/solo/graph/badge.svg?token=hBkQdB1XO5)](https://codecov.io/gh/hashgraph/solo)
 
-> [!WARNING]
+> \[!WARNING]
 > SPECIAL NOTICE: Introducing v1.0.0 comes with BREAKING CHANGES.  We have removed caching of the flags in the solo config file.  All commands will need required flags or user will need to answer the prompts.  See more details in our release notes: [release/tag/v1.0.0](https://github.com/hashgraph/solo/releases/tag/v1.0.0)
 
-
 An opinionated CLI tool to deploy and manage standalone test networks.
 
 ## Table of Contents

package.json

@@ -14,6 +14,7 @@
     "test": "cross-env MOCHA_SUITE_NAME=\"Unit Tests\" c8 --report-dir='coverage/unit' mocha 'test/unit/**/*.ts' --reporter-options configFile=mocha-multi-reporter.json,cmrOutput=mocha-junit-reporter+mochaFile+junit.xml",
     "test-e2e-all": "cross-env MOCHA_SUITE_NAME=\"Mocha E2E All Tests\" c8 --report-dir='coverage/e2e-all' mocha 'test/e2e/**/*.ts' --reporter-options configFile=mocha-multi-reporter.json,cmrOutput=mocha-junit-reporter+mochaFile+junit-e2e-all.xml",
     "test-e2e-integration": "cross-env MOCHA_SUITE_NAME=\"Mocha E2E Integration Tests\" c8 --report-dir='coverage/e2e-integration' mocha 'test/e2e/integration/**/*.ts' --reporter-options configFile=mocha-multi-reporter.json,cmrOutput=mocha-junit-reporter+mochaFile+junit-e2e-integration.xml",
+    "test-e2e-leases": "cross-env MOCHA_SUITE_NAME=\"Mocha E2E Lease Tests\" c8 --report-dir='coverage/e2e-leases' mocha 'test/e2e/integration/core/lease_manager.test.ts' --reporter-options configFile=mocha-multi-reporter.json,cmrOutput=mocha-junit-reporter+mochaFile+junit-e2e-integration.xml",
     "test-e2e-standard": "cross-env MOCHA_SUITE_NAME=\"Mocha E2E Standard Tests\" c8 --report-dir='coverage/e2e-standard' mocha 'test/e2e/**/*.ts' --ignore 'test/unit/**/*.ts' --ignore 'test/e2e/integration/**/*.ts' --ignore 'test/e2e/commands/mirror_node*.ts' --ignore 'test/e2e/commands/node*.ts' --ignore 'test/e2e/commands/separate_node*.ts' --ignore 'test/e2e/commands/relay*.ts' --reporter-options configFile=mocha-multi-reporter.json,cmrOutput=mocha-junit-reporter+mochaFile+junit-e2e-standard.xml --timeout 30000",
     "test-e2e-mirror-node": "cross-env MOCHA_SUITE_NAME=\"Mocha E2E Mirror Node Tests\" c8 --report-dir='coverage/e2e-mirror-node' mocha 'test/e2e/commands/mirror_node.test.ts' --reporter-options configFile=mocha-multi-reporter.json,cmrOutput=mocha-junit-reporter+mochaFile+junit-e2e-mirror-node.xml",
     "test-e2e-node-pem-stop": "cross-env MOCHA_SUITE_NAME=\"Mocha E2E Node PEM Stop Tests\" c8 --report-dir='coverage/e2e-node-pem-stop' mocha 'test/e2e/commands/node_pem_stop.test.ts' --reporter-options configFile=mocha-multi-reporter.json,cmrOutput=mocha-junit-reporter+mochaFile+junit-e2e-node-pem-stop.xml",

src/commands/mirror_node.ts

@@ -63,7 +63,7 @@ export class MirrorNodeCommand extends BaseCommand {
   async prepareHederaExplorerValuesArg (config: any) {
     let valuesArg = ''
 
-    const profileName = <string>this.configManager.getFlag<string>(flags.profileName)
+    const profileName = this.configManager.getFlag<string>(flags.profileName) as string
     const profileValuesFile = await this.profileManager.prepareValuesHederaExplorerChart(profileName)
     if (profileValuesFile) {
       valuesArg += this.prepareValuesFiles(profileValuesFile)

src/core/constants.ts

@@ -164,7 +164,7 @@ export const MINUTES = 60 * SECONDS
 
 export const LEASE_ACQUIRE_RETRY_TIMEOUT = +process.env.LEASE_ACQUIRE_RETRY_TIMEOUT || 20 * SECONDS
 export const MAX_LEASE_ACQUIRE_ATTEMPTS = +process.env.MAX_LEASE_ACQUIRE_ATTEMPTS || 10
-export const LEASE_RENEW_TIMEOUT = +process.env.LEASE_RENEW_TIMEOUT || 10 * SECONDS
+export const DEFAULT_LEASE_RENEW_TIMEOUT = 10 * SECONDS
 
 export const PODS_RUNNING_MAX_ATTEMPTS = +process.env.PODS_RUNNING_MAX_ATTEMPTS || 60 * 15
 export const PODS_RUNNING_DELAY = +process.env.PODS_RUNNING_DELAY || 1000

src/core/lease_manager.ts

@@ -19,11 +19,12 @@ import { flags } from '../commands/index.ts'
 import type { ConfigManager } from './config_manager.ts'
 import type { K8 } from './k8.ts'
 import type { SoloLogger } from './logging.ts'
-import { LEASE_RENEW_TIMEOUT, LEASE_ACQUIRE_RETRY_TIMEOUT, MAX_LEASE_ACQUIRE_ATTEMPTS, OS_USERNAME } from './constants.ts'
+import { DEFAULT_LEASE_RENEW_TIMEOUT, LEASE_ACQUIRE_RETRY_TIMEOUT, MAX_LEASE_ACQUIRE_ATTEMPTS, OS_USERNAME } from './constants.ts'
 import type { ListrTaskWrapper } from 'listr2'
 import chalk from 'chalk'
 import { sleep } from './helpers.ts'
 import { LeaseWrapper } from './lease_wrapper.ts'
+import type { V1Lease } from '@kubernetes/client-node'
 
 export class LeaseManager {
   constructor (
@@ -76,7 +77,8 @@ export class LeaseManager {
     }
 
     //? Renew lease with the callback
-    const intervalId = setInterval(renewLeaseCallback, LEASE_RENEW_TIMEOUT)
+    const renewalTimeout = +process.env.LEASE_RENEW_TIMEOUT || DEFAULT_LEASE_RENEW_TIMEOUT
+    const intervalId = setInterval(renewLeaseCallback, renewalTimeout)
 
     const releaseLeaseCallback = async () => {
       //? Stop renewing the lease once release callback is called
@@ -94,6 +96,21 @@ export class LeaseManager {
     return { releaseLease: releaseLeaseCallback }
   }
 
+  private async tryAcquireLease (username: string,
+                                 leaseName: string,
+                                  namespace: string,
+                                 task: ListrTaskWrapper<any, any, any>,
+                                 title: string): Promise<V1Lease> {
+    try {
+       return await this.k8.readNamespacedLease(leaseName, namespace)
+    } catch (error) {
+      if (error.meta.statusCode !== 404) {
+        task.title = `${title} - ${chalk.red(`failed to acquire lease, unexpected server response ${error.meta.statusCode}!`)}`
+      }
+      return Promise.resolve(null)
+    }
+  }
+
   private async acquireLeaseOrRetry (
     username: string,
     leaseName: string,
@@ -105,23 +122,10 @@ export class LeaseManager {
   ): Promise<void> {
     if (!attempt) attempt = 1
 
-    let exists = false
-
-    try {
-      const lease = await this.k8.readNamespacedLease(leaseName, namespace)
-
-      exists = !!lease
-    } catch (error) {
-      if (error.meta.statusCode !== 404) {
-        task.title = `${title} - ${chalk.red(`failed to acquire lease, unexpected server response ${error.meta.statusCode}!`)}` +
-          `, attempt: ${chalk.cyan(attempt.toString())}/${chalk.cyan(maxAttempts.toString())}`
-
-        throw new SoloError(`Failed to acquire lease: ${error.message}`)
-      }
-    }
+    let lease = await this.tryAcquireLease(username, leaseName, namespace, task, title)
 
     //? In case the lease is already acquired retry after cooldown
-    if (exists) {
+    while (!!lease && !this.isLeaseExpired(lease)) {
       attempt++
 
       if (attempt === maxAttempts) {
@@ -137,8 +141,11 @@ export class LeaseManager {
         `, attempt: ${chalk.cyan(attempt.toString())}/${chalk.cyan(maxAttempts.toString())}`
 
       await sleep(LEASE_ACQUIRE_RETRY_TIMEOUT)
+      lease = await this.tryAcquireLease(username, leaseName, namespace, task, title)
+    }
 
-      return this.acquireLeaseOrRetry(username, leaseName, namespace, task, title, attempt)
+    if (lease) {
+      await this.k8.deleteNamespacedLease(leaseName, namespace)
     }
 
     await this.k8.createNamespacedLease(namespace, leaseName, username)
@@ -154,4 +161,18 @@ export class LeaseManager {
     if (!await this.k8.hasNamespace(namespace)) return null
     return namespace
   }
+
+  private isLeaseExpired (lease: V1Lease) : boolean {
+    const now = Date.now()
+    const duration = lease.spec?.leaseDurationSeconds ? lease.spec?.leaseDurationSeconds : 20
+    let acquired = lease.spec?.acquireTime
+
+    if (lease.spec.renewTime) {
+      acquired = lease.spec?.renewTime
+    }
+
+    const deltaSec = (now - new Date(acquired).valueOf()) / 1000
+
+    return deltaSec > duration
+  }
 }

src/core/templates.ts

@@ -21,7 +21,7 @@ import { DataValidationError, SoloError, IllegalArgumentError, MissingArgumentEr
 import { constants } from './index.ts'
 import { type AccountId } from '@hashgraph/sdk'
 import type { NodeAlias, PodName } from '../types/aliases.ts'
-import { GrpcProxyTlsEnums} from './enumerations.ts'
+import { GrpcProxyTlsEnums } from './enumerations.ts'
 
 export class Templates {
   public static renderNetworkPodName (nodeAlias: NodeAlias): PodName {

test/e2e/integration/core/lease_manager.test.ts

@@ -20,7 +20,12 @@ import { expect } from 'chai'
 import { flags } from '../../../../src/commands/index.ts'
 import { e2eTestSuite, getDefaultArgv, TEST_CLUSTER } from '../../../test_util.ts'
 import * as version from '../../../../version.ts'
-import { LEASE_ACQUIRE_RETRY_TIMEOUT, MAX_LEASE_ACQUIRE_ATTEMPTS, MINUTES } from '../../../../src/core/constants.ts'
+import {
+  DEFAULT_LEASE_RENEW_TIMEOUT,
+  LEASE_ACQUIRE_RETRY_TIMEOUT,
+  MAX_LEASE_ACQUIRE_ATTEMPTS,
+  MINUTES
+} from '../../../../src/core/constants.ts'
 import { sleep } from '../../../../src/core/helpers.js'
 
 const namespace = 'lease-mngr-e2e'
@@ -75,5 +80,25 @@ e2eTestSuite(namespace, argv, undefined, undefined, undefined, undefined, undefi
 
       await initialLease.release()
     }).timeout(3 * MINUTES)
+
+    it('expired leases should be overwritten', async () => {
+      process.env.LEASE_RENEW_TIMEOUT = (DEFAULT_LEASE_RENEW_TIMEOUT * 4).toString()
+      const initialLease = leaseManager.instantiateLease()
+      const title = 'Initial Lease'
+      // @ts-ignore to access private property
+      await initialLease.acquireTask({ title }, title)
+
+      // Ensure lease expires
+      await sleep(LEASE_ACQUIRE_RETRY_TIMEOUT)
+
+      process.env.LEASE_RENEW_TIMEOUT = DEFAULT_LEASE_RENEW_TIMEOUT.toString()
+      const newLease = leaseManager.instantiateLease()
+      const newTitle = 'New Lease'
+      // @ts-ignore to access private property
+      await newLease.acquireTask({ newTitle }, newTitle, 8)
+
+      await initialLease.release()
+      await newLease.release()
+    }).timeout(3 * MINUTES)
   })
 })