updates for addressing errors

Signed-off-by: Jeromy Cannon <[email protected]>

Author: jeromy-cannon

src/commands/node.mjs

@@ -107,7 +107,8 @@ export class NodeCommand extends BaseCommand {
       try {
         const output = await this.k8.execContainer(podName, constants.ROOT_CONTAINER, ['tail', '-10', logfilePath])
         if (output && output.indexOf('Terminating Netty') < 0 && // make sure we are not at the beginning of a restart
-          output.indexOf(`Now current platform status = ${status}`) > 0) {
+            (output.indexOf(`Now current platform status = ${status}`) > 0 ||
+            output.indexOf(`is ${status}`))) { // 'is ACTIVE' is for newer versions, first seen in v0.49.0
           this.logger.debug(`Node ${nodeId} is ${status} [ attempt: ${attempt}/${maxAttempt}]`)
           isActive = true
           break
@@ -693,33 +694,51 @@ export class NodeCommand extends BaseCommand {
    * @param delay the delay between attempts
    * @returns {Promise<boolean>} true if the proxy is up
    */
-  async checkNetworkNodeProxyUp (nodeId, localPort, maxAttempts = 20, delay = 5000) {
-    const podArray = await this.k8.getPodsByLabel([`app=haproxy-${nodeId}`, 'fullstack.hedera.com/type=haproxy'])
+  async checkNetworkNodeProxyUp (nodeId, localPort, maxAttempts = 30, delay = 2000) {
+    const podLabels = [`app=haproxy-${nodeId}`, 'fullstack.hedera.com/type=haproxy']
+    let podArray = await this.k8.getPodsByLabel(podLabels)
 
     let attempts = 0
+    let status = null
     if (podArray.length > 0) {
-      const podName = podArray[0].metadata.name
-      this._portForwards.push(await this.k8.portForward(podName, localPort, 5555))
+      let podName = podArray[0].metadata.name
+      let portForwarder = null
+
       try {
-        await this.k8.testConnection('localhost', localPort)
-      } catch (e) {
-        throw new FullstackTestingError(`failed to create port forward for '${nodeId}' proxy on port ${localPort}`, e)
-      }
+        while (attempts < maxAttempts) {
+          if (attempts === 0) {
+            portForwarder = await this.k8.portForward(podName, localPort, 5555)
+            await this.k8.testConnection('localhost', localPort)
+          } else if (attempts % 5 === 0) {
+            this.logger.debug(`Recycling proxy ${podName} [attempt: ${attempts}/${maxAttempts}]`)
+            await this.k8.stopPortForward(portForwarder)
+            await this.k8.recyclePodByLabels(podLabels, 50)
+            podArray = await this.k8.getPodsByLabel(podLabels)
+            podName = podArray[0].metadata.name
+            portForwarder = await this.k8.portForward(podName, localPort, 5555)
+            await this.k8.testConnection('localhost', localPort)
+          }
 
-      while (attempts < maxAttempts) {
-        try {
-          const status = await this.getNodeProxyStatus(`http://localhost:${localPort}/v2/services/haproxy/stats/native?type=backend`)
+          status = await this.getNodeProxyStatus(`http://localhost:${localPort}/v2/services/haproxy/stats/native?type=backend`)
           if (status === 'UP') {
-            this.logger.debug(`Proxy ${podName} is UP. [attempt: ${attempts}/${maxAttempts}]`)
-            return true
+            break
           }
 
-          attempts++
           this.logger.debug(`Proxy ${podName} is not UP. Checking again in ${delay}ms ... [attempt: ${attempts}/${maxAttempts}]`)
+          attempts++
           await sleep(delay)
-        } catch (e) {
-          throw new FullstackTestingError(`failed to create port forward for '${nodeId}' proxy on port ${localPort}`, e)
         }
+      } catch (e) {
+        throw new FullstackTestingError(`failed to check proxy for '${nodeId}' on port ${localPort}: ${e.message}`, e)
+      } finally {
+        if (portForwarder !== null) {
+          this._portForwards.push(portForwarder)
+        }
+      }
+
+      if (status === 'UP') {
+        this.logger.debug(`Proxy ${podName} is UP. [attempt: ${attempts}/${maxAttempts}]`)
+        return true
       }
     }
 

src/core/k8.mjs

@@ -26,6 +26,7 @@ import * as tar from 'tar'
 import { v4 as uuid4 } from 'uuid'
 import { V1ObjectMeta, V1Secret } from '@kubernetes/client-node'
 import { constants } from './index.mjs'
+import { sleep } from './helpers.mjs'
 
 /**
  * A kubernetes API wrapper class providing custom functionalities required by solo
@@ -727,6 +728,27 @@ export class K8 {
     })
   }
 
+  async recyclePodByLabels (podLabels, maxAttempts = 50) {
+    const podArray = await this.getPodsByLabel(podLabels)
+    for (const pod of podArray) {
+      const podName = pod.metadata.name
+      await this.kubeClient.deleteNamespacedPod(podName, this.configManager.getFlag(flags.namespace))
+    }
+
+    let attempts = 0
+    while (attempts++ < maxAttempts) {
+      const status = await this.waitForPod(constants.POD_STATUS_RUNNING, podLabels)
+      if (status) {
+        const newPods = await this.getPodsByLabel(podLabels)
+        if (newPods.length === podArray.length) return newPods
+      }
+
+      await sleep(2000)
+    }
+
+    throw new FullstackTestingError(`pods are not running after deletion with labels [${podLabels.join(',')}]`)
+  }
+
   /**
    * Wait for pod
    * @param status phase of the pod

test/e2e/commands/node.test.mjs

@@ -48,14 +48,13 @@ describe.each([
   argv[flags.generateGossipKeys.name] = true
   argv[flags.generateTlsKeys.name] = true
   argv[flags.clusterName.name] = TEST_CLUSTER
-  argv[flags.chartDirectory.name] = 'charts' // TODO remove before merging PR, this allows pulling local charts
   const bootstrapResp = bootstrapNetwork(testName, argv)
   const accountManager = bootstrapResp.opts.accountManager
   const k8 = bootstrapResp.opts.k8
   const nodeCmd = bootstrapResp.cmd.nodeCmd
 
   afterAll(async () => {
-    await k8.deleteNamespace(namespace) // TODO uncomment this line before merging PR
+    await k8.deleteNamespace(namespace)
     await accountManager.close()
   })