feat(network node livecheck): implement a functionality to determine network node activeness with prometheus metrics (#567)

Signed-off-by: instamenta <[email protected]>

Author: instamenta

src/commands/node.mjs

@@ -55,8 +55,10 @@ import {
   DEFAULT_NETWORK_NODE_NAME,
   FREEZE_ADMIN_ACCOUNT,
   HEDERA_NODE_DEFAULT_STAKE_AMOUNT,
-  TREASURY_ACCOUNT_ID
+  TREASURY_ACCOUNT_ID,
+  LOCAL_HOST
 } from '../core/constants.mjs'
+import { NodeStatusCodes, NodeStatusEnums } from '../core/enumerations.mjs'
 
 /**
  * Defines the core functionalities of 'node' command
@@ -360,72 +362,116 @@ export class NodeCommand extends BaseCommand {
   }
 
   /**
+   * @param {string} namespace
    * @param {string} nodeId
-   * @param {number} [maxAttempt]
-   * @param {string} [status]
-   * @param {string} [logfile]
-   * @returns {Promise<boolean>}
+   * @param {TaskWrapper} task
+   * @param {string} title
+   * @param {number} index
+   * @param {number} [status]
+   * @param {number} [maxAttempts]
+   * @param {number} [delay]
+   * @param {number} [timeout]
+   * @returns {Promise<string>}
    */
-  async checkNetworkNodeState (nodeId, maxAttempt = 100, status = 'ACTIVE', logfile = 'output/hgcaa.log') {
+  async checkNetworkNodeActiveness (namespace, nodeId, task, title, index,
+    status = NodeStatusCodes.ACTIVE, maxAttempts = 120, delay = 1_000, timeout = 1_000) {
     nodeId = nodeId.trim()
     const podName = Templates.renderNetworkPodName(nodeId)
-    const logfilePath = `${constants.HEDERA_HAPI_PATH}/${logfile}`
+    const podPort = 9_999
+    const localPort = 19_000 + index
+    task.title = `${title} - status ${chalk.yellow('STARTING')}, attempt ${chalk.blueBright(`0/${maxAttempts}`)}`
+
+    const srv = await this.k8.portForward(podName, localPort, podPort)
+
     let attempt = 0
-    let isActive = false
+    let success = false
+    while (attempt < maxAttempts) {
+      const controller = new AbortController()
+
+      const timeoutId = setTimeout(() => {
+        task.title = `${title} - status ${chalk.yellow('TIMEOUT')}, attempt ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`
+        controller.abort()
+      }, timeout)
 
-    this.logger.debug(`Checking if node ${nodeId} is ${status}...`)
-    // check log file is accessible
-    let logFileAccessible = false
-    while (attempt++ < maxAttempt) {
       try {
-        if (await this.k8.hasFile(podName, constants.ROOT_CONTAINER, logfilePath)) {
-          logFileAccessible = true
-          break
+        const url = `http://${LOCAL_HOST}:${localPort}/metrics`
+        const response = await fetch(url, { signal: controller.signal })
+
+        if (!response.ok) {
+          task.title = `${title} - status ${chalk.yellow('UNKNOWN')}, attempt ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`
+          clearTimeout(timeoutId)
+          throw new Error() // Guard
         }
-      } catch (e) {
-      } // ignore errors
 
-      await sleep(1000)
-    }
+        const text = await response.text()
+        const statusLine = text
+          .split('\n')
+          .find(line => line.startsWith('platform_PlatformStatus'))
 
-    if (!logFileAccessible) {
-      throw new FullstackTestingError(`Logs are not accessible: ${logfilePath}`)
-    }
+        if (!statusLine) {
+          task.title = `${title} - status ${chalk.yellow('STARTING')}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`
+          clearTimeout(timeoutId)
+          throw new Error() // Guard
+        }
 
-    attempt = 0
-    while (attempt < maxAttempt) {
-      try {
-        const output = await this.k8.execContainer(podName, constants.ROOT_CONTAINER, ['tail', '-100', logfilePath])
-        if (output && output.indexOf('Terminating Netty') < 0 && // make sure we are not at the beginning of a restart
-          (output.indexOf(`Now current platform status = ${status}`) > 0 ||
-            output.indexOf(`Platform Status Change ${status}`) > 0 ||
-            output.indexOf(`is ${status}`) > 0 ||
-            output.indexOf(`"newStatus":"${status}"`) > 0)) {
-          this.logger.debug(`Node ${nodeId} is ${status} [ attempt: ${attempt}/${maxAttempt}]`)
-          isActive = true
+        const statusNumber = parseInt(statusLine.split(' ').pop())
+
+        if (statusNumber === status) {
+          task.title = `${title} - status ${chalk.green(NodeStatusEnums[status])}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`
+          success = true
+          clearTimeout(timeoutId)
           break
+        } else if (statusNumber === NodeStatusCodes.CATASTROPHIC_FAILURE) {
+          task.title = `${title} - status ${chalk.red('CATASTROPHIC_FAILURE')}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`
+          break
+        } else if (statusNumber) {
+          task.title = `${title} - status ${chalk.yellow(NodeStatusEnums[statusNumber])}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`
         }
-        this.logger.debug(`Node ${nodeId} is not ${status} yet. Trying again... [ attempt: ${attempt}/${maxAttempt} ]`)
-      } catch (e) {
-        this.logger.warn(`error in checking if node ${nodeId} is ${status}: ${e.message}. Trying again... [ attempt: ${attempt}/${maxAttempt} ]`)
-
-        // ls the HAPI path for debugging
-        await this.k8.execContainer(podName, constants.ROOT_CONTAINER, `ls -la ${constants.HEDERA_HAPI_PATH}`)
+        clearTimeout(timeoutId)
+      } catch {} // Catch all guard and fetch errors
 
-        // ls the output directory for debugging
-        await this.k8.execContainer(podName, constants.ROOT_CONTAINER, `ls -la ${constants.HEDERA_HAPI_PATH}/output`)
-      }
-      attempt += 1
-      await sleep(1000)
+      attempt++
+      clearTimeout(timeoutId)
+      await sleep(delay)
     }
 
-    this.logger.info(`!> -- Node ${nodeId} is ${status} -- <!`)
+    await this.k8.stopPortForward(srv)
 
-    if (!isActive) {
-      throw new FullstackTestingError(`node '${nodeId}' is not ${status} [ attempt = ${attempt}/${maxAttempt} ]`)
+    if (!success) {
+      throw new FullstackTestingError(`node '${nodeId}' is not ${NodeStatusEnums[status]}` +
+        `[ attempt = ${chalk.blueBright(`${attempt}/${maxAttempts}`)} ]`)
     }
 
-    return true
+    return podName
+  }
+
+  /**
+   * @param {Object} ctx
+   * @param {TaskWrapper} task
+   * @param {string[]} nodeIds
+   * @param {number} [status]
+   * @returns {Listr<any, any, any>}
+   */
+  checkNodeActivenessTask (ctx, task, nodeIds, status = NodeStatusCodes.ACTIVE) {
+    const { config: { namespace } } = ctx
+
+    const subTasks = nodeIds.map((nodeId, i) => {
+      const reminder = ('debugNodeId' in ctx.config && ctx.config.debugNodeId === nodeId) ? 'Please attach JVM debugger now.' : ''
+      const title = `Check network pod: ${chalk.yellow(nodeId)} ${chalk.red(reminder)}`
+
+      const subTask = async (ctx, task) => {
+        ctx.config.podNames[nodeId] = await this.checkNetworkNodeActiveness(namespace, nodeId, task, title, i, status)
+      }
+
+      return { title, task: subTask }
+    })
+
+    return task.newListr(subTasks, {
+      concurrent: true,
+      rendererOptions: {
+        collapseSubtasks: false
+      }
+    })
   }
 
   /**
@@ -490,67 +536,6 @@ export class NodeCommand extends BaseCommand {
     })
   }
 
-  /**
-   *
-   * @param {string} debugNodeId
-   * @param {TaskWrapper} task
-   * @param {string[]} nodeIds
-   * @return {Listr<any, any, any>}
-   */
-  checkNodeActiveTask (debugNodeId, task, nodeIds) {
-    const subTasks = []
-    for (const nodeId of nodeIds) {
-      let reminder = ''
-      if (debugNodeId === nodeId) {
-        reminder = ' Please attach JVM debugger now.'
-      }
-      if (this.configManager.getFlag(flags.app) !== '' && this.configManager.getFlag(flags.app) !== constants.HEDERA_APP_NAME) {
-        subTasks.push({
-          title: `Check node: ${chalk.yellow(nodeId)} ${chalk.red(reminder)}`,
-          task: async () => await this.checkNetworkNodeState(nodeId, 100, 'ACTIVE', 'output/swirlds.log')
-        })
-      } else {
-        subTasks.push({
-          title: `Check node: ${chalk.yellow(nodeId)} ${chalk.red(reminder)}`,
-          task: async () => await this.checkNetworkNodeState(nodeId)
-        })
-      }
-    }
-
-    // set up the sub-tasks
-    return task.newListr(subTasks, {
-      concurrent: true,
-      rendererOptions: {
-        collapseSubtasks: false
-      }
-    })
-  }
-
-  /**
-   * Return task for checking for if node is in freeze state
-   * @param {any} ctx
-   * @param {TaskWrapper} task
-   * @param {string[]} nodeIds
-   * @returns {*}
-   */
-  checkNodeFreezeTask (ctx, task, nodeIds) {
-    const subTasks = []
-    for (const nodeId of nodeIds) {
-      subTasks.push({
-        title: `Check node: ${chalk.yellow(nodeId)}`,
-        task: async () => await this.checkNetworkNodeState(nodeId, 100, 'FREEZE_COMPLETE')
-      })
-    }
-
-    // set up the sub-tasks
-    return task.newListr(subTasks, {
-      concurrent: false,
-      rendererOptions: {
-        collapseSubtasks: false
-      }
-    })
-  }
-
   /**
    * Return task for setup network nodes
    * @param {any} ctx
@@ -1199,7 +1184,7 @@ export class NodeCommand extends BaseCommand {
       {
         title: 'Check nodes are ACTIVE',
         task: (ctx, task) => {
-          return this.checkNodeActiveTask(ctx.config.debugNodeId, task, ctx.config.nodeIds)
+          return this.checkNodeActivenessTask(ctx, task, ctx.config.nodeIds)
         }
       },
       {
@@ -1533,7 +1518,7 @@ export class NodeCommand extends BaseCommand {
       {
         title: 'Check nodes are ACTIVE',
         task: (ctx, task) => {
-          return this.checkNodeActiveTask(ctx.config.debugNodeId, task, ctx.config.nodeIds)
+          return this.checkNodeActivenessTask(ctx, task, ctx.config.nodeIds)
         }
       },
       {
@@ -2033,8 +2018,7 @@ export class NodeCommand extends BaseCommand {
       {
         title: 'Check network nodes are frozen',
         task: (ctx, task) => {
-          const config = /** @type {NodeAddConfigClass} **/ ctx.config
-          return this.checkNodeFreezeTask(ctx, task, config.existingNodeIds)
+          return this.checkNodeActivenessTask(ctx, task, ctx.config.existingNodeIds, NodeStatusCodes.FREEZE_COMPLETE)
         }
       },
       {
@@ -2128,7 +2112,7 @@ export class NodeCommand extends BaseCommand {
       {
         title: 'Check all nodes are ACTIVE',
         task: async (ctx, task) => {
-          return this.checkNodeActiveTask(ctx.config.debugNodeId, task, ctx.config.allNodeIds)
+          return this.checkNodeActivenessTask(ctx, task, ctx.config.allNodeIds)
         }
       },
       {
@@ -2883,8 +2867,7 @@ export class NodeCommand extends BaseCommand {
       {
         title: 'Check network nodes are frozen',
         task: (ctx, task) => {
-          const config = /** @type {NodeUpdateConfigClass} **/ ctx.config
-          return this.checkNodeFreezeTask(ctx, task, config.existingNodeIds)
+          return this.checkNodeActivenessTask(ctx, task, ctx.config.existingNodeIds, NodeStatusCodes.FREEZE_COMPLETE)
         }
       },
       {
@@ -2962,7 +2945,7 @@ export class NodeCommand extends BaseCommand {
       {
         title: 'Check all nodes are ACTIVE',
         task: async (ctx, task) => {
-          return this.checkNodeActiveTask(ctx.config.debugNodeId, task, ctx.config.allNodeIds)
+          return this.checkNodeActivenessTask(ctx, task, ctx.config.allNodeIds)
         }
       },
       {
@@ -3198,8 +3181,7 @@ export class NodeCommand extends BaseCommand {
       {
         title: 'Check network nodes are frozen',
         task: (ctx, task) => {
-          const config = /** @type {NodeDeleteConfigClass} **/ ctx.config
-          return this.checkNodeFreezeTask(ctx, task, config.existingNodeIds)
+          return this.checkNodeActivenessTask(ctx, task, ctx.config.existingNodeIds, NodeStatusCodes.FREEZE_COMPLETE)
         }
       },
       {
@@ -3269,7 +3251,7 @@ export class NodeCommand extends BaseCommand {
       {
         title: 'Check all nodes are ACTIVE',
         task: async (ctx, task) => {
-          return this.checkNodeActiveTask(ctx.config.debugNodeId, task, ctx.config.allNodeIds)
+          return this.checkNodeActivenessTask(ctx, task, ctx.config.allNodeIds)
         }
       },
       {

src/core/enumerations.mjs

@@ -0,0 +1,45 @@
+/**
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the ""License"");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an ""AS IS"" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+'use strict'
+
+export const NodeStatusCodes = {
+  NO_VALUE: 0,
+  STARTING_UP: 1,
+  ACTIVE: 2,
+  BEHIND: 4,
+  FREEZING: 5,
+  FREEZE_COMPLETE: 6,
+  REPLAYING_EVENTS: 7,
+  OBSERVING: 8,
+  CHECKING: 9,
+  RECONNECT_COMPLETE: 10,
+  CATASTROPHIC_FAILURE: 11
+}
+
+export const NodeStatusEnums = {
+  0: 'NO_VALUE',
+  1: 'STARTING_UP',
+  2: 'ACTIVE',
+  4: 'BEHIND',
+  5: 'FREEZING',
+  6: 'FREEZE_COMPLETE',
+  7: 'REPLAYING_EVENTS',
+  8: 'OBSERVING',
+  9: 'CHECKING',
+  10: 'RECONNECT_COMPLETE',
+  11: 'CATASTROPHIC_FAILURE'
+}

test/e2e/e2e_node_util.js

@@ -165,8 +165,8 @@ export function e2eNodeKeyRefreshTest (testName, mode, releaseTag = HEDERA_PLATF
             expect(2)
             try {
               await expect(
-                nodeCmd.checkNetworkNodeState(nodeId,
-                  5)).rejects.toThrowError()
+                nodeCmd.checkNetworkNodeActiveness(namespace, nodeId, { title: '' }, '', 44, undefined, 15)
+              ).rejects.toThrowError()
             } catch (e) {
               expect(e).not.toBeNull()
             } finally {