From a907f4508e01c1383be4b5a4e6df577a28725335 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Thu, 21 Mar 2024 14:29:58 +0000 Subject: [PATCH 1/6] Squashed commit of the following: commit 0ede4eeb27bf26ef4aff23fccfb75b3c1306575f Author: Jeromy Cannon Date: Wed Mar 20 21:54:30 2024 +0000 timeout fetch plus log messages Signed-off-by: Jeromy Cannon commit abdabc26e127f359868cbb3ca31d01f858afd2d3 Author: Jeromy Cannon Date: Wed Mar 20 21:32:36 2024 +0000 fix check failure Signed-off-by: Jeromy Cannon commit 6369895c84585b79e1f95327a469ab5110e45774 Author: Jeromy Cannon Date: Wed Mar 20 20:55:24 2024 +0000 working version Signed-off-by: Jeromy Cannon Signed-off-by: Jeromy Cannon --- src/commands/node.mjs | 101 ++++++++++++++++++------- src/core/account_manager.mjs | 32 +------- src/core/constants.mjs | 3 + src/core/k8.mjs | 36 +++++++-- test/e2e/commands/01_node.test.mjs | 13 ++++ test/e2e/core/account_manager.test.mjs | 4 +- 6 files changed, 124 insertions(+), 65 deletions(-) diff --git a/src/commands/node.mjs b/src/commands/node.mjs index 0f136e407..78c7d8afd 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -44,6 +44,21 @@ export class NodeCommand extends BaseCommand { this.keyManager = opts.keyManager this.accountManager = opts.accountManager this.keytoolDepManager = opts.keytoolDepManager + this._portForwards = [] + } + + /** + * stops and closes the port forwards + * @returns {Promise} + */ + async close () { + if (this._portForwards) { + for (const srv of this._portForwards) { + await this.k8.stopPortForward(srv) + } + } + + this._portForwards = [] } async checkNetworkNodePod (namespace, nodeId) { @@ -637,10 +652,11 @@ export class NodeCommand extends BaseCommand { title: 'Check node proxies are ACTIVE', task: async (ctx, parentTask) => { const subTasks = [] + let localPort = constants.LOCAL_NODE_PROXY_START_PORT for (const nodeId of ctx.config.nodeIds) { subTasks.push({ title: `Check proxy for node: ${chalk.yellow(nodeId)}`, - task: async () => await self.checkNetworkNodeProxyUp(ctx.config.namespace, nodeId) + task: async () => await self.checkNetworkNodeProxyUp(ctx.config.namespace, nodeId, localPort++) }) } @@ -664,45 +680,48 @@ export class NodeCommand extends BaseCommand { throw new FullstackTestingError(`Error starting node: ${e.message}`, e) } finally { await self.accountManager.close() + await self.close() } return true } - async checkNetworkNodeProxyUp (namespace, nodeId, maxAttempts = 10, delay = 5000) { - const podArray = await this.k8.getPodsByLabel([`app=haproxy-${nodeId}`, 'fullstack.hedera.com/type=haproxy']) + /** + * Check if the network node proxy is up, requires close() to be called after + * @param namespace the namespace + * @param nodeId the node id + * @param localPort the local port to forward to + * @param maxAttempts the maximum number of attempts + * @param delay the delay between attempts + * @returns {Promise} true if the proxy is up + */ + async checkNetworkNodeProxyUp (namespace, nodeId, localPort, maxAttempts = 10, delay = 5000) { + const podArray = await this.k8.getPodsByLabel(namespace, [`app=haproxy-${nodeId}`, 'fullstack.hedera.com/type=haproxy']) let attempts = 0 if (podArray.length > 0) { const podName = podArray[0].metadata.name + this._portForwards.push(await this.k8.portForward(podName, localPort, 5555, namespace)) + try { + await this.k8.testConnection('localhost', localPort) + } catch (e) { + throw new FullstackTestingError(`failed to create port forward for '${nodeId}' proxy on port ${localPort}`, e) + } while (attempts < maxAttempts) { - const logResponse = await this.k8.kubeClient.readNamespacedPodLog( - podName, - namespace, - 'haproxy', - undefined, - undefined, - 1024, - undefined, - undefined, - undefined, - 4 - ) - - if (logResponse.response.statusCode !== 200) { - throw new FullstackTestingError(`Expected pod ${podName} log query to execute successful, but instead got a status of ${logResponse.response.statusCode}`) - } + try { + const status = await this.getNodeProxyStatus(`http://localhost:${localPort}/v2/services/haproxy/stats/native?type=backend`) + if (status === 'UP') { + this.logger.debug(`Proxy ${podName} is UP. [attempt: ${attempts}/${maxAttempts}]`) + return true + } - this.logger.debug(`Received HAProxy log from ${podName}`, { output: logResponse.body }) - if (logResponse.body.includes('Server be_servers/server1 is UP')) { - this.logger.debug(`Proxy ${podName} is UP [attempt: ${attempts}/${maxAttempts}]`) - return true + attempts++ + this.logger.debug(`Proxy ${podName} is not UP. Checking again in ${delay}ms ... [attempt: ${attempts}/${maxAttempts}]`) + await sleep(delay) + } catch (e) { + throw new FullstackTestingError(`failed to create port forward for '${nodeId}' proxy on port ${localPort}`, e) } - - attempts++ - this.logger.debug(`Proxy ${podName} is not UP. Checking again in ${delay}ms ... [attempt: ${attempts}/${maxAttempts}]`) - await sleep(delay) } } @@ -967,4 +986,32 @@ export class NodeCommand extends BaseCommand { } } } + + async getNodeProxyStatus (url) { + try { + this.logger.debug(`Fetching proxy status from: ${url}`) + const res = await fetch(url, { + method: 'GET', + signal: AbortSignal.timeout(5000), + headers: { + Authorization: `Basic ${Buffer.from( + `${constants.NODE_PROXY_USER_ID}:${constants.NODE_PROXY_PASSWORD}`).toString( + 'base64')}` + } + }) + const response = await res.json() + + if (res.status === 200) { + const status = response[0]?.stats?.filter( + (stat) => stat.name === 'http_backend')[0]?.stats?.status + this.logger.debug(`Proxy status: ${status}`) + return status + } else { + this.logger.debug(`Proxy request status code: ${res.status}`) + return null + } + } catch (e) { + this.logger.error(`Error in fetching proxy status: ${e.message}`, e) + } + } } diff --git a/src/core/account_manager.mjs b/src/core/account_manager.mjs index 7a01755c7..f925ce2e2 100644 --- a/src/core/account_manager.mjs +++ b/src/core/account_manager.mjs @@ -31,7 +31,6 @@ import { TransferTransaction } from '@hashgraph/sdk' import { FullstackTestingError, MissingArgumentError } from './errors.mjs' -import net from 'net' import { Templates } from './templates.mjs' const REASON_FAILED_TO_GET_KEYS = 'failed to get keys for accountId' @@ -58,8 +57,6 @@ const REJECTED = 'rejected' * */ export class AccountManager { - static _openPortForwardConnections = 0 - /** * creates a new AccountManager instance * @param logger the logger to use @@ -202,12 +199,10 @@ export class AccountManager { if (usePortForward) { this._portForwards.push(await this.k8.portForward(serviceObject.podName, localPort, port)) - AccountManager._openPortForwardConnections++ } nodes[`${host}:${targetPort}`] = AccountId.fromString(serviceObject.accountId) - await this.testConnection(serviceObject.podName, host, targetPort) - + await this.k8.testConnection(host, targetPort) localPort++ } @@ -464,31 +459,6 @@ export class AccountManager { return receipt.status === Status.Success } - /** - * to test the connection to the node within the network - * @param podName the podName is only used for logging messages and errors - * @param host the host of the target connection - * @param port the port of the target connection - * @returns {Promise} - */ - async testConnection (podName, host, port) { - const self = this - - return new Promise((resolve, reject) => { - const s = new net.Socket() - s.on('error', (e) => { - s.destroy() - reject(new FullstackTestingError(`failed to connect to '${host}:${port}': ${e.message}`, e)) - }) - - s.connect(port, host, () => { - self.logger.debug(`Connection test successful: ${host}:${port}`) - s.destroy() - resolve(true) - }) - }) - } - /** * creates a new Hedera account * @param namespace the namespace to store the Kubernetes key secret into diff --git a/src/core/constants.mjs b/src/core/constants.mjs index 90b6724af..df5ffd55e 100644 --- a/src/core/constants.mjs +++ b/src/core/constants.mjs @@ -83,7 +83,10 @@ export const GENESIS_KEY = process.env.GENESIS_KEY || '302e020100300506032b65700 export const SYSTEM_ACCOUNTS = [[3, 100], [200, 349], [400, 750], [900, 1000]] // do account 0.0.2 last and outside the loop export const TREASURY_ACCOUNT = 2 export const LOCAL_NODE_START_PORT = process.env.LOCAL_NODE_START_PORT || 30212 +export const LOCAL_NODE_PROXY_START_PORT = process.env.LOCAL_NODE_PROXY_START_PORT || 30313 export const ACCOUNT_CREATE_BATCH_SIZE = process.env.ACCOUNT_CREATE_BATCH_SIZE || 50 +export const NODE_PROXY_USER_ID = process.env.NODE_PROXY_USER_ID || 'admin' +export const NODE_PROXY_PASSWORD = process.env.NODE_PROXY_PASSWORD || 'adminpwd' export const POD_STATUS_RUNNING = 'Running' export const POD_STATUS_READY = 'Ready' diff --git a/src/core/k8.mjs b/src/core/k8.mjs index 8ace719d9..6550b2f8c 100644 --- a/src/core/k8.mjs +++ b/src/core/k8.mjs @@ -197,14 +197,14 @@ export class K8 { /** * Get pods by labels + * @param namespace the namespace of the pods * @param labels list of labels * @return {Promise>} */ - async getPodsByLabel (labels = []) { - const ns = this._getNamespace() + async getPodsByLabel (namespace, labels = []) { const labelSelector = labels.join(',') const result = await this.kubeClient.listNamespacedPod( - ns, + namespace, undefined, undefined, undefined, @@ -661,9 +661,10 @@ export class K8 { * @param podName pod name * @param localPort local port * @param podPort port of the pod + * @param namespace namespace of the pod (optional) */ - async portForward (podName, localPort, podPort) { - const ns = this._getNamespace() + async portForward (podName, localPort, podPort, namespace = null) { + const ns = namespace || this._getNamespace() const forwarder = new k8s.PortForward(this.kubeConfig, false) const server = await net.createServer((socket) => { forwarder.portForward(ns, podName, [podPort], socket, null, socket, 3) @@ -671,9 +672,34 @@ export class K8 { // add info for logging server.info = `${podName}:${podPort} -> ${constants.LOCAL_HOST}:${localPort}` + this.logger.debug(`Starting port-forwarder [${server.info}]`) return server.listen(localPort, constants.LOCAL_HOST) } + /** + * to test the connection to a pod within the network + * @param host the host of the target connection + * @param port the port of the target connection + * @returns {Promise} + */ + async testConnection (host, port) { + const self = this + + return new Promise((resolve, reject) => { + const s = new net.Socket() + s.on('error', (e) => { + s.destroy() + reject(new FullstackTestingError(`failed to connect to '${host}:${port}': ${e.message}`, e)) + }) + + s.connect(port, host, () => { + self.logger.debug(`Connection test successful: ${host}:${port}`) + s.destroy() + resolve(true) + }) + }) + } + /** * Stop the port forwarder server * diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index f0c43bec7..437907a7e 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -97,5 +97,18 @@ describe.each([ expect(e).toBeNull() } }, 20000) + + it('Node Proxy should be UP', async () => { + expect.assertions(1) + + try { + await expect(nodeCmd.checkNetworkNodeProxyUp(namespace, 'node0', 30313)).resolves.toBeTruthy() + } catch (e) { + nodeCmd.logger.showUserError(e) + expect(e).toBeNull() + } finally { + await nodeCmd.close() + } + }, 20000) }) }) diff --git a/test/e2e/core/account_manager.test.mjs b/test/e2e/core/account_manager.test.mjs index b4f9a4598..30d6e017e 100644 --- a/test/e2e/core/account_manager.test.mjs +++ b/test/e2e/core/account_manager.test.mjs @@ -40,13 +40,13 @@ describe('AccountManager', () => { // ports should be opened accountManager._portForwards.push(await k8.portForward(podName, localPort, podPort)) - const status = await accountManager.testConnection(podName, localHost, localPort) + const status = await k8.testConnection(localHost, localPort) expect(status).toBeTruthy() // ports should be closed await accountManager.close() try { - await accountManager.testConnection(podName, localHost, localPort) + await k8.testConnection(localHost, localPort) } catch (e) { expect(e.message.includes(`failed to connect to '${localHost}:${localPort}'`)).toBeTruthy() } From f023ea690b7fb7c01a920e150b0192c0772c1fb4 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Thu, 21 Mar 2024 14:40:10 +0000 Subject: [PATCH 2/6] updated fst chart version Signed-off-by: Jeromy Cannon --- version.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.mjs b/version.mjs index 829e43b11..e7fa297aa 100644 --- a/version.mjs +++ b/version.mjs @@ -21,4 +21,4 @@ export const JAVA_VERSION = '21.0.1+12' export const HELM_VERSION = 'v3.14.2' -export const FST_CHART_VERSION = 'v0.23.0' +export const FST_CHART_VERSION = 'v0.24.0' From 13811de4123f1ede34988fb672a61b98e402e6f0 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Thu, 21 Mar 2024 15:12:34 +0000 Subject: [PATCH 3/6] fix test case Signed-off-by: Jeromy Cannon --- test/e2e/commands/02_account.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/commands/02_account.test.mjs b/test/e2e/commands/02_account.test.mjs index ba5f72be9..405f836e3 100644 --- a/test/e2e/commands/02_account.test.mjs +++ b/test/e2e/commands/02_account.test.mjs @@ -63,7 +63,7 @@ describe('AccountCommand', () => { describe('node proxies should be UP', () => { for (const nodeId of argv[flags.nodeIDs.name].split(',')) { it(`proxy should be UP: ${nodeId} `, async () => { - await nodeCmd.checkNetworkNodeProxyUp(namespace, nodeId) + await nodeCmd.checkNetworkNodeProxyUp(namespace, nodeId, 30399) }, 30000) } }) From cf6b6f6b27c809a672c86e56a207441d12d8f241 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Thu, 21 Mar 2024 15:22:15 +0000 Subject: [PATCH 4/6] fix test case Signed-off-by: Jeromy Cannon --- test/e2e/commands/02_account.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/commands/02_account.test.mjs b/test/e2e/commands/02_account.test.mjs index 405f836e3..701a52427 100644 --- a/test/e2e/commands/02_account.test.mjs +++ b/test/e2e/commands/02_account.test.mjs @@ -61,9 +61,10 @@ describe('AccountCommand', () => { }) describe('node proxies should be UP', () => { + let localPort = 30399 for (const nodeId of argv[flags.nodeIDs.name].split(',')) { it(`proxy should be UP: ${nodeId} `, async () => { - await nodeCmd.checkNetworkNodeProxyUp(namespace, nodeId, 30399) + await nodeCmd.checkNetworkNodeProxyUp(namespace, nodeId, localPort++) }, 30000) } }) From 422b7667541b76a522ea282bc0a278f3c0c7c2fb Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Fri, 22 Mar 2024 07:39:44 +0000 Subject: [PATCH 5/6] removed namespace based on pr comment Signed-off-by: Jeromy Cannon --- src/commands/node.mjs | 4 ++-- src/core/k8.mjs | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/commands/node.mjs b/src/commands/node.mjs index 78c7d8afd..818b3962a 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -696,12 +696,12 @@ export class NodeCommand extends BaseCommand { * @returns {Promise} true if the proxy is up */ async checkNetworkNodeProxyUp (namespace, nodeId, localPort, maxAttempts = 10, delay = 5000) { - const podArray = await this.k8.getPodsByLabel(namespace, [`app=haproxy-${nodeId}`, 'fullstack.hedera.com/type=haproxy']) + const podArray = await this.k8.getPodsByLabel([`app=haproxy-${nodeId}`, 'fullstack.hedera.com/type=haproxy']) let attempts = 0 if (podArray.length > 0) { const podName = podArray[0].metadata.name - this._portForwards.push(await this.k8.portForward(podName, localPort, 5555, namespace)) + this._portForwards.push(await this.k8.portForward(podName, localPort, 5555)) try { await this.k8.testConnection('localhost', localPort) } catch (e) { diff --git a/src/core/k8.mjs b/src/core/k8.mjs index 6550b2f8c..a3cb87110 100644 --- a/src/core/k8.mjs +++ b/src/core/k8.mjs @@ -197,14 +197,14 @@ export class K8 { /** * Get pods by labels - * @param namespace the namespace of the pods * @param labels list of labels * @return {Promise>} */ - async getPodsByLabel (namespace, labels = []) { + async getPodsByLabel (labels = []) { + const ns = this._getNamespace() const labelSelector = labels.join(',') const result = await this.kubeClient.listNamespacedPod( - namespace, + ns, undefined, undefined, undefined, @@ -661,10 +661,9 @@ export class K8 { * @param podName pod name * @param localPort local port * @param podPort port of the pod - * @param namespace namespace of the pod (optional) */ - async portForward (podName, localPort, podPort, namespace = null) { - const ns = namespace || this._getNamespace() + async portForward (podName, localPort, podPort) { + const ns = this._getNamespace() const forwarder = new k8s.PortForward(this.kubeConfig, false) const server = await net.createServer((socket) => { forwarder.portForward(ns, podName, [podPort], socket, null, socket, 3) From f85e60b1eb4ec307de963dc13879d7c1cfa4f243 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Fri, 22 Mar 2024 07:41:34 +0000 Subject: [PATCH 6/6] removed namespace based on pr comment Signed-off-by: Jeromy Cannon --- src/commands/node.mjs | 5 ++--- test/e2e/commands/01_node.test.mjs | 2 +- test/e2e/commands/02_account.test.mjs | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/commands/node.mjs b/src/commands/node.mjs index 818b3962a..8f96f9aac 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -656,7 +656,7 @@ export class NodeCommand extends BaseCommand { for (const nodeId of ctx.config.nodeIds) { subTasks.push({ title: `Check proxy for node: ${chalk.yellow(nodeId)}`, - task: async () => await self.checkNetworkNodeProxyUp(ctx.config.namespace, nodeId, localPort++) + task: async () => await self.checkNetworkNodeProxyUp(nodeId, localPort++) }) } @@ -688,14 +688,13 @@ export class NodeCommand extends BaseCommand { /** * Check if the network node proxy is up, requires close() to be called after - * @param namespace the namespace * @param nodeId the node id * @param localPort the local port to forward to * @param maxAttempts the maximum number of attempts * @param delay the delay between attempts * @returns {Promise} true if the proxy is up */ - async checkNetworkNodeProxyUp (namespace, nodeId, localPort, maxAttempts = 10, delay = 5000) { + async checkNetworkNodeProxyUp (nodeId, localPort, maxAttempts = 10, delay = 5000) { const podArray = await this.k8.getPodsByLabel([`app=haproxy-${nodeId}`, 'fullstack.hedera.com/type=haproxy']) let attempts = 0 diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index 437907a7e..36b6b2eb6 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -102,7 +102,7 @@ describe.each([ expect.assertions(1) try { - await expect(nodeCmd.checkNetworkNodeProxyUp(namespace, 'node0', 30313)).resolves.toBeTruthy() + await expect(nodeCmd.checkNetworkNodeProxyUp('node0', 30313)).resolves.toBeTruthy() } catch (e) { nodeCmd.logger.showUserError(e) expect(e).toBeNull() diff --git a/test/e2e/commands/02_account.test.mjs b/test/e2e/commands/02_account.test.mjs index 701a52427..8a5903773 100644 --- a/test/e2e/commands/02_account.test.mjs +++ b/test/e2e/commands/02_account.test.mjs @@ -64,7 +64,7 @@ describe('AccountCommand', () => { let localPort = 30399 for (const nodeId of argv[flags.nodeIDs.name].split(',')) { it(`proxy should be UP: ${nodeId} `, async () => { - await nodeCmd.checkNetworkNodeProxyUp(namespace, nodeId, localPort++) + await nodeCmd.checkNetworkNodeProxyUp(nodeId, localPort++) }, 30000) } })