Skip to content

Commit a7b3279

Browse files
JeffreyDallasJeffreyDallas
authored
feat: expose pod check attempts and delays as environment controlled variables (#812)
Signed-off-by: Jeffrey Tang <[email protected]>
1 parent 5ac04bc commit a7b3279

File tree

8 files changed

+50
-29
lines changed

8 files changed

+50
-29
lines changed

‎src/commands/mirror_node.ts

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -238,43 +238,43 @@ export class MirrorNodeCommand extends BaseCommand {
238238
task: async () => await self.k8.waitForPodReady([
239239
'app.kubernetes.io/component=postgresql',
240240
'app.kubernetes.io/name=postgres'
241-
], 1, 300, 2000)
241+
], 1, constants.PODS_READY_MAX_ATTEMPTS, constants.PODS_READY_DELAY)
242242
},
243243
{
244244
title: 'Check REST API',
245245
task: async () => await self.k8.waitForPodReady([
246246
'app.kubernetes.io/component=rest',
247247
'app.kubernetes.io/name=rest'
248-
], 1, 300, 2000)
248+
], 1, constants.PODS_READY_MAX_ATTEMPTS, constants.PODS_READY_DELAY)
249249
},
250250
{
251251
title: 'Check GRPC',
252252
task: async () => await self.k8.waitForPodReady([
253253
'app.kubernetes.io/component=grpc',
254254
'app.kubernetes.io/name=grpc'
255-
], 1, 300, 2000)
255+
], 1, constants.PODS_READY_MAX_ATTEMPTS, constants.PODS_READY_DELAY)
256256
},
257257
{
258258
title: 'Check Monitor',
259259
task: async () => await self.k8.waitForPodReady([
260260
'app.kubernetes.io/component=monitor',
261261
'app.kubernetes.io/name=monitor'
262-
], 1, 300, 2000)
262+
], 1, constants.PODS_READY_MAX_ATTEMPTS, constants.PODS_READY_DELAY)
263263
},
264264
{
265265
title: 'Check Importer',
266266
task: async () => await self.k8.waitForPodReady([
267267
'app.kubernetes.io/component=importer',
268268
'app.kubernetes.io/name=importer'
269-
], 1, 300, 2000)
269+
], 1, constants.PODS_READY_MAX_ATTEMPTS, constants.PODS_READY_DELAY)
270270
},
271271
{
272272
title: 'Check Hedera Explorer',
273273
skip: (ctx) => !ctx.config.deployHederaExplorer,
274274
task: async () => await self.k8.waitForPodReady([
275275
'app.kubernetes.io/component=hedera-explorer',
276276
'app.kubernetes.io/name=hedera-explorer'
277-
], 1, 300, 2000)
277+
], 1, constants.PODS_READY_MAX_ATTEMPTS, constants.PODS_READY_DELAY)
278278
}
279279
], {
280280
concurrent: true,

‎src/commands/network.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -336,7 +336,7 @@ export class NetworkCommand extends BaseCommand {
336336
await self.k8.waitForPods([constants.POD_PHASE_RUNNING], [
337337
'solo.hedera.com/type=network-node',
338338
`solo.hedera.com/node-name=${nodeAlias}`
339-
], 1, 60 * 15, 1000) // timeout 15 minutes
339+
], 1, constants.PODS_RUNNING_MAX_ATTEMPTS, constants.PODS_RUNNING_DELAY)
340340
})
341341
}
342342

@@ -363,7 +363,7 @@ export class NetworkCommand extends BaseCommand {
363363
task: async () =>
364364
await self.k8.waitForPods([constants.POD_PHASE_RUNNING], [
365365
'solo.hedera.com/type=haproxy'
366-
], 1, 60 * 15, 1000) // timeout 15 minutes
366+
], 1, constants.PODS_RUNNING_MAX_ATTEMPTS, constants.PODS_RUNNING_DELAY)
367367
})
368368
}
369369

@@ -374,7 +374,7 @@ export class NetworkCommand extends BaseCommand {
374374
task: async () =>
375375
await self.k8.waitForPods([constants.POD_PHASE_RUNNING], [
376376
'solo.hedera.com/type=envoy-proxy'
377-
], 1, 60 * 15, 1000) // timeout 15 minutes
377+
], 1, constants.PODS_RUNNING_MAX_ATTEMPTS, constants.PODS_RUNNING_DELAY)
378378
})
379379
}
380380

@@ -399,7 +399,7 @@ export class NetworkCommand extends BaseCommand {
399399
task: async () =>
400400
await self.k8.waitForPodReady([
401401
'v1.min.io/tenant=minio'
402-
], 1, 60 * 5, 1000) // timeout 5 minutes
402+
], 1, constants.PODS_RUNNING_MAX_ATTEMPTS, constants.PODS_RUNNING_DELAY)
403403
})
404404

405405
// set up the sub-tasks

‎src/commands/node/tasks.ts

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@ import {
2929
} from '../../core/index.ts'
3030
import {
3131
DEFAULT_NETWORK_NODE_NAME,
32-
FREEZE_ADMIN_ACCOUNT, HEDERA_NODE_DEFAULT_STAKE_AMOUNT,
33-
LOCAL_HOST, SECONDS,
32+
FREEZE_ADMIN_ACCOUNT,
33+
HEDERA_NODE_DEFAULT_STAKE_AMOUNT,
34+
LOCAL_HOST,
35+
SECONDS,
3436
TREASURY_ACCOUNT_ID
3537
} from '../../core/constants.ts'
3638
import {
@@ -282,7 +284,9 @@ export class NodeCommandTasks {
282284

283285
async _checkNetworkNodeActiveness (namespace: string, nodeAlias: NodeAlias, task: ListrTaskWrapper<any, any, any>,
284286
title: string, index: number, status = NodeStatusCodes.ACTIVE,
285-
maxAttempts = 120, delay = 1_000, timeout = 1_000
287+
maxAttempts = constants.NETWORK_NODE_ACTIVE_MAX_ATTEMPTS,
288+
delay = constants.NETWORK_NODE_ACTIVE_DELAY,
289+
timeout = constants.NETWORK_NODE_ACTIVE_TIMEOUT
286290
) {
287291
nodeAlias = nodeAlias.trim() as NodeAlias
288292
const podName = Templates.renderNetworkPodName(nodeAlias)
@@ -364,7 +368,7 @@ export class NodeCommandTasks {
364368
title: `Check proxy for node: ${chalk.yellow(nodeAlias)}`,
365369
task: async () => await this.k8.waitForPodReady(
366370
[`app=haproxy-${nodeAlias}`, 'solo.hedera.com/type=haproxy'],
367-
1, 300, 2000)
371+
1, constants.NETWORK_PROXY_MAX_ATTEMPTS, constants.NETWORK_PROXY_DELAY)
368372
})
369373
}
370374

@@ -626,7 +630,9 @@ export class NodeCommandTasks {
626630
}
627631

628632
/** Check if the network node pod is running */
629-
async checkNetworkNodePod (namespace: string, nodeAlias: NodeAlias, maxAttempts = 60, delay = 2000) {
633+
async checkNetworkNodePod (namespace: string, nodeAlias: NodeAlias,
634+
maxAttempts = constants.PODS_RUNNING_MAX_ATTEMPTS,
635+
delay = constants.PODS_RUNNING_DELAY) {
630636
nodeAlias = nodeAlias.trim() as NodeAlias
631637
const podName = Templates.renderNetworkPodName(nodeAlias)
632638

@@ -1221,7 +1227,7 @@ export class NodeCommandTasks {
12211227
await this.k8.waitForPods([constants.POD_PHASE_RUNNING], [
12221228
'solo.hedera.com/type=network-node',
12231229
`solo.hedera.com/node-name=${nodeAlias}`
1224-
], 1, 60 * 15, 1000) // timeout 15 minutes
1230+
], 1, constants.PODS_RUNNING_MAX_ATTEMPTS, constants.PODS_RUNNING_DELAY) // timeout 15 minutes
12251231
})
12261232
}
12271233

‎src/commands/relay.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ export class RelayCommand extends BaseCommand {
234234
await self.k8.waitForPods([constants.POD_PHASE_RUNNING], [
235235
'app=hedera-json-rpc-relay',
236236
`app.kubernetes.io/instance=${config.releaseName}`
237-
], 1, 900, 1000)
237+
], 1, constants.RELAY_PODS_RUNNING_MAX_ATTEMPTS, constants.RELAY_PODS_RUNNING_DELAY)
238238

239239
// reset nodeAlias
240240
self.configManager.setFlag(flags.nodeAliasesUnparsed, '')
@@ -248,7 +248,7 @@ export class RelayCommand extends BaseCommand {
248248
await self.k8.waitForPodReady([
249249
'app=hedera-json-rpc-relay',
250250
`app.kubernetes.io/instance=${config.releaseName}`
251-
], 1, 100, 2000)
251+
], 1, constants.RELAY_PODS_READY_MAX_ATTEMPTS, constants.RELAY_PODS_READY_DELAY)
252252
} catch (e: Error | any) {
253253
throw new SoloError(`Relay ${config.releaseName} is not ready: ${e.message}`, e)
254254
}

‎src/core/constants.ts

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,21 @@ export const JVM_DEBUG_PORT = 5005
162162
export const SECONDS = 1000
163163
export const MINUTES = 60 * SECONDS
164164

165-
export const LEASE_AQUIRE_RETRY_TIMEOUT = 20 * SECONDS
166-
export const MAX_LEASE_ACQUIRE_ATTEMPTS = 10
167-
export const LEASE_RENEW_TIMEOUT = 10 * SECONDS
165+
export const LEASE_ACQUIRE_RETRY_TIMEOUT = +process.env.LEASE_ACQUIRE_RETRY_TIMEOUT || 20 * SECONDS
166+
export const MAX_LEASE_ACQUIRE_ATTEMPTS = +process.env.MAX_LEASE_ACQUIRE_ATTEMPTS || 10
167+
export const LEASE_RENEW_TIMEOUT = +process.env.LEASE_RENEW_TIMEOUT || 10 * SECONDS
168+
169+
export const PODS_RUNNING_MAX_ATTEMPTS = +process.env.PODS_RUNNING_MAX_ATTEMPTS || 60 * 15
170+
export const PODS_RUNNING_DELAY = +process.env.PODS_RUNNING_DELAY || 1000
171+
export const NETWORK_NODE_ACTIVE_MAX_ATTEMPTS = +process.env.NETWORK_NODE_ACTIVE_MAX_ATTEMPTS || 120
172+
export const NETWORK_NODE_ACTIVE_DELAY = +process.env.NETWORK_NODE_ACTIVE_DELAY || 1_000
173+
export const NETWORK_NODE_ACTIVE_TIMEOUT = +process.env.NETWORK_NODE_ACTIVE_TIMEOUT || 1_000
174+
export const NETWORK_PROXY_MAX_ATTEMPTS = +process.env.NETWORK_PROXY_MAX_ATTEMPTS || 300
175+
export const NETWORK_PROXY_DELAY = +process.env.NETWORK_PROXY_DELAY || 2000
176+
export const PODS_READY_MAX_ATTEMPTS = +process.env.PODS_READY_MAX_ATTEMPTS || 300
177+
export const PODS_READY_DELAY = +process.env.PODS_READY_DELAY || 2_000
178+
export const RELAY_PODS_RUNNING_MAX_ATTEMPTS = +process.env.RELAY_PODS_RUNNING_MAX_ATTEMPTS || 900
179+
export const RELAY_PODS_RUNNING_DELAY = +process.env.RELAY_PODS_RUNNING_DELAY || 1_000
180+
export const RELAY_PODS_READY_MAX_ATTEMPTS = +process.env.RELAY_PODS_READY_MAX_ATTEMPTS || 100
181+
export const RELAY_PODS_READY_DELAY = +process.env.RELAY_PODS_READY_DELAY || 1_000
182+

‎src/core/k8.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -910,8 +910,8 @@ export class K8 {
910910
}
911911
}
912912

913-
async waitForPods (phases = [constants.POD_PHASE_RUNNING], labels: string[] = [], podCount = 1, maxAttempts = 10,
914-
delay = 500, podItemPredicate?: (items: k8s.V1Pod) => any): Promise<k8s.V1Pod[]> {
913+
async waitForPods (phases = [constants.POD_PHASE_RUNNING], labels: string[] = [], podCount = 1, maxAttempts = constants.PODS_RUNNING_MAX_ATTEMPTS,
914+
delay = constants.PODS_RUNNING_DELAY, podItemPredicate?: (items: k8s.V1Pod) => any): Promise<k8s.V1Pod[]> {
915915
const ns = this._getNamespace()
916916
const labelSelector = labels.join(',')
917917

‎src/core/lease_manager.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import { flags } from '../commands/index.ts'
1919
import type { ConfigManager } from './config_manager.ts'
2020
import type { K8 } from './k8.ts'
2121
import type { SoloLogger } from './logging.ts'
22-
import { LEASE_RENEW_TIMEOUT, LEASE_AQUIRE_RETRY_TIMEOUT, MAX_LEASE_ACQUIRE_ATTEMPTS, OS_USERNAME } from './constants.ts'
22+
import { LEASE_RENEW_TIMEOUT, LEASE_ACQUIRE_RETRY_TIMEOUT, MAX_LEASE_ACQUIRE_ATTEMPTS, OS_USERNAME } from './constants.ts'
2323
import type { ListrTaskWrapper } from 'listr2'
2424
import chalk from 'chalk'
2525
import { sleep } from './helpers.ts'
@@ -131,12 +131,12 @@ export class LeaseManager {
131131
throw new SoloError(`Failed to acquire lease, max attempt reached ${attempt}`)
132132
}
133133

134-
this.logger.info(`Lease is already taken retrying in ${LEASE_AQUIRE_RETRY_TIMEOUT}`)
134+
this.logger.info(`Lease is already taken retrying in ${LEASE_ACQUIRE_RETRY_TIMEOUT}`)
135135

136-
task.title = `${title} - ${chalk.gray(`lease exists, attempting again in ${LEASE_AQUIRE_RETRY_TIMEOUT} seconds`)}` +
136+
task.title = `${title} - ${chalk.gray(`lease exists, attempting again in ${LEASE_ACQUIRE_RETRY_TIMEOUT} seconds`)}` +
137137
`, attempt: ${chalk.cyan(attempt.toString())}/${chalk.cyan(maxAttempts.toString())}`
138138

139-
await sleep(LEASE_AQUIRE_RETRY_TIMEOUT)
139+
await sleep(LEASE_ACQUIRE_RETRY_TIMEOUT)
140140

141141
return this.acquireLeaseOrRetry(username, leaseName, namespace, task, title, attempt)
142142
}

‎test/e2e/integration/core/lease_manager.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import { expect } from 'chai'
2020
import { flags } from '../../../../src/commands/index.ts'
2121
import { e2eTestSuite, getDefaultArgv, TEST_CLUSTER } from '../../../test_util.ts'
2222
import * as version from '../../../../version.ts'
23-
import { LEASE_AQUIRE_RETRY_TIMEOUT, MAX_LEASE_ACQUIRE_ATTEMPTS, MINUTES } from '../../../../src/core/constants.ts'
23+
import { LEASE_ACQUIRE_RETRY_TIMEOUT, MAX_LEASE_ACQUIRE_ATTEMPTS, MINUTES } from '../../../../src/core/constants.ts'
2424
import { sleep } from '../../../../src/core/helpers.js'
2525

2626
const namespace = 'lease-mngr-e2e'
@@ -68,7 +68,7 @@ e2eTestSuite(namespace, argv, undefined, undefined, undefined, undefined, undefi
6868
// @ts-ignore to access private property
6969
await blockedLease.acquireTask({ title }, title, MAX_LEASE_ACQUIRE_ATTEMPTS - 1)
7070

71-
await sleep(LEASE_AQUIRE_RETRY_TIMEOUT * 2)
71+
await sleep(LEASE_ACQUIRE_RETRY_TIMEOUT * 2)
7272
} catch (e: Error | any) {
7373
expect(e.message).to.contain('Failed to acquire lease, max attempt reached')
7474
}

0 commit comments

Comments
 (0)