feat: add basic multi-cluster support to solo network deploy

.github/workflows/script/update_md.sh

@@ -27,7 +27,7 @@ export SOLO_INIT_OUTPUT=$( cat init.log | tee test.log )
 solo node keys --gossip-keys --tls-keys -i node1,node2,node3 | tee keys.log
 export SOLO_NODE_KEY_PEM_OUTPUT=$( cat keys.log | tee test.log )
 
-solo deployment create -n "${SOLO_NAMESPACE}" --context kind-${SOLO_CLUSTER_NAME} --email "${SOLO_EMAIL}" --deployment-clusters kind-${SOLO_CLUSTER_NAME} --deployment "${SOLO_DEPLOYMENT}" | tee deployment-create.log
+solo deployment create -i node1,node2,node3 -n "${SOLO_NAMESPACE}" --context kind-${SOLO_CLUSTER_NAME} --email "${SOLO_EMAIL}" --deployment-clusters kind-${SOLO_CLUSTER_NAME} --deployment "${SOLO_DEPLOYMENT}" | tee deployment-create.log
 export SOLO_DEPLOYMENT_CREATE_OUTPUT=$( cat deployment-create.log | tee test.log )
 
 solo cluster setup -s "${SOLO_CLUSTER_SETUP_NAMESPACE}" | tee cluster-setup.log

Taskfile.helper.yml

@@ -165,7 +165,7 @@ tasks:
     deps:
       - task: "init"
     cmds:
-      - SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- deployment create -n {{ .SOLO_NAMESPACE }} --context kind-${SOLO_CLUSTER_NAME} --email {{ .SOLO_EMAIL }} --deployment-clusters kind-${SOLO_CLUSTER_NAME} --deployment "${SOLO_DEPLOYMENT}" --dev
+      - SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- deployment create -n {{ .SOLO_NAMESPACE }} --context kind-${SOLO_CLUSTER_NAME} --email {{ .SOLO_EMAIL }} --deployment-clusters kind-${SOLO_CLUSTER_NAME} --cluster-ref kind-${SOLO_CLUSTER_NAME} --deployment "${SOLO_DEPLOYMENT}" --node-aliases {{.node_identifiers}} --dev
 
   solo:keys:
     silent: true

src/commands/account.ts

@@ -2,15 +2,14 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 import chalk from 'chalk';
-import {BaseCommand} from './base.js';
+import {BaseCommand, type Opts} from './base.js';
 import {IllegalArgumentError, SoloError} from '../core/errors.js';
 import {Flags as flags} from './flags.js';
 import {Listr} from 'listr2';
 import * as constants from '../core/constants.js';
 import {FREEZE_ADMIN_ACCOUNT} from '../core/constants.js';
 import {type AccountManager} from '../core/account_manager.js';
 import {type AccountId, AccountInfo, HbarUnit, PrivateKey} from '@hashgraph/sdk';
-import {type Opts} from '../types/command_types.js';
 import {ListrLease} from '../core/lease/listr_lease.js';
 import {type CommandBuilder} from '../types/aliases.js';
 import {sleep} from '../core/helpers.js';
@@ -33,7 +32,7 @@
     super(opts);
 
     if (!opts || !opts.accountManager)
       throw new IllegalArgumentError('An instance of core/AccountManager is required', opts.accountManager as any);
 
     this.accountManager = opts.accountManager;
     this.accountInfo = null;
@@ -68,7 +67,7 @@
       try {
         const privateKey = PrivateKey.fromStringDer(newAccountInfo.privateKey);
         newAccountInfo.privateKeyRaw = privateKey.toStringRaw();
       } catch (e: Error | any) {
         this.logger.error(`failed to retrieve EVM address for accountId ${newAccountInfo.accountId}`);
       }
     }
@@ -109,7 +108,7 @@
     return this.accountManager.accountInfoQuery(ctx.config.accountId);
   }
 
   async updateAccountInfo(ctx: any) {
     let amount = ctx.config.amount;
     if (ctx.config.ed25519PrivateKey) {
       if (
@@ -145,7 +144,7 @@
     return await this.accountManager.transferAmount(constants.TREASURY_ACCOUNT_ID, toAccountId, amount);
   }
 
   async init(argv: any) {
     const self = this;
 
     interface Context {
@@ -212,7 +211,7 @@
                 {
                   title: 'Update special account key sets',
                   task: ctx => {
                     const subTasks: any[] = [];
                     const realm = constants.HEDERA_NODE_ACCOUNT_ID_START.realm;
                     const shard = constants.HEDERA_NODE_ACCOUNT_ID_START.shard;
                     for (const currentSet of ctx.accountsBatchedSet) {
@@ -286,7 +285,7 @@
 
     try {
       await tasks.run();
     } catch (e: Error | any) {
       throw new SoloError(`Error in creating account: ${e.message}`, e);
     } finally {
       await this.closeConnections();
@@ -298,7 +297,7 @@
     return true;
   }
 
   async create(argv: any) {
     const self = this;
     const lease = await self.leaseManager.create();
 
@@ -374,7 +373,7 @@
 
     try {
       await tasks.run();
     } catch (e: Error | any) {
       throw new SoloError(`Error in creating account: ${e.message}`, e);
     } finally {
       await lease.release();
@@ -384,7 +383,7 @@
     return true;
   }
 
   async update(argv: any) {
     const self = this;
 
     interface Context {

src/commands/base.ts

@@ -13,7 +13,6 @@
 import {type ChartManager} from '../core/chart_manager.js';
 import {type ConfigManager} from '../core/config_manager.js';
 import {type DependencyManager} from '../core/dependency_managers/index.js';
-import {type Opts} from '../types/command_types.js';
 import {type CommandFlag} from '../types/flag_types.js';
 import {type Lease} from '../core/lease/lease.js';
 import {Listr} from 'listr2';
@@ -22,12 +21,42 @@
 import fs from 'fs';
 import {Task} from '../core/task.js';
 import {ConsensusNode} from '../core/model/consensus_node.js';
+import {type ClusterRef, type ClusterRefs} from '../core/config/remote/types.js';
+import {Flags} from './flags.js';
+import {type Cluster} from '../core/config/remote/cluster.js';
+import {Templates} from '../core/templates.js';
+import {type SoloLogger} from '../core/logging.js';
+import {type PackageDownloader} from '../core/package_downloader.js';
+import {type PlatformInstaller} from '../core/platform_installer.js';
+import {type KeyManager} from '../core/key_manager.js';
+import {type AccountManager} from '../core/account_manager.js';
+import {type ProfileManager} from '../core/profile_manager.js';
+import {type CertificateManager} from '../core/certificate_manager.js';
 import {type NodeAlias} from '../types/aliases.js';
 
 export interface CommandHandlers {
   parent: BaseCommand;
 }
 
+export interface Opts {
+  logger: SoloLogger;
+  helm: Helm;
+  k8Factory: K8Factory;
+  downloader: PackageDownloader;
+  platformInstaller: PlatformInstaller;
+  chartManager: ChartManager;
+  configManager: ConfigManager;
+  depManager: DependencyManager;
+  keyManager: KeyManager;
+  accountManager: AccountManager;
+  profileManager: ProfileManager;
+  leaseManager: LeaseManager;
+  certificateManager: CertificateManager;
+  localConfig: LocalConfig;
+  remoteConfigManager: RemoteConfigManager;
+  parent?: BaseCommand;
+}
+
 export abstract class BaseCommand extends ShellRunner {
   protected readonly helm: Helm;
   protected readonly k8Factory: K8Factory;
@@ -73,6 +102,7 @@
     return `${chartRepo}/${chartReleaseName}`;
   }
 
+  // FIXME @Deprecated. Use prepareValuesFilesMap instead to support multi-cluster
   public prepareValuesFiles(valuesFile: string) {
     let valuesArg = '';
     if (valuesFile) {
@@ -86,6 +116,87 @@
     return valuesArg;
   }
 
+  /**
+   * Prepare the values files map for each cluster
+   *
+   * <p> Order of precedence:
+   * <ol>
+   *   <li> Chart's default values file (if chartDirectory is set) </li>
+   *   <li> Profile values file </li>
+   *   <li> User's values file </li>
+   * </ol>
+   * @param clusterRefs - the map of cluster references
+   * @param valuesFileInput - the values file input string
+   * @param chartDirectory - the chart directory
+   * @param profileValuesFile - the profile values file
+   */
+  static prepareValuesFilesMap(
+    clusterRefs: ClusterRefs,
+    chartDirectory?: string,
+    profileValuesFile?: string,
+    valuesFileInput?: string,
+  ): Record<ClusterRef, string> {
+    // initialize the map with an empty array for each cluster-ref
+    const valuesFiles: Record<ClusterRef, string> = {
+      [Flags.KEY_COMMON]: '',
+    };
+    Object.keys(clusterRefs).forEach(clusterRef => {
+      valuesFiles[clusterRef] = '';
+    });
+
+    // add the chart's default values file for each cluster-ref if chartDirectory is set
+    // this should be the first in the list of values files as it will be overridden by user's input
+    if (chartDirectory) {
+      const chartValuesFile = path.join(chartDirectory, 'solo-deployment', 'values.yaml');
+      for (const clusterRef in valuesFiles) {
+        valuesFiles[clusterRef] += ` --values ${chartValuesFile}`;
+      }
+    }
+
+    if (profileValuesFile) {
+      const parsed = Flags.parseValuesFilesInput(profileValuesFile);
+      Object.entries(parsed).forEach(([clusterRef, files]) => {
+        let vf = '';
+        files.forEach(file => {
+          vf += ` --values ${file}`;
+        });
+
+        if (clusterRef === Flags.KEY_COMMON) {
+          Object.entries(valuesFiles).forEach(([cf]) => {
+            valuesFiles[cf] += vf;
+          });
+        } else {
+          valuesFiles[clusterRef] += vf;
+        }
+      });
+    }
+
+    if (valuesFileInput) {
+      const parsed = Flags.parseValuesFilesInput(valuesFileInput);
+      Object.entries(parsed).forEach(([clusterRef, files]) => {
+        let vf = '';
+        files.forEach(file => {
+          vf += ` --values ${file}`;
+        });
+
+        if (clusterRef === Flags.KEY_COMMON) {
+          Object.entries(valuesFiles).forEach(([clusterRef]) => {
+            valuesFiles[clusterRef] += vf;
+          });
+        } else {
+          valuesFiles[clusterRef] += vf;
+        }
+      });
+    }
+
+    if (Object.keys(valuesFiles).length > 1) {
+      // delete the common key if there is another cluster to use
+      delete valuesFiles[Flags.KEY_COMMON];
+    }
+
+    return valuesFiles;
+  }
+
   public getConfigManager(): ConfigManager {
     return this.configManager;
   }
@@ -105,6 +216,7 @@
     // build the dynamic class that will keep track of which properties are used
     const NewConfigClass = class {
       private usedConfigs: Map<string, number>;
+
       constructor() {
         // the map to keep track of which properties are used
         this.usedConfigs = new Map();
@@ -254,6 +366,7 @@
    */
   public getConsensusNodes(): ConsensusNode[] {
     const consensusNodes: ConsensusNode[] = [];
+    const clusters: Record<ClusterRef, Cluster> = this.getRemoteConfigManager().clusters;
 
     try {
       if (!this.getRemoteConfigManager()?.components?.consensusNodes) return [];
@@ -262,18 +375,30 @@
     }
 
     // using the remoteConfigManager to get the consensus nodes
-    Object.values(this.getRemoteConfigManager().components.consensusNodes).forEach(node => {
-      consensusNodes.push(
-        new ConsensusNode(
-          node.name as NodeAlias,
-          node.nodeId,
-          node.namespace,
-          node.cluster,
-          // use local config to get the context
-          this.getLocalConfig().clusterRefs[node.cluster],
-        ),
-      );
-    });
+    if (this.getRemoteConfigManager()?.components?.consensusNodes) {
+      Object.values(this.getRemoteConfigManager().components.consensusNodes).forEach(node => {
+        consensusNodes.push(
+          new ConsensusNode(
+            node.name as NodeAlias,
+            node.nodeId,
+            node.namespace,
+            node.cluster,
+            // use local config to get the context
+            this.getLocalConfig().clusterRefs[node.cluster],
+            clusters[node.cluster].dnsBaseDomain,
+            clusters[node.cluster].dnsConsensusNodePattern,
+            Templates.renderConsensusNodeFullyQualifiedDomainName(
+              node.name as NodeAlias,
+              node.nodeId,
+              node.namespace,
+              node.cluster,
+              clusters[node.cluster].dnsBaseDomain,
+              clusters[node.cluster].dnsConsensusNodePattern,
+            ),
+          ),
+        );
+      });
+    }
 
     // return the consensus nodes
     return consensusNodes;
@@ -292,4 +417,18 @@
     });
     return contexts;
   }
+
+  /**
+   * Gets a list of distinct cluster references from the consensus nodes
+   * @returns an object of cluster references
+   */
+  public getClusterRefs(): ClusterRefs {
+    const clustersRefs: ClusterRefs = {};
+    this.getConsensusNodes().forEach(node => {
+      if (!Object.keys(clustersRefs).includes(node.cluster)) {
+        clustersRefs[node.cluster] = node.context;
+      }
+    });
+    return clustersRefs;
+  }
 }

src/commands/cluster/configs.ts

@@ -72,7 +72,7 @@ export const resetConfigBuilder = async function (argv, ctx, task) {
   this.parent.getConfigManager().update(argv);
 
   ctx.config = {
-    clusterName: this.parent.getConfigManager().getFlag(flags.clusterName) as string,
+    clusterName: this.parent.getConfigManager().getFlag(flags.clusterRef) as string,
     clusterSetupNamespace: this.parent.getConfigManager().getFlag(flags.clusterSetupNamespace) as string,
   } as ClusterResetConfigClass;
 

src/commands/cluster/flags.ts

@@ -15,7 +15,7 @@ export const SETUP_FLAGS = {
   requiredFlagsWithDisabledPrompt: [],
   optionalFlags: [
     flags.chartDirectory,
-    flags.clusterName,
+    flags.clusterRef,
     flags.clusterSetupNamespace,
     flags.deployCertManager,
     flags.deployCertManagerCrds,
@@ -29,7 +29,7 @@ export const SETUP_FLAGS = {
 export const RESET_FLAGS = {
   requiredFlags: [],
   requiredFlagsWithDisabledPrompt: [],
-  optionalFlags: [flags.clusterName, flags.clusterSetupNamespace, flags.force, flags.quiet],
+  optionalFlags: [flags.clusterRef, flags.clusterSetupNamespace, flags.force, flags.quiet],
 };
 
 export const CONNECT_FLAGS = {
@@ -39,7 +39,7 @@ export const CONNECT_FLAGS = {
     flags.devMode,
     flags.deployment,
     flags.quiet,
-    flags.clusterName,
+    flags.clusterRef,
     flags.context,
     flags.namespace,
     flags.userEmailAddress,

src/commands/cluster/index.ts

@@ -4,8 +4,7 @@
 
 import * as ContextFlags from './flags.js';
 import {YargsCommand} from '../../core/yargs_command.js';
-import {BaseCommand} from './../base.js';
-import {type Opts} from '../../types/command_types.js';
+import {BaseCommand, type Opts} from './../base.js';
 import {ClusterCommandTasks} from './tasks.js';
 import {ClusterCommandHandlers} from './handlers.js';
 import {DEFAULT_FLAGS, RESET_FLAGS, SETUP_FLAGS} from './flags.js';

src/commands/cluster/tasks.ts

@@ -266,13 +266,18 @@ export class ClusterCommandTasks {
         const configManager = this.parent.getConfigManager();
         const isQuiet = configManager.getFlag<boolean>(flags.quiet);
         const deploymentName: string = configManager.getFlag<DeploymentName>(flags.deployment);
-        let clusters = splitFlagInput(configManager.getFlag<string>(flags.clusterName));
+        let clusters = splitFlagInput(configManager.getFlag<string>(flags.clusterRef));
         const contexts = splitFlagInput(configManager.getFlag<string>(flags.context));
         const namespace = configManager.getFlag<NamespaceName>(flags.namespace);
         const localConfig = this.parent.getLocalConfig();
         let selectedContext: string;
         let selectedCluster: string;
 
+        // TODO - BEGIN... added this because it was confusing why we have both clusterRef and deploymentClusters
+        if (clusters?.length === 0) {
+          clusters = splitFlagInput(configManager.getFlag<string>(flags.deploymentClusters));
+        }
+
         // If one or more contexts are provided, use the first one
         if (contexts.length) {
           selectedContext = contexts[0];
@@ -319,7 +324,7 @@ export class ClusterCommandTasks {
 
             // Prompt user for clusters and contexts
             else {
-              const promptedClusters = await flags.clusterName.prompt(task, '');
+              const promptedClusters = await flags.clusterRef.prompt(task, '');
               clusters = splitFlagInput(promptedClusters);
 
               for (const cluster of clusters) {