feat: update solo to support pathPrefix of mirror node importer (#1204)

Signed-off-by: Jeffrey Tang <[email protected]>

Author: JeffreyDallas

.github/workflows/flow-gcs-test.yaml

@@ -62,15 +62,15 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           job_name: "gcs-storage-test (${{ matrix.storageType }})"
 
-      - name: Create GCS bucket
-        # create a new bucket and use job runner id as prefix
+      - name: Create Bucket Name and Prefix
         run: |
-          export BUCKET_NAME=${{ steps.jobs.outputs.job_id }}-solo-streams
-          gcloud storage buckets create gs://${BUCKET_NAME} --project=${{ vars.GCP_S3_PROJECT_ID }}
+          export BUCKET_NAME="solo-ci-test-streams"
+          export PREFIX=${{ steps.jobs.outputs.job_id }}
+
           echo "BUCKET_NAME=${BUCKET_NAME}" >> $GITHUB_ENV
+          echo "PREFIX=${PREFIX}" >> $GITHUB_ENV          
           
-          export BACKUP_BUCKET_NAME=${{ steps.jobs.outputs.job_id }}-solo-backups
-          gcloud storage buckets create gs://${BACKUP_BUCKET_NAME} --project=${{ vars.GCP_S3_PROJECT_ID }}
+          export BACKUP_BUCKET_NAME="solo-ci-backups"
           echo "BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME}" >> $GITHUB_ENV
 
       - name: Setup Node
@@ -108,13 +108,14 @@ jobs:
           GCS_ACCESS_KEY: ${{ secrets.GCP_S3_ACCESS_KEY }}
           GCS_SECRET_KEY: ${{ secrets.GCP_S3_SECRET_KEY }}
           BUCKET_NAME: ${{ env.BUCKET_NAME }}
-          BACKUP_BUCKET_NAME: ${{ env.BACKUP_BUCKET_NAME }}
+          PREFIX: ${{ env.PREFIX }}
+          BACKUP_BUCKET_NAME: ${{ env.BACKUP_BUCKET_NAME }}/${{ steps.jobs.outputs.job_id }}
           STORAGE_TYPE: ${{ matrix.storageType }}
           GCP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.GCP_SERVICE_ACCOUNT_TOKEN }}
         run: |
           .github/workflows/script/gcs_test.sh
 
-      - name: Delete Bucket after Test
+      - name: Delete Test Directory after Test
         run: |
-          gcloud storage rm --recursive gs://${BUCKET_NAME} --project=${{ vars.GCP_S3_PROJECT_ID }} 
-          gcloud storage rm --recursive gs://${BACKUP_BUCKET_NAME} --project=${{ vars.GCP_S3_PROJECT_ID }}
+          gcloud storage rm --recursive gs://${BUCKET_NAME}/${{ steps.jobs.outputs.job_id }} --project=${{ vars.GCP_S3_PROJECT_ID }}
+          gcloud storage rm --recursive gs://${BACKUP_BUCKET_NAME}/${{ steps.jobs.outputs.job_id }} --project=${{ vars.GCP_S3_PROJECT_ID }}

.github/workflows/script/gcs_test.sh

@@ -36,6 +36,13 @@ if [ -z "${GCP_SERVICE_ACCOUNT_TOKEN}" ]; then
   exit 1
 fi
 
+if [ -z "${PREFIX}" ]; then
+  echo "PREFIX is not set"
+else
+  echo "Using PREFIX: ${PREFIX}"
+  BUCKET_PREFIX_OPTION="--storage-bucket-prefix ${PREFIX}"
+fi
+
 echo "${GCP_SERVICE_ACCOUNT_TOKEN}" > gcp_service_account.json
 
 echo "Using bucket name: ${streamBucket}"
@@ -45,6 +52,7 @@ SOLO_CLUSTER_NAME=solo-e2e
 SOLO_NAMESPACE=solo-e2e
 SOLO_CLUSTER_SETUP_NAMESPACE=solo-setup
 
+
 kind delete cluster -n "${SOLO_CLUSTER_NAME}"
 kind create cluster -n "${SOLO_CLUSTER_NAME}"
 npm run solo-test -- init
@@ -54,7 +62,7 @@ npm run solo-test -- node keys --gossip-keys --tls-keys -i node1
 npm run solo-test -- network deploy -i node1 -n "${SOLO_NAMESPACE}" \
   --storage-endpoint "https://storage.googleapis.com" \
   --storage-access-key "${GCS_ACCESS_KEY}" --storage-secrets "${GCS_SECRET_KEY}" \
-  --storage-type "${storageType}" --storage-bucket "${streamBucket}" \
+  --storage-type "${storageType}" --storage-bucket "${streamBucket}" $BUCKET_PREFIX_OPTION \
   --backup-bucket "${streamBackupBucket}" \
   --google-credential gcp_service_account.json
 
@@ -63,24 +71,31 @@ npm run solo-test -- node start -i node1 -n "${SOLO_NAMESPACE}"
 npm run solo-test -- mirror-node deploy --namespace "${SOLO_NAMESPACE}" \
   --storage-endpoint "https://storage.googleapis.com" \
   --storage-access-key "${GCS_ACCESS_KEY}" --storage-secrets "${GCS_SECRET_KEY}" \
-  --storage-type "${storageType}" --storage-bucket "${streamBucket}"
+  --storage-type "${storageType}" --storage-bucket "${streamBucket}" $BUCKET_PREFIX_OPTION
+
+npm run solo-test -- explorer deploy -n "${SOLO_NAMESPACE}" -s "${SOLO_CLUSTER_SETUP_NAMESPACE}"
 
 kubectl port-forward -n "${SOLO_NAMESPACE}" svc/haproxy-node1-svc 50211:50211 > /dev/null 2>&1 &
-kubectl port-forward -n "${SOLO_NAMESPACE}" svc/hedera-explorer 8080:80 > /dev/null 2>&1 &
+
+explorer_svc="$(kubectl get svc -l app.kubernetes.io/component=hedera-explorer -n ${SOLO_NAMESPACE} --output json | jq -r '.items[].metadata.name')"
+kubectl port-forward -n "${SOLO_NAMESPACE}" svc/"${explorer_svc}" 8080:80 > /dev/null 2>&1 &
 
 cd ..; create_test_account ; cd -
 
 node examples/create-topic.js
 
 npm run solo-test -- node stop -i node1 -n "${SOLO_NAMESPACE}"
 
+echo "Waiting for backup uploader to run"
 # manually call script "backup.sh" from container backup-uploader since it only runs every 5 minutes
-kubectl exec network-node1-0 -c backup-uploader -n solo-e2e -- /backup.sh > /dev/null 2>&1
+kubectl exec network-node1-0 -c backup-uploader -n solo-e2e -- /backup.sh
 
-# retrieve logs and check if it include the message type "error"
+echo "Retrieve logs and check if it include the error message"
 # example : {"level":"error","msg":"Updated modification time ......}
 kubectl logs network-node1-0 -c backup-uploader -n solo-e2e > backup-uploader.log
 if grep -q \""error\"" backup-uploader.log; then
   echo "Backup uploader logs contain error message"
   exit 1
 fi
+
+npm run solo-test -- network destroy -n "${SOLO_NAMESPACE}" --force -q

docs/content/User/AccessHederaServices.md

@@ -23,7 +23,7 @@ Once the nodes are up, you may now expose various services (using `k9s` (shift-f
 * Hedera explorer: `solo-deployment-hedera-explorer`
   ```bash
   #enable portforwarding for hedera explorer, can be access at http://localhost:8080/
-  kubectl port-forward svc/hedera-explorer -n "${SOLO_NAMESPACE}" 8080:80 &
+  kubectl port-forward svc/hedera-explorer-hedera-explorer-chart -n "${SOLO_NAMESPACE}" 8080:80 &
   ```
 * JSON Rpc Relays
 

docs/content/User/SoloWithMirrorNode.md

@@ -18,7 +18,7 @@ solo node start     -n "${SOLO_NAMESPACE}" -i node1,node2
 solo mirror-node deploy -n "${SOLO_NAMESPACE}"
 
 kubectl port-forward svc/haproxy-node1-svc -n "${SOLO_NAMESPACE}" 50211:50211 > /dev/null 2>&1 &
-kubectl port-forward svc/hedera-explorer -n "${SOLO_NAMESPACE}" 8080:80 > /dev/null 2>&1 &
+kubectl port-forward svc/hedera-explorer-hedera-explorer-chart -n "${SOLO_NAMESPACE}" 8080:80 > /dev/null 2>&1 &
 
 ```
 

src/commands/flags.ts

@@ -1670,6 +1670,17 @@ export class Flags {
     prompt: undefined,
   };
 
+  static readonly storageBucketPrefix: CommandFlag = {
+    constName: 'storageBucketPrefix',
+    name: 'storage-bucket-prefix',
+    definition: {
+      defaultValue: '',
+      describe: 'path prefix of storage bucket',
+      type: 'string',
+    },
+    prompt: undefined,
+  };
+
   static readonly backupBucket: CommandFlag = {
     constName: 'backupBucket',
     name: 'backup-bucket',
@@ -1786,6 +1797,7 @@ export class Flags {
     Flags.storageSecrets,
     Flags.storageEndpoint,
     Flags.storageBucket,
+    Flags.storageBucketPrefix,
     Flags.backupBucket,
     Flags.googleCredential,
     Flags.tlsClusterIssuerType,

src/commands/mirror_node.ts

@@ -53,6 +53,7 @@ interface MirrorNodeDeployConfigClass {
   storageSecrets: string;
   storageEndpoint: string;
   storageBucket: string;
+  storageBucketPrefix: string;
 }
 
 interface Context {
@@ -97,6 +98,7 @@ export class MirrorNodeCommand extends BaseCommand {
       flags.storageSecrets,
       flags.storageEndpoint,
       flags.storageBucket,
+      flags.storageBucketPrefix,
     ];
   }
 
@@ -116,6 +118,10 @@ export class MirrorNodeCommand extends BaseCommand {
     if (config.storageBucket) {
       valuesArg += ` --set importer.config.hedera.mirror.importer.downloader.bucketName=${config.storageBucket}`;
     }
+    if (config.storageBucketPrefix) {
+      this.logger.info(`Setting storage bucket prefix to ${config.storageBucketPrefix}`);
+      valuesArg += ` --set importer.config.hedera.mirror.importer.downloader.pathPrefix=${config.storageBucketPrefix}`;
+    }
 
     let storageType = '';
     if (config.storageType && config.storageAccessKey && config.storageSecrets && config.storageEndpoint) {

src/commands/network.ts

@@ -77,6 +77,7 @@ export interface NetworkDeployConfigClass {
   storageSecrets: string;
   storageEndpoint: string;
   storageBucket: string;
+  storageBucketPrefix: string;
   backupBucket: string;
   googleCredential: string;
 }
@@ -147,6 +148,7 @@ export class NetworkCommand extends BaseCommand {
       flags.storageSecrets,
       flags.storageEndpoint,
       flags.storageBucket,
+      flags.storageBucketPrefix,
       flags.backupBucket,
       flags.googleCredential,
     ];
@@ -253,6 +255,7 @@ export class NetworkCommand extends BaseCommand {
     storageSecrets: string;
     storageEndpoint: string;
     storageBucket: string;
+    storageBucketPrefix: string;
     backupBucket: string;
     googleCredential: string;
     loadBalancerEnabled: boolean;
@@ -304,6 +307,10 @@ export class NetworkCommand extends BaseCommand {
       valuesArg += ` --set minio-server.tenant.buckets[0].name=${config.storageBucket}`;
     }
 
+    if (config.storageBucketPrefix) {
+      valuesArg += ` --set cloud.buckets.streamBucketPrefix=${config.storageBucketPrefix}`;
+    }
+
     if (config.backupBucket) {
       valuesArg += ' --set defaults.sidecars.backupUploader.enabled=true';
       valuesArg += ` --set defaults.sidecars.backupUploader.config.backupBucket=${config.backupBucket}`;
@@ -387,6 +394,7 @@ export class NetworkCommand extends BaseCommand {
       flags.storageSecrets,
       flags.storageEndpoint,
       flags.storageBucket,
+      flags.storageBucketPrefix,
     ]);
 
     await this.configManager.executePrompt(task, NetworkCommand.DEPLOY_FLAGS_LIST);

version.ts

@@ -20,9 +20,9 @@
  */
 
 export const HELM_VERSION = 'v3.14.2';
-export const SOLO_CHART_VERSION = '0.42.10';
+export const SOLO_CHART_VERSION = '0.43.0';
 export const HEDERA_PLATFORM_VERSION = 'v0.59.0-main.x5322bdc';
 export const LOCAL_HEDERA_PLATFORM_VERSION = 'v0.58.3';
-export const MIRROR_NODE_VERSION = '0.120.1';
+export const MIRROR_NODE_VERSION = 'v0.122.0-rc1';
 export const HEDERA_EXPLORER_VERSION = '24.12.0';
 export const HEDERA_JSON_RPC_RELAY_VERSION = 'v0.63.2';