feat: use gcs for stream file storage (#994)

Signed-off-by: Jeffrey Tang <[email protected]>
Signed-off-by: JeffreyDallas <[email protected]>
Signed-off-by: Jeromy Cannon <[email protected]>
Co-authored-by: Jeromy Cannon <[email protected]>

Author: JeffreyDallas

.github/workflows/flow-gcs-test.yaml

@@ -0,0 +1,113 @@
+##
+# Copyright (C) 2023-2024 Hedera Hashgraph, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+name: "Test GCS as bucket storage"
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+jobs:
+  gcs-storage-test:
+    timeout-minutes: 20
+    runs-on: solo-linux-large
+    strategy:
+      matrix:
+        storageType: ["gcs_only", "gcs_and_minio"]
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
+        with:
+          egress-policy: audit
+
+      - name: Checkout Code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Authenticate to Google Cloud
+        id: google-auth
+        uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
+        with:
+          workload_identity_provider: "projects/652966097426/locations/global/workloadIdentityPools/solo-bucket-dev-pool/providers/gh-provider"
+          service_account: "solo-bucket-reader-writer@solo-bucket-dev.iam.gserviceaccount.com"
+
+      - name: Setup Google Cloud SDK
+        uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2
+
+      - name: Get Current Job Log URL
+        uses: Tiryoh/gha-jobid-action@v1
+        id: jobs
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          job_name: "gcs-storage-test (${{ matrix.storageType }})"
+
+      - name: Create GCS bucket
+        # create a new bucket and use job runner id as prefix
+        run: |
+          export BUCKET_NAME=${{ steps.jobs.outputs.job_id }}-solo-streams
+          gcloud storage buckets create gs://${BUCKET_NAME} --project=${{ vars.GCP_S3_PROJECT_ID }}
+          echo "BUCKET_NAME=${BUCKET_NAME}" >> $GITHUB_ENV
+
+      - name: Setup Node
+        uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
+        with:
+          node-version: 20
+          cache: npm
+
+      - name: Setup Helm
+        uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
+        with:
+          version: "v3.12.3" # helm version
+
+      - name: Setup Kind
+        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+        with:
+          install_only: true
+          node_image: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
+          version: v0.21.0
+          kubectl_version: v1.28.6
+          verbosity: 3
+          wait: 120s
+
+      - name: Install Dependencies
+        id: npm-deps
+        run: |
+          npm ci
+          npm install -g @hashgraph/solo
+
+      - name: Compile Project
+        run: npm run build
+
+      - name: Run GCS Test Script for type ${{ matrix.channel }}
+        env:
+          GCS_ACCESS_KEY: ${{ secrets.GCP_S3_ACCESS_KEY }}
+          GCS_SECRET_KEY: ${{ secrets.GCP_S3_SECRET_KEY }}
+          BUCKET_NAME: ${{ env.BUCKET_NAME }}
+          STORAGE_TYPE: ${{ matrix.storageType }}
+        run: |
+          .github/workflows/script/gcs_test.sh
+
+      - name: Delete Bucket after Test
+        run: |
+          gcloud storage rm --recursive gs://${BUCKET_NAME} --project=${{ vars.GCP_S3_PROJECT_ID }} 

.github/workflows/script/gcs_test.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+set -eo pipefail
+
+source .github/workflows/script/helper.sh
+
+if [ -z "${GCS_ACCESS_KEY}" ]; then
+  echo "GCS_ACCESS_KEY is not set. Exiting..."
+  exit 1
+fi
+
+if [ -z "${GCS_SECRET_KEY}" ]; then
+  echo "GCS_SECRET_KEY is not set. Exiting..."
+  exit 1
+fi
+
+if [ -z "${BUCKET_NAME}" ]; then
+  streamBucket="solo-ci-test-streams"
+else
+  streamBucket=${BUCKET_NAME}
+fi
+
+if [ -z "${STORAGE_TYPE}" ]; then
+  storageType="gcs_and_minio"
+else
+  storageType=${STORAGE_TYPE}
+fi
+
+echo "Using bucket name: ${streamBucket}"
+echo "Test storage type: ${storageType}"
+
+SOLO_CLUSTER_NAME=solo-e2e
+SOLO_NAMESPACE=solo-e2e
+SOLO_CLUSTER_SETUP_NAMESPACE=solo-setup
+
+kind delete cluster -n "${SOLO_CLUSTER_NAME}"
+kind create cluster -n "${SOLO_CLUSTER_NAME}"
+npm run solo-test -- init
+npm run solo-test -- cluster setup \
+  -s "${SOLO_CLUSTER_SETUP_NAMESPACE}"
+npm run solo-test -- node keys --gossip-keys --tls-keys -i node1
+npm run solo-test -- network deploy -i node1 -n "${SOLO_NAMESPACE}" \
+  --storage-endpoint "https://storage.googleapis.com" \
+  --storage-access-key "${GCS_ACCESS_KEY}" --storage-secrets "${GCS_SECRET_KEY}" \
+  --storage-type "${storageType}" --storage-bucket "${streamBucket}"
+
+npm run solo-test -- node setup -i node1 -n "${SOLO_NAMESPACE}"
+npm run solo-test -- node start -i node1 -n "${SOLO_NAMESPACE}"
+npm run solo-test -- mirror-node deploy --namespace "${SOLO_NAMESPACE}" \
+  --storage-endpoint "https://storage.googleapis.com" \
+  --storage-access-key "${GCS_ACCESS_KEY}" --storage-secrets "${GCS_SECRET_KEY}" \
+  --storage-type "${storageType}" --storage-bucket "${streamBucket}"
+
+kubectl port-forward -n "${SOLO_NAMESPACE}" svc/haproxy-node1-svc 50211:50211 > /dev/null 2>&1 &
+kubectl port-forward -n "${SOLO_NAMESPACE}" svc/hedera-explorer 8080:80 > /dev/null 2>&1 &
+
+cd ..; create_test_account ; cd -
+
+node examples/create-topic.js
+
+npm run solo-test -- node stop -i node1 -n "${SOLO_NAMESPACE}"

.github/workflows/script/helper.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -eo pipefail
+
+function create_test_account ()
+{
+  echo "Create test account with solo network"
+  cd solo
+
+  # create new account and extract account id
+  npm run solo-test -- account create -n solo-e2e --hbar-amount 100 --generate-ecdsa-key --set-alias > test.log
+  export OPERATOR_ID=$(grep "accountId" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
+  echo "OPERATOR_ID=${OPERATOR_ID}"
+  rm test.log
+
+  # get private key of the account
+  npm run solo-test -- account get -n solo-e2e --account-id ${OPERATOR_ID} --private-key > test.log
+  export OPERATOR_KEY=$(grep "privateKey" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
+  export CONTRACT_TEST_KEY_ONE=0x$(grep "privateKeyRaw" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
+  echo "CONTRACT_TEST_KEY_ONE=${CONTRACT_TEST_KEY_ONE}"
+  rm test.log
+
+  npm run solo-test -- account create -n solo-e2e --hbar-amount 100 --generate-ecdsa-key --set-alias > test.log
+  export SECOND_KEY=$(grep "accountId" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
+  npm run solo-test -- account get -n solo-e2e --account-id ${SECOND_KEY} --private-key > test.log
+  export CONTRACT_TEST_KEY_TWO=0x$(grep "privateKeyRaw" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
+  echo "CONTRACT_TEST_KEY_TWO=${CONTRACT_TEST_KEY_TWO}"
+  rm test.log
+
+  export CONTRACT_TEST_KEYS=${CONTRACT_TEST_KEY_ONE},$'\n'${CONTRACT_TEST_KEY_TWO}
+  export HEDERA_NETWORK="local-node"
+
+  echo "OPERATOR_KEY=${OPERATOR_KEY}"
+  echo "HEDERA_NETWORK=${HEDERA_NETWORK}"
+  echo "CONTRACT_TEST_KEYS=${CONTRACT_TEST_KEYS}"
+
+  cd -
+}

.github/workflows/script/solo_smoke_test.sh

@@ -9,7 +9,7 @@ set -eo pipefail
 # Then run smart contract test, and also javascript sdk sample test to interact with solo network
 #
 
-function_name=""
+source .github/workflows/script/helper.sh
 
 function enable_port_forward ()
 {
@@ -73,41 +73,6 @@ function start_contract_test ()
   return $result
 }
 
-function create_test_account ()
-{
-  echo "Create test account with solo network"
-  cd solo
-
-  # create new account and extract account id
-  npm run solo-test -- account create -n solo-e2e --hbar-amount 100 --generate-ecdsa-key --set-alias > test.log
-  export OPERATOR_ID=$(grep "accountId" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
-  echo "OPERATOR_ID=${OPERATOR_ID}"
-  rm test.log
-
-  # get private key of the account
-  npm run solo-test -- account get -n solo-e2e --account-id ${OPERATOR_ID} --private-key > test.log
-  export OPERATOR_KEY=$(grep "privateKey" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
-  export CONTRACT_TEST_KEY_ONE=0x$(grep "privateKeyRaw" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
-  echo "CONTRACT_TEST_KEY_ONE=${CONTRACT_TEST_KEY_ONE}"
-  rm test.log
-
-  npm run solo-test -- account create -n solo-e2e --hbar-amount 100 --generate-ecdsa-key --set-alias > test.log
-  export SECOND_KEY=$(grep "accountId" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
-  npm run solo-test -- account get -n solo-e2e --account-id ${SECOND_KEY} --private-key > test.log
-  export CONTRACT_TEST_KEY_TWO=0x$(grep "privateKeyRaw" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
-  echo "CONTRACT_TEST_KEY_TWO=${CONTRACT_TEST_KEY_TWO}"
-  rm test.log
-
-  export CONTRACT_TEST_KEYS=${CONTRACT_TEST_KEY_ONE},$'\n'${CONTRACT_TEST_KEY_TWO}
-  export HEDERA_NETWORK="local-node"
-
-  echo "OPERATOR_KEY=${OPERATOR_KEY}"
-  echo "HEDERA_NETWORK=${HEDERA_NETWORK}"
-  echo "CONTRACT_TEST_KEYS=${CONTRACT_TEST_KEYS}"
-
-  cd -
-}
-
 function start_sdk_test ()
 {
   cd solo

examples/create-topic.js

@@ -14,9 +14,18 @@
  * limitations under the License.
  *
  */
-import {Wallet, LocalProvider, TopicCreateTransaction, TopicMessageSubmitTransaction} from '@hashgraph/sdk';
+import {
+  Wallet,
+  LocalProvider,
+  TopicCreateTransaction,
+  TopicMessageSubmitTransaction,
+  AccountCreateTransaction,
+  PrivateKey,
+  Hbar,
+} from '@hashgraph/sdk';
 
 import dotenv from 'dotenv';
+import http from 'http';
 
 dotenv.config();
 
@@ -30,12 +39,11 @@ async function main() {
 
   const wallet = new Wallet(process.env.OPERATOR_ID, process.env.OPERATOR_KEY, provider);
 
+  const TEST_MESSAGE = 'Hello World';
   try {
-    console.log('before create topic');
     // create topic
     let transaction = await new TopicCreateTransaction().freezeWithSigner(wallet);
     transaction = await transaction.signWithSigner(wallet);
-    console.log('after sign transaction');
     const createResponse = await transaction.executeWithSigner(wallet);
     const createReceipt = await createResponse.getReceiptWithSigner(wallet);
 
@@ -44,18 +52,73 @@ async function main() {
     // send one message
     let topicMessageSubmitTransaction = await new TopicMessageSubmitTransaction({
       topicId: createReceipt.topicId,
-      message: 'Hello World',
+      message: TEST_MESSAGE,
     }).freezeWithSigner(wallet);
     topicMessageSubmitTransaction = await topicMessageSubmitTransaction.signWithSigner(wallet);
     const sendResponse = await topicMessageSubmitTransaction.executeWithSigner(wallet);
 
     const sendReceipt = await sendResponse.getReceiptWithSigner(wallet);
 
     console.log(`topic sequence number = ${sendReceipt.topicSequenceNumber.toString()}`);
+
+    await new Promise(resolve => setTimeout(resolve, 1000));
+
+    // send a create account transaction to push record stream files to mirror node
+    const newKey = PrivateKey.generate();
+    let accountCreateTransaction = await new AccountCreateTransaction()
+      .setInitialBalance(new Hbar(10))
+      .setKey(newKey.publicKey)
+      .freezeWithSigner(wallet);
+    accountCreateTransaction = await accountCreateTransaction.signWithSigner(wallet);
+    const accountCreationResponse = await accountCreateTransaction.executeWithSigner(wallet);
+    const accountCreationReceipt = await accountCreationResponse.getReceiptWithSigner(wallet);
+    console.log(`account id = ${accountCreationReceipt.accountId.toString()}`);
+
+    await new Promise(resolve => setTimeout(resolve, 1000));
+
+    // Check submit message result should success
+    const queryURL = `http://localhost:8080/api/v1/topics/${createReceipt.topicId}/messages`;
+    let received = false;
+    let receivedMessage = '';
+
+    // wait until the transaction reached consensus and retrievable from the mirror node API
+    let retry = 0;
+    while (!received && retry < 10) {
+      const req = http.request(queryURL, {method: 'GET', timeout: 100, headers: {Connection: 'close'}}, res => {
+        res.setEncoding('utf8');
+        res.on('data', chunk => {
+          // convert chunk to json object
+          const obj = JSON.parse(chunk);
+          if (obj.messages.length === 0) {
+            console.log('No messages yet');
+          } else {
+            // convert message from base64 to utf-8
+            const base64 = obj.messages[0].message;
+            const buff = Buffer.from(base64, 'base64');
+            receivedMessage = buff.toString('utf-8');
+            console.log(`Received message: ${receivedMessage}`);
+            received = true;
+          }
+        });
+      });
+      req.on('error', e => {
+        console.log(`problem with request: ${e.message}`);
+      });
+      req.end(); // make the request
+      // wait and try again
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      retry++;
+    }
+    if (receivedMessage === TEST_MESSAGE) {
+      console.log('Message received successfully');
+    } else {
+      console.error('Message received but not match: ' + receivedMessage);
+      // eslint-disable-next-line n/no-process-exit
+      process.exit(1);
+    }
   } catch (error) {
     console.error(error);
   }
-
   provider.close();
 }