feat(mirror-node): Enhance mirror-node external database feature (#1230)

Signed-off-by: instamenta <[email protected]>

Author: instamenta

.github/workflows/flow-task-test.yaml

@@ -75,7 +75,7 @@ jobs:
 
       - name: Run Example Task File Test with type ${{ matrix.type }}
         run: |
+          cd examples/external-database-test
           export CONSENSUS_NODE_VERSION=v0.58.3
-          SOLO_CLUSTER_NAME=solo-task-test-${{ matrix.type }}-${{ github.run_id }}-${{ github.run_attempt }} task default
-          .github/workflows/script/solo_smoke_test.sh ${{ matrix.type }}
-          task clean
+          SOLO_CLUSTER_NAME=solo-task-test-${{ matrix.type }}-${{ github.run_id }}-${{ github.run_attempt }} task install
+          task destroy

.github/workflows/script/helper.sh

@@ -7,23 +7,23 @@ function create_test_account ()
   cd solo
 
   # create new account and extract account id
-  npm run solo-test -- account create --deployment solo-deployment --hbar-amount 100 --generate-ecdsa-key --set-alias > test.log
+  npm run solo-test -- account create --deployment solo-e2e --hbar-amount 100 --generate-ecdsa-key --set-alias > test.log
   export OPERATOR_ID=$(grep "accountId" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
   echo "OPERATOR_ID=${OPERATOR_ID}"
   rm test.log
 
   # get private key of the account
-  npm run solo-test -- account get --deployment solo-deployment --account-id ${OPERATOR_ID} --private-key > test.log
+  npm run solo-test -- account get --deployment solo-e2e --account-id ${OPERATOR_ID} --private-key > test.log
 
   # retrieve the field privateKey but not privateKeyRaw
   export OPERATOR_KEY=$(grep "privateKey" test.log | grep -v "privateKeyRaw" | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
   export CONTRACT_TEST_KEY_ONE=0x$(grep "privateKeyRaw" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
   echo "CONTRACT_TEST_KEY_ONE=${CONTRACT_TEST_KEY_ONE}"
   rm test.log
 
-  npm run solo-test -- account create --deployment solo-deployment --hbar-amount 100 --generate-ecdsa-key --set-alias > test.log
+  npm run solo-test -- account create --deployment solo-e2e --hbar-amount 100 --generate-ecdsa-key --set-alias > test.log
   export SECOND_KEY=$(grep "accountId" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
-  npm run solo-test -- account get --deployment solo-deployment --account-id ${SECOND_KEY} --private-key > test.log
+  npm run solo-test -- account get --deployment solo-e2e --account-id ${SECOND_KEY} --private-key > test.log
   export CONTRACT_TEST_KEY_TWO=0x$(grep "privateKeyRaw" test.log | awk '{print $2}' | sed 's/"//g'| sed 's/,//g')
   echo "CONTRACT_TEST_KEY_TWO=${CONTRACT_TEST_KEY_TWO}"
   rm test.log

.github/workflows/script/solo_smoke_test.sh

@@ -59,7 +59,7 @@ function start_background_transactions ()
   # generate accounts as background traffic for two minutes
   # so record stream files can be kept pushing to mirror node
   cd solo
-  npm run solo-test -- account create --deployment solo-deployment --create-amount 15 > /dev/null 2>&1 &
+  npm run solo-test -- account create --deployment solo-e2e --create-amount 15 > /dev/null 2>&1 &
   cd -
 }
 
@@ -120,12 +120,9 @@ function check_importer_log()
 # then call solo account init before deploy mirror and relay node
 if [ "$1" == "account-init" ]; then
   echo "Call solo account init"
-  npm run solo-test -- account init --deployment solo-deployment
+  npm run solo-test -- account init --deployment solo-e2e
 fi
 
-task solo:mirror-node
-task solo:relay
-
 echo "Change to parent directory"
 
 cd ../

Taskfile.helper.yml

@@ -478,3 +478,32 @@ tasks:
     cmds:
       - echo "Cleaning up temporary files..."
       - rm -f /tmp/solo-${USER}-* || true
+
+  solo:external-database:
+    silent: false
+    desc: setup external database PostgreSQL with helm
+    cmds:
+      - |
+        {{.solo_bin_dir}}/helm install {{.postgres_name}} https://charts.bitnami.com/bitnami/postgresql-12.1.2.tgz \
+          --set image.tag=16.4.0 \
+          --namespace {{.postgres_database_namespace}} --create-namespace \
+          --set global.postgresql.auth.postgresPassword={{.postgres_password}} \
+          --set primary.persistence.enabled=false --set secondary.enabled=false
+      - name: "Wait for PostgreSQL pod to be ready"
+        cmd: |
+          kubectl wait --for=condition=ready pod/{{.postgres_container_name}} \
+          -n {{.postgres_database_namespace}} --timeout=160s
+      - name: "Copy init.sql inside the database pod"
+        cmd: |
+          kubectl cp ../external-database-test/scripts/init.sh \
+          {{.postgres_container_name}}:/tmp/init.sh \
+          -n {{.postgres_database_namespace}}
+      - name: "Make init.sh executable"
+        cmd: |
+          kubectl exec -it {{.postgres_container_name}} \
+          -n {{.postgres_database_namespace}} -- chmod +x /tmp/init.sh
+      - name: "Execute init.sh inside the database pod"
+        cmd: |
+          kubectl exec -it {{.postgres_container_name}} \
+          -n {{.postgres_database_namespace}} \
+          -- /bin/bash /tmp/init.sh "{{.postgres_username}}" "{{.postgres_password}}"

examples/custom-mirror-node-database/scripts/init.sh

@@ -0,0 +1,65 @@
+version: 3
+includes:
+  main:
+    taskfile: ../../Taskfile.helper.yml
+    flatten: true
+vars:
+  use_port_forwards: "true"
+  postgres_username: "postgres"
+  postgres_password: "XXXXXXXXX"
+  postgres_name: "my-postgresql"
+  postgres_container_name: "{{.postgres_name}}-0"
+  postgres_host_fqdn: "{{.postgres_name}}.database.svc.cluster.local"
+  postgres_container_fdqn: "{{.postgres_container_name}}.database.svc.cluster.local"
+  postgres_mirror_node_database_name: "mirror_node"
+  postgres_database_namespace: "database"
+env:
+  SOLO_NETWORK_SIZE: "1"
+  SOLO_DEPLOYMENT: "solo-e2e"
+  SOLO_NAMESPACE: "solo-e2e"
+  SOLO_CLUSTER_NAME: "solo-e2e"
+  MIRROR_NODE_DEPLOY_EXTRA_FLAGS: "--use-external-database --external-database-host {{.postgres_host_fqdn}} --external-database-owner-username {{.postgres_username}} --external-database-owner-password {{.postgres_password}}"
+tasks:
+  default:
+    silent: true
+    desc: install Solo, create a kind cluster, deploy the network, set it up, and start it
+    deps:
+      - task: "init"
+    cmds:
+      - echo "This command is meant to deploy a Solo network to a Kind cluster on your local machine, "
+      - echo "ctrl-c if this is not what you want to do."
+      - sleep 5
+
+  install:
+    desc: create the cluster, solo init, solo cluster create, solo node keys, solo network deploy
+    deps:
+      - task: "init"
+    cmds:
+      - task: "cluster:create"
+      - task: "solo:init"
+      - task: "solo:cluster:setup"
+      - task: "solo:deployment:create"
+      - task: "solo:keys"
+      - task: "solo:network:deploy"
+      - task: "solo:node:setup"
+      - task: "solo:node:start"
+      - task: "solo:external-database"
+      - task: "solo:mirror-node"
+      - name: "Copy database-seeding-query.sql inside the database pod"
+        cmd: |
+          kubectl cp {{.HOME}}/.solo/cache/database-seeding-query.sql {{.postgres_container_name}}:/tmp/database-seeding-query.sql \
+          -n {{.postgres_database_namespace}}
+      - name: "Execute the database-seeding–query.sql against the database"
+        cmd: |
+          kubectl exec -it {{.postgres_container_name}} -n {{.postgres_database_namespace}} -- env PGPASSWORD={{.postgres_password}} psql -U {{.postgres_username}} \
+            -f /tmp/database-seeding-query.sql \
+            -d {{.postgres_mirror_node_database_name}}
+      - task: "solo:relay"
+      - name: "Run smoke test"
+        cmd: (cd ../../ && ./.github/workflows/script/solo_smoke_test.sh)
+  destroy:
+    desc: destroy relay, mirror-node, and network
+    deps:
+      - task: "init"
+    cmds:
+      - task: "cluster:destroy"