feat: production readiness pass custom mirror node postgresql credentials and server address

examples/custom-mirror-node-database/scripts/init.sh

@@ -0,0 +1,144 @@
+# Example if initialization script for the external database,
+# values must match those inside values.yaml if used
+
+cat > init1.sh << 'EOF'
+#!/bin/bash
+set -e
+
+export HEDERA_MIRROR_DATABASE_NAME="mirror_node"
+
+export HEDERA_MIRROR_GRAPHQL_DB_HOST="localhost"
+export HEDERA_MIRROR_GRAPHQL_DB_NAME="${HEDERA_MIRROR_DATABASE_NAME}"
+export HEDERA_MIRROR_GRAPHQL_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_GRAPHQL_DB_USERNAME="mirror_graphql"
+
+export HEDERA_MIRROR_GRPC_DB_HOST="localhost"
+export HEDERA_MIRROR_GRPC_DB_NAME="${HEDERA_MIRROR_DATABASE_NAME}"
+export HEDERA_MIRROR_GRPC_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_GRPC_DB_USERNAME="mirror_grpc"
+
+export HEDERA_MIRROR_IMPORTER_DB_HOST="localhost"
+export HEDERA_MIRROR_IMPORTER_DB_NAME="${HEDERA_MIRROR_DATABASE_NAME}"
+export HEDERA_MIRROR_IMPORTER_DB_OWNER="${HEDERA_MIRROR_DATABASE_NAME}"
+export HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_IMPORTER_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_IMPORTER_DB_RESTPASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_IMPORTER_DB_RESTUSERNAME="mirror_rest"
+export HEDERA_MIRROR_IMPORTER_DB_SCHEMA="public"
+export HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA="temporary"
+export HEDERA_MIRROR_IMPORTER_DB_USERNAME="mirror_importer"
+
+export HEDERA_MIRROR_RESTJAVA_DB_HOST="localhost"
+export HEDERA_MIRROR_RESTJAVA_DB_NAME="${HEDERA_MIRROR_DATABASE_NAME}"
+export HEDERA_MIRROR_RESTJAVA_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_RESTJAVA_DB_USERNAME="mirror_rest_java"
+
+export HEDERA_MIRROR_REST_DB_HOST="localhost"
+export HEDERA_MIRROR_REST_DB_NAME="${HEDERA_MIRROR_DATABASE_NAME}"
+export HEDERA_MIRROR_REST_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_REST_DB_USERNAME="mirror_rest"
+
+export HEDERA_MIRROR_ROSETTA_DB_HOST="localhost"
+export HEDERA_MIRROR_ROSETTA_DB_NAME="${HEDERA_MIRROR_DATABASE_NAME}"
+export HEDERA_MIRROR_ROSETTA_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_ROSETTA_DB_USERNAME="mirror_rosetta"
+
+export HEDERA_MIRROR_WEB3_DB_HOST="localhost"
+export HEDERA_MIRROR_WEB3_DB_NAME="${HEDERA_MIRROR_DATABASE_NAME}"
+export HEDERA_MIRROR_WEB3_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_WEB3_DB_USERNAME="mirror_web3"
+
+PGHBACONF="/opt/bitnami/postgresql/conf/pg_hba.conf"
+if [[ -f "${PGHBACONF}" ]]; then
+  cp "${PGHBACONF}" "${PGHBACONF}.bak"
+  echo "local all all trust" > "${PGHBACONF}"
+  pg_ctl reload
+fi
+
+psql -d "user=postgres connect_timeout=3" \
+  --set ON_ERROR_STOP=1 \
+  --set "dbName=${HEDERA_MIRROR_IMPORTER_DB_NAME}" \
+  --set "dbSchema=${HEDERA_MIRROR_IMPORTER_DB_SCHEMA}" \
+  --set "graphQLPassword=${HEDERA_MIRROR_GRAPHQL_DB_PASSWORD}" \
+  --set "graphQLUsername=${HEDERA_MIRROR_GRAPHQL_DB_USERNAME}" \
+  --set "grpcPassword=${HEDERA_MIRROR_GRPC_DB_PASSWORD}" \
+  --set "grpcUsername=${HEDERA_MIRROR_GRPC_DB_USERNAME}" \
+  --set "importerPassword=${HEDERA_MIRROR_IMPORTER_DB_PASSWORD}" \
+  --set "importerUsername=${HEDERA_MIRROR_IMPORTER_DB_USERNAME}" \
+  --set "ownerUsername=${HEDERA_MIRROR_IMPORTER_DB_OWNER}" \
+  --set "ownerPassword=${HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD}" \
+  --set "restPassword=${HEDERA_MIRROR_IMPORTER_DB_RESTPASSWORD}" \
+  --set "restUsername=${HEDERA_MIRROR_IMPORTER_DB_RESTUSERNAME}" \
+  --set "restJavaPassword=${HEDERA_MIRROR_RESTJAVA_DB_PASSWORD}" \
+  --set "restJavaUsername=${HEDERA_MIRROR_RESTJAVA_DB_USERNAME}" \
+  --set "rosettaPassword=${HEDERA_MIRROR_ROSETTA_DB_PASSWORD}" \
+  --set "rosettaUsername=${HEDERA_MIRROR_ROSETTA_DB_USERNAME}" \
+  --set "web3Password=${HEDERA_MIRROR_WEB3_DB_PASSWORD}" \
+  --set "web3Username=${HEDERA_MIRROR_WEB3_DB_USERNAME}" \
+  --set "tempSchema=${HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA}" <<__SQL__
+
+-- Create database & owner
+create user :ownerUsername with login password :'ownerPassword';
+create database :dbName with owner :ownerUsername;
+
+-- Create roles
+create role readonly;
+create role readwrite in role readonly;
+create role temporary_admin in role readwrite;
+
+-- Create users
+create user :graphQLUsername with login password :'graphQLPassword' in role readonly;
+create user :grpcUsername with login password :'grpcPassword' in role readonly;
+create user :importerUsername with login password :'importerPassword' in role readwrite admin :ownerUsername;
+create user :restJavaUsername with login password :'restJavaPassword' in role readonly;
+create user :rosettaUsername with login password :'rosettaPassword' in role readonly;
+create user :web3Username with login password :'web3Password' in role readonly;
+alter user :ownerUsername with createrole;
+
+-- Grant temp schema admin privileges
+grant temporary_admin to :ownerUsername;
+grant temporary_admin to :importerUsername;
+
+-- Add extensions
+\connect :dbName
+create extension if not exists btree_gist;
+create extension if not exists pg_stat_statements;
+create extension if not exists pg_trgm;
+
+-- Create schema
+\connect :dbName :ownerUsername
+create schema if not exists :dbSchema authorization :ownerUsername;
+grant usage on schema :dbSchema to public;
+revoke create on schema :dbSchema from public;
+
+-- Create temp table schema
+create schema if not exists :tempSchema authorization temporary_admin;
+grant usage on schema :tempSchema to public;
+revoke create on schema :tempSchema from public;
+
+-- Grant readonly privileges
+grant connect on database :dbName to readonly;
+grant select on all tables in schema :dbSchema, :tempSchema to readonly;
+grant select on all sequences in schema :dbSchema, :tempSchema to readonly;
+grant usage on schema :dbSchema, :tempSchema to readonly;
+alter default privileges in schema :dbSchema, :tempSchema grant select on tables to readonly;
+alter default privileges in schema :dbSchema, :tempSchema grant select on sequences to readonly;
+
+-- Grant readwrite privileges
+grant insert, update, delete on all tables in schema :dbSchema to readwrite;
+grant usage on all sequences in schema :dbSchema to readwrite;
+alter default privileges in schema :dbSchema grant insert, update, delete on tables to readwrite;
+alter default privileges in schema :dbSchema grant usage on sequences to readwrite;
+
+-- Alter search path
+\connect postgres postgres
+alter database :dbName set search_path = :dbSchema, public, :tempSchema;
+__SQL__
+
+if [[ -f "${PGHBACONF}.bak" ]]; then
+  mv "${PGHBACONF}.bak" "${PGHBACONF}"
+  pg_ctl reload
+fi
+EOF
+chmod +x init1.sh
+./init1.sh

examples/custom-mirror-node-database/values.yaml

@@ -0,0 +1,42 @@
+db:
+  host: "my-postgresql.mirror-db.svc.cluster.local"
+  name: mirror_node
+  enabled: true
+  owner:
+    username: postgres
+    password: "XXXXXXXXXXXX"
+
+stackgres:
+  enabled: false
+
+postgresql:
+  enabled: false
+
+importer:
+  enabled: true
+  db:
+    password: "XXXXXXXXXXXX"
+graphql:
+  enabled: true
+  db:
+    password: "XXXXXXXXXXXX"
+grpc:
+  enabled: true
+  db:
+    password: "XXXXXXXXXXXX"
+rest:
+  enabled: true
+  db:
+    password: "XXXXXXXXXXXX"
+restjava:
+  enabled: true
+  db:
+    password: "XXXXXXXXXXXX"
+web3:
+  enabled: true
+  db:
+    password: "XXXXXXXXXXXX"
+rosetta:
+  enabled: false
+  db:
+    password: "XXXXXXXXXXXX"

src/commands/flags.ts

@@ -1516,6 +1516,17 @@ export class Flags {
     },
   };
 
+  static readonly customMirrorNodeDatabaseValuePath: CommandFlag = {
+    constName: 'customMirrorNodeDatabaseValuePath',
+    name: 'custom-mirror-node-database-values-path',
+    definition: {
+      describe: 'Path to custom mirror node database values',
+      defaultValue: '',
+      type: 'string',
+    },
+    prompt: undefined,
+  };
+
   static readonly grpcTlsKeyPath: CommandFlag = {
     constName: 'grpcTlsKeyPath',
     name: 'grpc-tls-key',
@@ -1799,6 +1810,7 @@ export class Flags {
     Flags.updateAccountKeys,
     Flags.userEmailAddress,
     Flags.valuesFile,
+    Flags.customMirrorNodeDatabaseValuePath,
   ];
 
   /** Resets the definition.disablePrompt for all flags */

src/commands/mirror_node.ts

@@ -29,7 +29,9 @@
 import {ListrLease} from '../core/lease/listr_lease.js';
 import {ComponentType} from '../core/config/remote/enumerations.js';
 import {MirrorNodeComponent} from '../core/config/remote/components/mirror_node_component.js';
-import type {SoloListrTask} from '../types/index.js';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import type {Optional, SoloListrTask} from '../types/index.js';
 import type {Namespace} from '../core/config/remote/types.js';
 
 interface MirrorNodeDeployConfigClass {
@@ -51,6 +53,7 @@
   clusterSetupNamespace: string;
   soloChartVersion: string;
   pinger: boolean;
+  customMirrorNodeDatabaseValuePath: Optional<string>;
   storageType: constants.StorageType;
   storageAccessKey: string;
   storageSecrets: string;
@@ -100,6 +103,7 @@
       flags.pinger,
       flags.clusterSetupNamespace,
       flags.soloChartVersion,
+      flags.customMirrorNodeDatabaseValuePath,
       flags.storageType,
       flags.storageAccessKey,
       flags.storageSecrets,
@@ -285,6 +289,20 @@
                 {
                   title: 'Deploy mirror-node',
                   task: async ctx => {
+                    if (ctx.config.customMirrorNodeDatabaseValuePath) {
+                      if (!fs.existsSync(ctx.config.customMirrorNodeDatabaseValuePath)) {
+                        throw new SoloError('Path provided for custom mirror node database value is not found');
+                      }
+
+                      // Check if the file has a .yaml or .yml extension
+                      const fileExtension = path.extname(ctx.config.customMirrorNodeDatabaseValuePath);
+                      if (fileExtension !== '.yaml' && fileExtension !== '.yml') {
+                        throw new SoloError('The provided file is not a valid YAML file (.yaml or .yml)');
+                      }
+
+                      ctx.config.valuesArg += ` --values ${ctx.config.customMirrorNodeDatabaseValuePath}`;
+                    }
+
                     await self.chartManager.install(
                       ctx.config.namespace,
                       constants.MIRROR_NODE_RELEASE_NAME,
@@ -357,6 +375,7 @@
                       constants.PODS_READY_MAX_ATTEMPTS,
                       constants.PODS_READY_DELAY,
                     ),
+                  skip: ctx => !!ctx.config.customMirrorNodeDatabaseValuePath,
                 },
                 {
                   title: 'Check REST API',
@@ -424,7 +443,7 @@
               [
                 {
                   title: 'Insert data in public.file_data',
-                  task: async () => {
+                  task: async ctx => {
                     const namespace = self.configManager.getFlag<string>(flags.namespace) as string;
 
                     const feesFileIdNum = 111;
@@ -440,6 +459,11 @@
                     }, ${exchangeRatesFileIdNum}, 17);`;
                     const sqlQuery = [importFeesQuery, importExchangeRatesQuery].join('\n');
 
+                    if (ctx.config.customMirrorNodeDatabaseValuePath) {
+                      fs.writeFileSync(path.join(constants.SOLO_CACHE_DIR, 'database-seeding-query.sql'), sqlQuery);
+                      return;
+                    }
+
                     const pods = await this.k8.getPodsByLabel(['app.kubernetes.io/name=postgres']);
                     if (pods.length === 0) {
                       throw new SoloError('postgres pod not found');