fix: copy all tables on blue-green

Author: 0xnigir1

scripts/migrations/README-DB-BLUE-GREEN.md

@@ -7,7 +7,7 @@ This utility provides scripts for managing PostgreSQL databases in a blue-green
 The scripts offer two main operations:
 
 1. **Database Creation** - Create both blue and green databases if they don't exist
-2. **Cache Data Copy** - Copy external services cache data from one database to the other
+2. **Database Copy** - Create an exact copy of all tables and data from one database to the other
 
 ## Database Creation
 
@@ -25,35 +25,48 @@ pnpm db:create-databases
 3. If either database doesn't exist, it creates it
 4. This operation can be run multiple times without error, as it only creates databases when needed
 
-## Cache Data Copy
+## Database Copy
 
-Use this operation to copy cache data between blue and green databases. This is essential for maintaining cache consistency during blue-green deployments.
+Use this operation to create an exact copy of one database to the other. This is essential for maintaining consistency during blue-green deployments.
 
 ```bash
-# Copy cache data from blue to green
+# Copy all tables and data from blue to green
 pnpm db:copy-cache --copyFrom=blue
 
-# Copy cache data from green to blue
+# Copy all tables and data from green to blue
 pnpm db:copy-cache --copyFrom=green
 
 # Using the shorthand parameter
 pnpm db:copy-cache -f blue
 ```
 
-### How Cache Data Copy Works
+### How Database Copy Works
 
 1. The script reads the PostgreSQL connection details from the `DATABASE_URL` environment variable
-2. It handles the two specific cache tables: `price_cache` and `metadata_cache`
-3. For each table:
-    - It truncates the target table
-    - Copies all data from the source to the target table
+2. It determines the source and target databases based on the `copyFrom` parameter
+3. It retrieves a list of all tables in the source database
+4. For each table:
+    - It truncates the target table (removing all existing data)
+    - Copies all data from the source table to the target table
     - Processes data in batches to avoid memory issues with large tables
+5. After completion, the target database is an exact copy of the source database
 
-### Cache Tables
+## Blue-Green Deployment Process
 
-The script only copies the following tables, which contain cached data from external services:
+Here's a typical workflow for using this utility in a blue-green deployment:
 
--   `price_cache`: Stores token price information
--   `metadata_cache`: Stores token metadata
+```bash
+# Step 1: Ensure both databases exist
+pnpm db:create-databases
+
+# Step 2: Deploy new version to the inactive environment (e.g., green)
+# (Your deployment steps here)
+
+# Step 3: Copy data from the active environment to the inactive one
+pnpm db:copy-cache --copyFrom=blue
+
+# Step 4: Switch traffic to the newly updated environment
+# (Your traffic switching steps here)
+```
 
-All other tables are managed through the regular migration process and are not part of the blue-green deployment cache copying strategy.
+This process allows for zero-downtime deployments by maintaining two parallel database environments.

scripts/migrations/src/copyCache.script.ts

@@ -5,26 +5,20 @@ import { hideBin } from "yargs/helpers";
 
 import { Logger, stringify } from "@grants-stack-indexer/shared";
 
-import {
-    BLUE_DB,
-    CACHE_TABLES,
-    ConnectionDetails,
-    extractConnectionDetails,
-    GREEN_DB,
-} from "./constants.js";
+import { BLUE_DB, ConnectionDetails, extractConnectionDetails, GREEN_DB } from "./constants.js";
 import { getDatabaseConfigFromEnv } from "./schemas/index.js";
 
-const { Pool } = pg;
-
 configDotenv();
 
+const { Pool } = pg;
+
 /**
- * This script copies cache data between blue and green databases.
+ * This script copies all table data between blue and green databases.
  *
  * It performs the following steps:
  * 1. Loads environment variables from .env file
  * 2. Gets database configuration from environment
- * 3. Copies cache tables from source to target database
+ * 3. Copies all tables from source to target database, resetting destination tables first
  *
  * Environment variables required:
  * - DATABASE_URL: PostgreSQL connection string (used to extract host, port, user, password)
@@ -43,6 +37,10 @@ interface CopyCacheCommandArgs {
 }
 
 // Define interfaces for database query results
+interface TableNameRow {
+    table_name: string;
+}
+
 interface ColumnNameRow {
     column_name: string;
 }
@@ -64,12 +62,57 @@ const parseArguments = (): CopyCacheCommandArgs => {
         .parseSync() as CopyCacheCommandArgs;
 };
 
+/**
+ * Get all tables in the public schema
+ */
+export const getAllTables = async (
+    db: string,
+    connectionDetails: ConnectionDetails,
+): Promise<string[]> => {
+    const logger = Logger.getInstance();
+    const { host, port, user, password } = connectionDetails;
+
+    const pool = new Pool({
+        host,
+        port: parseInt(port, 10),
+        user,
+        password,
+        database: db,
+        ssl:
+            process.env.NODE_ENV === "production"
+                ? {
+                      rejectUnauthorized: false,
+                  }
+                : undefined,
+        connectionTimeoutMillis: 15000,
+        idleTimeoutMillis: 10000,
+        max: 5,
+    });
+
+    try {
+        logger.info(`Getting all tables from database '${db}'...`);
+
+        const result = await pool.query<TableNameRow>(`
+            SELECT table_name 
+            FROM information_schema.tables 
+            WHERE table_schema = 'public' 
+            AND table_type = 'BASE TABLE'
+            ORDER BY table_name
+        `);
+
+        const tables = result.rows.map((row) => row.table_name);
+        logger.info(`Found ${tables.length} tables in database '${db}'`);
+        return tables;
+    } catch (error) {
+        logger.error(`Failed to get tables: ${stringify(error)}`);
+        throw error;
+    } finally {
+        await pool.end();
+    }
+};
+
 /**
  * Copy table data between databases
- * @param sourceDb - The source database name
- * @param targetDb - The target database name
- * @param tableName - The table name to copy
- * @param connectionDetails - The connection details for the source and target databases
  */
 export const copyTableData = async (
     sourceDb: string,
@@ -145,7 +188,7 @@ export const copyTableData = async (
         const dataResult = await sourcePool.query<DatabaseRow>(`SELECT * FROM "${tableName}"`);
 
         if (dataResult.rows.length === 0) {
-            logger.info(`No data in source table '${tableName}'. Skipping.`);
+            logger.info(`No data in source table '${tableName}'. Skipping insert.`);
             return;
         }
 
@@ -196,37 +239,36 @@ export const copyTableData = async (
 };
 
 /**
- * Copy cache data from one database to another
- * @param sourceDb - The source database name
- * @param targetDb - The target database name
- * @param connectionDetails - The connection details for the source and target databases
+ * Copy all tables data from one database to another
  */
-export const copyCacheData = async (
+export const copyAllTableData = async (
     sourceDb: string,
     targetDb: string,
     connectionDetails: ConnectionDetails,
 ): Promise<void> => {
     const logger = Logger.getInstance();
 
     try {
-        logger.info(`Copying cache data from '${sourceDb}' to '${targetDb}'...`);
+        logger.info(`Copying all table data from '${sourceDb}' to '${targetDb}'...`);
 
-        // Log which cache tables we'll copy
-        logger.info(`Using ${CACHE_TABLES.length} cache tables: ${CACHE_TABLES.join(", ")}`);
+        // Get all tables from source database
+        const tables = await getAllTables(sourceDb, connectionDetails);
 
-        if (CACHE_TABLES.length === 0) {
-            logger.warn("No cache tables defined. Nothing to copy.");
+        if (tables.length === 0) {
+            logger.warn("No tables found in source database. Nothing to copy.");
             return;
         }
 
+        logger.info(`Found ${tables.length} tables to copy from source database`);
+
         // Copy each table
-        for (const table of CACHE_TABLES) {
+        for (const table of tables) {
             await copyTableData(sourceDb, targetDb, table, connectionDetails);
         }
 
-        logger.info(`✅ Successfully copied cache data from '${sourceDb}' to '${targetDb}'`);
+        logger.info(`✅ Successfully copied all table data from '${sourceDb}' to '${targetDb}'`);
     } catch (error) {
-        logger.error(`Failed to copy cache data: ${stringify(error)}`);
+        logger.error(`Failed to copy table data: ${stringify(error)}`);
         throw error;
     }
 };
@@ -244,10 +286,10 @@ export const main = async (): Promise<void> => {
     const sourceDb = sourceColor === "blue" ? BLUE_DB : GREEN_DB;
     const targetDb = sourceColor === "blue" ? GREEN_DB : BLUE_DB;
 
-    logger.info(`Copying cache data from ${sourceColor} to ${targetColor}...`);
-    await copyCacheData(sourceDb, targetDb, connectionDetails);
+    logger.info(`Copying all table data from ${sourceColor} to ${targetColor}...`);
+    await copyAllTableData(sourceDb, targetDb, connectionDetails);
 
-    logger.info(`✅ Cache data copied from ${sourceColor} to ${targetColor} successfully`);
+    logger.info(`✅ Database ${targetColor} is now an exact copy of ${sourceColor}`);
 
     process.exit(0);
 };