[VOQ][saidump] Enhance saidump with new option -r to parser the JSON file and displays/format the right output (#1288)

Why I did it
Fix issue: sonic-net/sonic-buildimage#13561
The existing saidump use https://github.com/sonic-net/sonic-swss-common/blob/master/common/table_dump.lua script which loops the ASIC_DB more than 5 seconds and blocks other processes access.

This solution uses the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table.
Related PRs:
sonic-net/sonic-utilities#2972
sonic-net/sonic-buildimage#16466

Author: JunhongMao

saidump/saidump.cpp

@@ -2,6 +2,10 @@
 #include <string>
 #include <set>
 #include <sstream>
+#include <iostream>
+#include <fstream>
+#include <regex>
+#include <climits>
 
 extern "C" {
 #include <sai.h>
@@ -10,32 +14,44 @@ extern "C" {
 #include "swss/table.h"
 #include "meta/sai_serialize.h"
 #include "sairediscommon.h"
+#include <nlohmann/json.hpp>
 
 #include <getopt.h>
 
 // TODO split to multiple cpp
 
 using namespace swss;
+using json = nlohmann::json;
+
+// Default value: 100 MB
+constexpr int64_t RDB_JSON_MAX_SIZE = 1024 * 1024 * 100;
 
 struct CmdOptions
 {
     bool skipAttributes;
     bool dumpTempView;
     bool dumpGraph;
+    std::string rdbJsonFile;
+    uint64_t rdbJSonSizeLimit;
 };
 
-CmdOptions g_cmdOptions;
-std::map<sai_object_id_t, const TableMap*> g_oid_map;
+
+static CmdOptions g_cmdOptions;
+static std::map<sai_object_id_t, const TableMap*> g_oid_map;
 
 void printUsage()
 {
     SWSS_LOG_ENTER();
 
-    std::cout << "Usage: saidump [-t] [-g] [-h]" << std::endl;
+    std::cout << "Usage: saidump [-t] [-g] [-r] [-m] [-h]" << std::endl;
     std::cout << "    -t --tempView:" << std::endl;
     std::cout << "        Dump temp view" << std::endl;
     std::cout << "    -g --dumpGraph:" << std::endl;
     std::cout << "        Dump current graph" << std::endl;
+    std::cout << "    -r --rdb:" << std::endl;
+    std::cout << "        Dump by parsing the RDB JSON file, which is created by rdbtools based on Redis dump.rdb that is generated by Redis SAVE command" << std::endl;
+    std::cout << "    -m --max:" << std::endl;
+    std::cout << "        Config the the RDB JSON file's max size in MB, which is optional with default value 100MB" << std::endl;
     std::cout << "    -h --help:" << std::endl;
     std::cout << "        Print out this message" << std::endl;
 }
@@ -48,15 +64,19 @@ CmdOptions handleCmdLine(int argc, char **argv)
 
     options.dumpTempView = false;
     options.dumpGraph = false;
+    options.rdbJSonSizeLimit = RDB_JSON_MAX_SIZE;
 
-    const char* const optstring = "gth";
+    const char* const optstring = "gtr:m:h";
+    uint64_t result = 0;
 
     while (true)
     {
         static struct option long_options[] =
         {
             { "dumpGraph",      no_argument,       0, 'g' },
             { "tempView",       no_argument,       0, 't' },
+            { "rdb",            required_argument, 0, 'r' },
+            { "max",            required_argument, 0, 'm' },
             { "help",           no_argument,       0, 'h' },
             { 0,                0,                 0,  0  }
         };
@@ -82,6 +102,33 @@ CmdOptions handleCmdLine(int argc, char **argv)
                 options.dumpTempView = true;
                 break;
 
+            case 'r':
+                SWSS_LOG_NOTICE("Dumping from %s", optarg);
+                options.rdbJsonFile = std::string(optarg);
+                break;
+
+            case 'm':
+                if(!regex_match(optarg, std::regex(R"([+]?\d+)")))              //only positive numeric chars are valid, such as 3984, +3232, etc.
+                {
+                    SWSS_LOG_WARN("invalid option -m %s", optarg);
+                    printUsage();
+                    exit(EXIT_SUCCESS);
+                }
+
+                result = strtoull(optarg, NULL, 0);
+
+                if((errno == ERANGE && result == ULLONG_MAX) || result == 0 || result >= INT_MAX)
+                {
+                    SWSS_LOG_WARN("invalid option -m %s", optarg);
+                    printUsage();
+                    exit(EXIT_SUCCESS);
+                }
+
+                options.rdbJSonSizeLimit = result * 1024 * 1024;
+                SWSS_LOG_NOTICE("Configure the RDB JSON MAX size to %llu MB", options.rdbJSonSizeLimit / 1024 / 1024);
+
+                break;
+
             case 'h':
                 printUsage();
                 exit(EXIT_SUCCESS);
@@ -399,7 +446,153 @@ void dumpGraph(const TableDump& td)
     std::cout << "}" << std::endl;
 }
 
-int main(int argc, char ** argv)
+#define SWSS_LOG_ERROR_AND_STDERR(format, ...) { fprintf(stderr, format"\n", ##__VA_ARGS__); SWSS_LOG_ERROR(format, ##__VA_ARGS__); }
+
+/**
+ * @brief Process the input JSON file to make sure it's a valid JSON file for the JSON library.
+ */
+static sai_status_t preProcessFile(const std::string file_name)
+{
+    SWSS_LOG_ENTER();
+
+    std::ifstream input_file(file_name);
+
+    if (!input_file.is_open())
+    {
+        SWSS_LOG_ERROR_AND_STDERR("Failed to open the input file %s.", file_name.c_str());
+        return SAI_STATUS_FAILURE;
+    }
+
+    input_file.seekg(0, std::ios::end);     // Move to the end of the file
+    uint64_t file_size = input_file.tellg(); // Get the current position
+    SWSS_LOG_NOTICE("Get %s's size %" PRIu64 " Bytes, limit: %" PRIu64 " MB.", file_name.c_str(), file_size, g_cmdOptions.rdbJSonSizeLimit / 1024 / 1024);
+
+    if (file_size >= g_cmdOptions.rdbJSonSizeLimit)
+    {
+        SWSS_LOG_ERROR_AND_STDERR("Get %s's size failure or its size %" PRIu64 " >= %" PRIu64 " MB.", file_name.c_str(), file_size, g_cmdOptions.rdbJSonSizeLimit / 1024 / 1024);
+        return SAI_STATUS_FAILURE;
+    }
+
+    input_file.seekg(0);     // Move to the begin of the file
+
+    // Read the content of the input file into a string
+    std::string content((std::istreambuf_iterator<char>(input_file)),
+                   std::istreambuf_iterator<char>());
+
+    content = regex_replace(content, std::regex("\\},\\{\\r"), ",");
+
+    std::ofstream outputFile(file_name);
+
+    if (!outputFile.is_open())
+    {
+        SWSS_LOG_ERROR_AND_STDERR("Failed to open the output file %s.", file_name.c_str());
+        return SAI_STATUS_FAILURE;
+    }
+
+    //Remove the 1st and last char to make sure its format is same as previous output
+    if (content.size() >= 2 && content[0] == '[' && content[content.length()-1] == ']')
+    {
+        outputFile << content.substr(1, content.size()-2);
+    }
+    else
+    {
+        outputFile << content;
+    }
+
+    return SAI_STATUS_SUCCESS;
+}
+
+static sai_status_t dumpFromRedisRdbJson(const std::string file_name)
+{
+    SWSS_LOG_ENTER();
+
+    std::ifstream input_file(file_name);
+
+    if (!input_file.is_open())
+    {
+        SWSS_LOG_ERROR_AND_STDERR("The file %s does not exist for dumping from Redis RDB JSON file.", file_name.c_str());
+        return SAI_STATUS_FAILURE;
+    }
+
+    try
+    {
+        // Parse the JSON data from the file (validation)
+        nlohmann::json jsonData;
+        input_file >> jsonData;
+
+        SWSS_LOG_DEBUG("JSON file is valid.");
+
+        for (json::iterator it = jsonData.begin(); it != jsonData.end(); ++it)
+        {
+            json jj_key = it.key();
+
+            std::string keystr = jj_key;
+            std::string item_name = keystr;
+            size_t pos = keystr.find_first_of(":");
+
+            if (pos != std::string::npos)
+            {
+                if(ASIC_STATE_TABLE != keystr.substr(0, pos))  // filter out non ASIC_STATE
+                {
+                    continue;
+                }
+
+                item_name = keystr.substr(pos + 1);
+
+                if (item_name.find(":") != std::string::npos)
+                {
+                    item_name.replace(item_name.find_first_of(":"), 1, " ");
+                }
+            }
+            else
+            {
+                continue;
+            }
+
+            std::cout << item_name << " " << std::endl;
+
+            json jj = it.value();
+
+            if (!it->is_object())
+            {
+                continue;
+            }
+
+            TableMap map;
+
+            for (json::iterator itt = jj.begin(); itt != jj.end(); ++itt)
+            {
+                if (itt.key() != "NULL")
+                {
+                    map[itt.key()] = itt.value();
+                }
+            }
+
+            constexpr size_t LINE_IDENT = 4;
+            size_t max_len = get_max_attr_len(map);
+            std::string str_indent = pad_string("", LINE_IDENT);
+
+            for (const auto&field: map)
+            {
+                std::cout << str_indent << pad_string(field.first, max_len) << " : ";
+                std::cout << field.second << std::endl;
+            }
+
+            std::cout << std::endl;
+        }
+
+        return SAI_STATUS_SUCCESS;
+    }
+    catch (std::exception &ex)
+    {
+        SWSS_LOG_ERROR_AND_STDERR("JSON file %s is invalid.", file_name.c_str());
+        SWSS_LOG_ERROR_AND_STDERR("JSON parsing error: %s.", ex.what());
+    }
+
+    return SAI_STATUS_FAILURE;
+}
+
+int main(int argc, char **argv)
 {
     swss::Logger::getInstance().setMinPrio(swss::Logger::SWSS_DEBUG);
 
@@ -411,6 +604,17 @@ int main(int argc, char ** argv)
 
     g_cmdOptions = handleCmdLine(argc, argv);
 
+
+    if (g_cmdOptions.rdbJsonFile.size() > 0)
+    {
+        if (SAI_STATUS_FAILURE == preProcessFile(g_cmdOptions.rdbJsonFile))
+        {
+            return EXIT_FAILURE;
+        }
+
+        return dumpFromRedisRdbJson(g_cmdOptions.rdbJsonFile);
+    }
+
     swss::DBConnector db("ASIC_DB", 0);
 
     std::string table = ASIC_STATE_TABLE;