From 42d9913e155439561fbb0b1561a55d6aef598787 Mon Sep 17 00:00:00 2001 From: Judy Joseph Date: Wed, 8 Jul 2020 16:54:56 -0700 Subject: [PATCH 01/11] DBConnector classes to understand the namespace. --- common/dbconnector.cpp | 211 +++++++++++++++++++++++++++++++---------- common/dbconnector.h | 50 +++++++--- 2 files changed, 198 insertions(+), 63 deletions(-) diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index dc080d9b2..d497a3b26 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -16,17 +16,11 @@ using namespace std; namespace swss { -void SonicDBConfig::initialize(const string &file) +void SonicDBConfig::parse_config(const string &file, + std::unordered_map &inst_entry, + std::unordered_map &db_entry, + std::unordered_map &separator_entry) { - - SWSS_LOG_ENTER(); - - if (m_init) - { - SWSS_LOG_ERROR("SonicDBConfig already initialized"); - throw runtime_error("SonicDBConfig already initialized"); - } - ifstream i(file); if (i.good()) { @@ -40,7 +34,7 @@ void SonicDBConfig::initialize(const string &file) string socket = it.value().at("unix_socket_path"); string hostname = it.value().at("hostname"); int port = it.value().at("port"); - m_inst_info[instName] = {socket, {hostname, port}}; + inst_entry[instName] = {socket, hostname, port}; } for (auto it = j["DATABASES"].begin(); it!= j["DATABASES"].end(); it++) @@ -49,12 +43,12 @@ void SonicDBConfig::initialize(const string &file) string instName = it.value().at("instance"); int dbId = it.value().at("id"); string separator = it.value().at("separator"); - m_db_info[dbName] = {instName, dbId, separator}; + db_entry[dbName] = {instName, dbId, separator}; - m_db_separator.emplace(dbId, separator); + separator_entry.emplace(dbId, separator); } - m_init = true; } + catch (domain_error& e) { SWSS_LOG_ERROR("key doesn't exist in json object, NULL value has no iterator >> %s\n", e.what()); @@ -73,32 +67,126 @@ void SonicDBConfig::initialize(const string &file) } } -string SonicDBConfig::getDbInst(const string &dbName) +void SonicDBConfig::initialize_global_config(const string &file) +{ + std::string local_file, ns_name; + std::unordered_map db_entry; + std::unordered_map inst_entry; + std::unordered_map separator_entry; + + SWSS_LOG_ENTER(); + + if (m_global_init) + { + SWSS_LOG_ERROR("SonicDBConfig Global config is already initialized"); + return; + } + + ifstream i(file); + if (i.good()) + { + try + { + json j; + i >> j; + + for (auto& element : j["INCLUDES"]) + { + local_file.append(DEFAULT_SONIC_DB_CONFIG_DIR); + local_file.append(element["include"]); + + if(element["namespace"].empty()) + { + ns_name = ""; + } + else + { + ns_name = element["namespace"]; + } + + parse_config(local_file, inst_entry, db_entry, separator_entry); + m_inst_info[ns_name] = inst_entry; + m_db_info[ns_name] = db_entry; + m_db_separator[ns_name] = separator_entry; + + inst_entry.clear(); + db_entry.clear(); + separator_entry.clear(); + local_file.clear(); + } + } + + catch (domain_error& e) + { + SWSS_LOG_ERROR("key doesn't exist in json object, NULL value has no iterator >> %s\n", e.what()); + throw runtime_error("key doesn't exist in json object, NULL value has no iterator >> " + string(e.what())); + } + catch (exception &e) + { + SWSS_LOG_ERROR("Sonic database config file syntax error >> %s\n", e.what()); + throw runtime_error("Sonic database config file syntax error >> " + string(e.what())); + } + } + + // Set it as the global config file is already parsed and init done. + m_global_init = true; +} + +void SonicDBConfig::initialize(const string &file, const string &nameSpace) +{ + std::unordered_map db_entry; + std::unordered_map inst_entry; + std::unordered_map separator_entry; + + SWSS_LOG_ENTER(); + + if (m_init) + { + SWSS_LOG_ERROR("SonicDBConfig already initialized"); + throw runtime_error("SonicDBConfig already initialized"); + } + + if(nameSpace.empty()) + { + parse_config(file, inst_entry, db_entry, separator_entry); + m_inst_info[""] = inst_entry; + m_db_info[""] = db_entry; + m_db_separator[""] = separator_entry; + } + else + initialize_global_config(); + + + // Set it as the config file is already parsed and init done. + m_init = true; +} + +string SonicDBConfig::getDbInst(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(); - return m_db_info.at(dbName).instName; + initialize(nameSpace); + return m_db_info[nameSpace].at(dbName).instName; } -int SonicDBConfig::getDbId(const string &dbName) +int SonicDBConfig::getDbId(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(); - return m_db_info.at(dbName).dbId; + initialize(nameSpace); + return m_db_info[nameSpace].at(dbName).dbId; } -string SonicDBConfig::getSeparator(const string &dbName) +string SonicDBConfig::getSeparator(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(); - return m_db_info.at(dbName).separator; + initialize(nameSpace); + return m_db_info[nameSpace].at(dbName).separator; } -string SonicDBConfig::getSeparator(int dbId) +string SonicDBConfig::getSeparator(int dbId, const string &nameSpace) { if (!m_init) - initialize(); - return m_db_separator.at(dbId); + initialize(nameSpace); + return m_db_separator[nameSpace].at(dbId); } string SonicDBConfig::getSeparator(const DBConnector* db) @@ -109,42 +197,61 @@ string SonicDBConfig::getSeparator(const DBConnector* db) } string dbName = db->getDbName(); + string nameSpace = db->getNamespace(); if (dbName.empty()) { - return getSeparator(db->getDbId()); + return getSeparator(db->getDbId(), nameSpace); } else { - return getSeparator(dbName); + return getSeparator(dbName, nameSpace); } } -string SonicDBConfig::getDbSock(const string &dbName) +string SonicDBConfig::getDbSock(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(); - return m_inst_info.at(getDbInst(dbName)).first; + initialize(nameSpace); + return m_inst_info[nameSpace].at(getDbInst(dbName)).socket; } -string SonicDBConfig::getDbHostname(const string &dbName) +string SonicDBConfig::getDbHostname(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(); - return m_inst_info.at(getDbInst(dbName)).second.first; + initialize(nameSpace); + return m_inst_info[nameSpace].at(getDbInst(dbName)).hostName; } -int SonicDBConfig::getDbPort(const string &dbName) +int SonicDBConfig::getDbPort(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(); - return m_inst_info.at(getDbInst(dbName)).second.second; + initialize(nameSpace); + return m_inst_info[nameSpace].at(getDbInst(dbName)).port; +} + +vector SonicDBConfig::getNamespaces() +{ + vector list; + + if (!m_global_init) + initialize_global_config(); + + for (auto it = m_inst_info.cbegin(); it != m_inst_info.cend(); ++it) { + if(!((it->first).empty())) + list.push_back(it->first); + } + + return list; } constexpr const char *SonicDBConfig::DEFAULT_SONIC_DB_CONFIG_FILE; -unordered_map>> SonicDBConfig::m_inst_info; -unordered_map SonicDBConfig::m_db_info; -unordered_map SonicDBConfig::m_db_separator; +constexpr const char *SonicDBConfig::DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE; +constexpr const char *SonicDBConfig::DEFAULT_SONIC_DB_CONFIG_DIR; +unordered_map> SonicDBConfig::m_inst_info; +unordered_map> SonicDBConfig::m_db_info; +unordered_map> SonicDBConfig::m_db_separator; bool SonicDBConfig::m_init = false; +bool SonicDBConfig::m_global_init = false; constexpr const char *DBConnector::DEFAULT_UNIXSOCKET; @@ -164,7 +271,8 @@ DBConnector::~DBConnector() DBConnector::DBConnector(int dbId, const string& hostname, int port, unsigned int timeout) : - m_dbId(dbId) + m_dbId(dbId), + m_nameSpace("") { struct timeval tv = {0, (suseconds_t)timeout * 1000}; @@ -181,7 +289,8 @@ DBConnector::DBConnector(int dbId, const string& hostname, int port, } DBConnector::DBConnector(int dbId, const string& unixPath, unsigned int timeout) : - m_dbId(dbId) + m_dbId(dbId), + m_nameSpace("") { struct timeval tv = {0, (suseconds_t)timeout * 1000}; @@ -197,25 +306,26 @@ DBConnector::DBConnector(int dbId, const string& unixPath, unsigned int timeout) select(this); } -DBConnector::DBConnector(const string& dbName, unsigned int timeout, bool isTcpConn) - : m_dbId(SonicDBConfig::getDbId(dbName)) +DBConnector::DBConnector(const string& dbName, unsigned int timeout, bool isTcpConn, const string& nameSpace) + : m_dbId(SonicDBConfig::getDbId(dbName, nameSpace)) , m_dbName(dbName) + , m_nameSpace(nameSpace) { struct timeval tv = {0, (suseconds_t)timeout * 1000}; if (timeout) { if (isTcpConn) - m_conn = redisConnectWithTimeout(SonicDBConfig::getDbHostname(dbName).c_str(), SonicDBConfig::getDbPort(dbName), tv); + m_conn = redisConnectWithTimeout(SonicDBConfig::getDbHostname(dbName, nameSpace).c_str(), SonicDBConfig::getDbPort(dbName, nameSpace), tv); else - m_conn = redisConnectUnixWithTimeout(SonicDBConfig::getDbSock(dbName).c_str(), tv); + m_conn = redisConnectUnixWithTimeout(SonicDBConfig::getDbSock(dbName, nameSpace).c_str(), tv); } else { if (isTcpConn) - m_conn = redisConnect(SonicDBConfig::getDbHostname(dbName).c_str(), SonicDBConfig::getDbPort(dbName)); + m_conn = redisConnect(SonicDBConfig::getDbHostname(dbName, nameSpace).c_str(), SonicDBConfig::getDbPort(dbName, nameSpace)); else - m_conn = redisConnectUnix(SonicDBConfig::getDbSock(dbName).c_str()); + m_conn = redisConnectUnix(SonicDBConfig::getDbSock(dbName, nameSpace).c_str()); } if (m_conn->err) @@ -240,6 +350,11 @@ string DBConnector::getDbName() const return m_dbName; } +string DBConnector::getNamespace() const +{ + return m_nameSpace; +} + DBConnector *DBConnector::newConnector(unsigned int timeout) const { DBConnector *ret; diff --git a/common/dbconnector.h b/common/dbconnector.h index 62fbbde96..928b95e33 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -12,6 +12,14 @@ namespace swss { class DBConnector; +class SonicInstInfo +{ +public: + std::string socket; + std::string hostName; + int port; +}; + class SonicDBInfo { public: @@ -23,26 +31,36 @@ class SonicDBInfo class SonicDBConfig { public: - static void initialize(const std::string &file = DEFAULT_SONIC_DB_CONFIG_FILE); - static std::string getDbInst(const std::string &dbName); - static int getDbId(const std::string &dbName); - static std::string getSeparator(const std::string &dbName); - static std::string getSeparator(int dbId); + static void parse_config(const std::string &file, + std::unordered_map &inst_entry, + std::unordered_map &db_entry, + std::unordered_map &separator_entry); + static void initialize(const std::string &file = DEFAULT_SONIC_DB_CONFIG_FILE, const std::string &nameSpace = ""); + static void initialize_global_config(const std::string &file = DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE); + static std::string getDbInst(const std::string &dbName, const std::string &nameSpace = ""); + static int getDbId(const std::string &dbName, const std::string &nameSpace = ""); + static std::string getSeparator(const std::string &dbName, const std::string &nameSpace = ""); + static std::string getSeparator(int dbId, const std::string &nameSpace = ""); static std::string getSeparator(const DBConnector* db); - static std::string getDbSock(const std::string &dbName); - static std::string getDbHostname(const std::string &dbName); - static int getDbPort(const std::string &dbName); + static std::string getDbSock(const std::string &dbName, const std::string &nameSpace = ""); + static std::string getDbHostname(const std::string &dbName, const std::string &nameSpace = ""); + static int getDbPort(const std::string &dbName, const std::string &nameSpace = ""); + static std::vector getNamespaces(); static bool isInit() { return m_init; }; + static bool isGlobalInit() { return m_global_init; }; private: + static constexpr const char *DEFAULT_SONIC_DB_CONFIG_DIR = "/var/run/redis/sonic-db/"; static constexpr const char *DEFAULT_SONIC_DB_CONFIG_FILE = "/var/run/redis/sonic-db/database_config.json"; - // { instName, { unix_socket_path, {hostname, port} } } - static std::unordered_map>> m_inst_info; - // { dbName, {instName, dbId} } - static std::unordered_map m_db_info; - // { dbIp, separator } - static std::unordered_map m_db_separator; + static constexpr const char *DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE = "/var/run/redis/sonic-db/database_global.json"; + // { namespace { instName, { unix_socket_path, hostname, port } } } + static std::unordered_map> m_inst_info; + // { namespace, { dbName, {instName, dbId} } } + static std::unordered_map> m_db_info; + // { namespace, { dbIp, separator } } + static std::unordered_map> m_db_separator; static bool m_init; + static bool m_global_init; }; class DBConnector @@ -59,13 +77,14 @@ class DBConnector */ DBConnector(int dbId, const std::string &hostname, int port, unsigned int timeout); DBConnector(int dbId, const std::string &unixPath, unsigned int timeout); - DBConnector(const std::string &dbName, unsigned int timeout, bool isTcpConn = false); + DBConnector(const std::string &dbName, unsigned int timeout, bool isTcpConn = false, const std::string &nameSpace = ""); ~DBConnector(); redisContext *getContext() const; int getDbId() const; std::string getDbName() const; + std::string getNamespace() const; static void select(DBConnector *db); @@ -84,6 +103,7 @@ class DBConnector redisContext *m_conn; int m_dbId; std::string m_dbName; + std::string m_nameSpace; }; } From 7918b4ad50f331969d283523c0563a218c374b71 Mon Sep 17 00:00:00 2001 From: Judy Joseph Date: Thu, 9 Jul 2020 17:00:44 -0700 Subject: [PATCH 02/11] Updates to the initial commit --- common/dbconnector.cpp | 21 ++++++++++++++------- common/dbconnector.h | 23 ++++++++++++----------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index d497a3b26..5dca656eb 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -97,7 +97,7 @@ void SonicDBConfig::initialize_global_config(const string &file) if(element["namespace"].empty()) { - ns_name = ""; + ns_name = EMPTY_NAMESPACE; } else { @@ -127,6 +127,11 @@ void SonicDBConfig::initialize_global_config(const string &file) throw runtime_error("Sonic database config file syntax error >> " + string(e.what())); } } + else + { + SWSS_LOG_ERROR("Sonic database global config file doesn't exist at %s\n", file.c_str()); + throw runtime_error("Sonic database global config file doesn't exist at " + file); + } // Set it as the global config file is already parsed and init done. m_global_init = true; @@ -146,17 +151,18 @@ void SonicDBConfig::initialize(const string &file, const string &nameSpace) throw runtime_error("SonicDBConfig already initialized"); } + // namespace string is empty, use the file given as input to parse. if(nameSpace.empty()) { parse_config(file, inst_entry, db_entry, separator_entry); - m_inst_info[""] = inst_entry; - m_db_info[""] = db_entry; - m_db_separator[""] = separator_entry; + m_inst_info[EMPTY_NAMESPACE] = inst_entry; + m_db_info[EMPTY_NAMESPACE] = db_entry; + m_db_separator[EMPTY_NAMESPACE] = separator_entry; } else + // namespace is not empty, use DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE. initialize_global_config(); - // Set it as the config file is already parsed and init done. m_init = true; } @@ -236,6 +242,7 @@ vector SonicDBConfig::getNamespaces() if (!m_global_init) initialize_global_config(); + // This API returns back non-empty namespaces. for (auto it = m_inst_info.cbegin(); it != m_inst_info.cend(); ++it) { if(!((it->first).empty())) list.push_back(it->first); @@ -272,7 +279,7 @@ DBConnector::~DBConnector() DBConnector::DBConnector(int dbId, const string& hostname, int port, unsigned int timeout) : m_dbId(dbId), - m_nameSpace("") + m_nameSpace(EMPTY_NAMESPACE) { struct timeval tv = {0, (suseconds_t)timeout * 1000}; @@ -290,7 +297,7 @@ DBConnector::DBConnector(int dbId, const string& hostname, int port, DBConnector::DBConnector(int dbId, const string& unixPath, unsigned int timeout) : m_dbId(dbId), - m_nameSpace("") + m_nameSpace(EMPTY_NAMESPACE) { struct timeval tv = {0, (suseconds_t)timeout * 1000}; diff --git a/common/dbconnector.h b/common/dbconnector.h index 928b95e33..b4600c1a3 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -7,6 +7,7 @@ #include #include +#define EMPTY_NAMESPACE "" namespace swss { @@ -35,16 +36,16 @@ class SonicDBConfig std::unordered_map &inst_entry, std::unordered_map &db_entry, std::unordered_map &separator_entry); - static void initialize(const std::string &file = DEFAULT_SONIC_DB_CONFIG_FILE, const std::string &nameSpace = ""); + static void initialize(const std::string &file = DEFAULT_SONIC_DB_CONFIG_FILE, const std::string &nameSpace = EMPTY_NAMESPACE); static void initialize_global_config(const std::string &file = DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE); - static std::string getDbInst(const std::string &dbName, const std::string &nameSpace = ""); - static int getDbId(const std::string &dbName, const std::string &nameSpace = ""); - static std::string getSeparator(const std::string &dbName, const std::string &nameSpace = ""); - static std::string getSeparator(int dbId, const std::string &nameSpace = ""); + static std::string getDbInst(const std::string &dbName, const std::string &nameSpace = EMPTY_NAMESPACE); + static int getDbId(const std::string &dbName, const std::string &nameSpace = EMPTY_NAMESPACE); + static std::string getSeparator(const std::string &dbName, const std::string &nameSpace = EMPTY_NAMESPACE); + static std::string getSeparator(int dbId, const std::string &nameSpace = EMPTY_NAMESPACE); static std::string getSeparator(const DBConnector* db); - static std::string getDbSock(const std::string &dbName, const std::string &nameSpace = ""); - static std::string getDbHostname(const std::string &dbName, const std::string &nameSpace = ""); - static int getDbPort(const std::string &dbName, const std::string &nameSpace = ""); + static std::string getDbSock(const std::string &dbName, const std::string &nameSpace = EMPTY_NAMESPACE); + static std::string getDbHostname(const std::string &dbName, const std::string &nameSpace = EMPTY_NAMESPACE); + static int getDbPort(const std::string &dbName, const std::string &nameSpace = EMPTY_NAMESPACE); static std::vector getNamespaces(); static bool isInit() { return m_init; }; static bool isGlobalInit() { return m_global_init; }; @@ -55,9 +56,9 @@ class SonicDBConfig static constexpr const char *DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE = "/var/run/redis/sonic-db/database_global.json"; // { namespace { instName, { unix_socket_path, hostname, port } } } static std::unordered_map> m_inst_info; - // { namespace, { dbName, {instName, dbId} } } + // { namespace, { dbName, {instName, dbId, separator} } } static std::unordered_map> m_db_info; - // { namespace, { dbIp, separator } } + // { namespace, { dbId, separator } } static std::unordered_map> m_db_separator; static bool m_init; static bool m_global_init; @@ -77,7 +78,7 @@ class DBConnector */ DBConnector(int dbId, const std::string &hostname, int port, unsigned int timeout); DBConnector(int dbId, const std::string &unixPath, unsigned int timeout); - DBConnector(const std::string &dbName, unsigned int timeout, bool isTcpConn = false, const std::string &nameSpace = ""); + DBConnector(const std::string &dbName, unsigned int timeout, bool isTcpConn = false, const std::string &nameSpace = EMPTY_NAMESPACE); ~DBConnector(); From 77e89185f496a4f508731cbea0511cf4ebada0ec Mon Sep 17 00:00:00 2001 From: Judy Joseph Date: Mon, 13 Jul 2020 18:03:23 -0700 Subject: [PATCH 03/11] Updates for comments --- common/dbconnector.cpp | 34 +++++++++++++++++----------------- common/dbconnector.h | 22 +++++++++++----------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index 5dca656eb..e5607a7d1 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -16,8 +16,8 @@ using namespace std; namespace swss { -void SonicDBConfig::parse_config(const string &file, - std::unordered_map &inst_entry, +void SonicDBConfig::parseDatabaseConfig(const string &file, + std::unordered_map &inst_entry, std::unordered_map &db_entry, std::unordered_map &separator_entry) { @@ -67,11 +67,11 @@ void SonicDBConfig::parse_config(const string &file, } } -void SonicDBConfig::initialize_global_config(const string &file) +void SonicDBConfig::initializeGlobalConfig(const string &file) { std::string local_file, ns_name; std::unordered_map db_entry; - std::unordered_map inst_entry; + std::unordered_map inst_entry; std::unordered_map separator_entry; SWSS_LOG_ENTER(); @@ -104,7 +104,7 @@ void SonicDBConfig::initialize_global_config(const string &file) ns_name = element["namespace"]; } - parse_config(local_file, inst_entry, db_entry, separator_entry); + parseDatabaseConfig(local_file, inst_entry, db_entry, separator_entry); m_inst_info[ns_name] = inst_entry; m_db_info[ns_name] = db_entry; m_db_separator[ns_name] = separator_entry; @@ -140,7 +140,7 @@ void SonicDBConfig::initialize_global_config(const string &file) void SonicDBConfig::initialize(const string &file, const string &nameSpace) { std::unordered_map db_entry; - std::unordered_map inst_entry; + std::unordered_map inst_entry; std::unordered_map separator_entry; SWSS_LOG_ENTER(); @@ -154,14 +154,14 @@ void SonicDBConfig::initialize(const string &file, const string &nameSpace) // namespace string is empty, use the file given as input to parse. if(nameSpace.empty()) { - parse_config(file, inst_entry, db_entry, separator_entry); + parseDatabaseConfig(file, inst_entry, db_entry, separator_entry); m_inst_info[EMPTY_NAMESPACE] = inst_entry; m_db_info[EMPTY_NAMESPACE] = db_entry; m_db_separator[EMPTY_NAMESPACE] = separator_entry; } else // namespace is not empty, use DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE. - initialize_global_config(); + initializeGlobalConfig(); // Set it as the config file is already parsed and init done. m_init = true; @@ -218,14 +218,14 @@ string SonicDBConfig::getDbSock(const string &dbName, const string &nameSpace) { if (!m_init) initialize(nameSpace); - return m_inst_info[nameSpace].at(getDbInst(dbName)).socket; + return m_inst_info[nameSpace].at(getDbInst(dbName)).unix_socket_path; } string SonicDBConfig::getDbHostname(const string &dbName, const string &nameSpace) { if (!m_init) initialize(nameSpace); - return m_inst_info[nameSpace].at(getDbInst(dbName)).hostName; + return m_inst_info[nameSpace].at(getDbInst(dbName)).hostname; } int SonicDBConfig::getDbPort(const string &dbName, const string &nameSpace) @@ -240,7 +240,7 @@ vector SonicDBConfig::getNamespaces() vector list; if (!m_global_init) - initialize_global_config(); + initializeGlobalConfig(); // This API returns back non-empty namespaces. for (auto it = m_inst_info.cbegin(); it != m_inst_info.cend(); ++it) { @@ -254,7 +254,7 @@ vector SonicDBConfig::getNamespaces() constexpr const char *SonicDBConfig::DEFAULT_SONIC_DB_CONFIG_FILE; constexpr const char *SonicDBConfig::DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE; constexpr const char *SonicDBConfig::DEFAULT_SONIC_DB_CONFIG_DIR; -unordered_map> SonicDBConfig::m_inst_info; +unordered_map> SonicDBConfig::m_inst_info; unordered_map> SonicDBConfig::m_db_info; unordered_map> SonicDBConfig::m_db_separator; bool SonicDBConfig::m_init = false; @@ -279,7 +279,7 @@ DBConnector::~DBConnector() DBConnector::DBConnector(int dbId, const string& hostname, int port, unsigned int timeout) : m_dbId(dbId), - m_nameSpace(EMPTY_NAMESPACE) + m_namespace(EMPTY_NAMESPACE) { struct timeval tv = {0, (suseconds_t)timeout * 1000}; @@ -297,7 +297,7 @@ DBConnector::DBConnector(int dbId, const string& hostname, int port, DBConnector::DBConnector(int dbId, const string& unixPath, unsigned int timeout) : m_dbId(dbId), - m_nameSpace(EMPTY_NAMESPACE) + m_namespace(EMPTY_NAMESPACE) { struct timeval tv = {0, (suseconds_t)timeout * 1000}; @@ -308,7 +308,7 @@ DBConnector::DBConnector(int dbId, const string& unixPath, unsigned int timeout) if (m_conn->err) throw system_error(make_error_code(errc::address_not_available), - "Unable to connect to redis (unixs-socket)"); + "Unable to connect to redis (unix-socket)"); select(this); } @@ -316,7 +316,7 @@ DBConnector::DBConnector(int dbId, const string& unixPath, unsigned int timeout) DBConnector::DBConnector(const string& dbName, unsigned int timeout, bool isTcpConn, const string& nameSpace) : m_dbId(SonicDBConfig::getDbId(dbName, nameSpace)) , m_dbName(dbName) - , m_nameSpace(nameSpace) + , m_namespace(nameSpace) { struct timeval tv = {0, (suseconds_t)timeout * 1000}; @@ -359,7 +359,7 @@ string DBConnector::getDbName() const string DBConnector::getNamespace() const { - return m_nameSpace; + return m_namespace; } DBConnector *DBConnector::newConnector(unsigned int timeout) const diff --git a/common/dbconnector.h b/common/dbconnector.h index b4600c1a3..b54baa431 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -7,17 +7,17 @@ #include #include -#define EMPTY_NAMESPACE "" +#define EMPTY_NAMESPACE std::string() namespace swss { class DBConnector; -class SonicInstInfo +class RedisInstInfo { public: - std::string socket; - std::string hostName; + std::string unix_socket_path; + std::string hostname; int port; }; @@ -32,12 +32,8 @@ class SonicDBInfo class SonicDBConfig { public: - static void parse_config(const std::string &file, - std::unordered_map &inst_entry, - std::unordered_map &db_entry, - std::unordered_map &separator_entry); static void initialize(const std::string &file = DEFAULT_SONIC_DB_CONFIG_FILE, const std::string &nameSpace = EMPTY_NAMESPACE); - static void initialize_global_config(const std::string &file = DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE); + static void initializeGlobalConfig(const std::string &file = DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE); static std::string getDbInst(const std::string &dbName, const std::string &nameSpace = EMPTY_NAMESPACE); static int getDbId(const std::string &dbName, const std::string &nameSpace = EMPTY_NAMESPACE); static std::string getSeparator(const std::string &dbName, const std::string &nameSpace = EMPTY_NAMESPACE); @@ -55,13 +51,17 @@ class SonicDBConfig static constexpr const char *DEFAULT_SONIC_DB_CONFIG_FILE = "/var/run/redis/sonic-db/database_config.json"; static constexpr const char *DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE = "/var/run/redis/sonic-db/database_global.json"; // { namespace { instName, { unix_socket_path, hostname, port } } } - static std::unordered_map> m_inst_info; + static std::unordered_map> m_inst_info; // { namespace, { dbName, {instName, dbId, separator} } } static std::unordered_map> m_db_info; // { namespace, { dbId, separator } } static std::unordered_map> m_db_separator; static bool m_init; static bool m_global_init; + static void parseDatabaseConfig(const std::string &file, + std::unordered_map &inst_entry, + std::unordered_map &db_entry, + std::unordered_map &separator_entry); }; class DBConnector @@ -104,7 +104,7 @@ class DBConnector redisContext *m_conn; int m_dbId; std::string m_dbName; - std::string m_nameSpace; + std::string m_namespace; }; } From b188047da3cfb5dc3b5e63ffd56bff795cd07453 Mon Sep 17 00:00:00 2001 From: Judy Joseph Date: Thu, 16 Jul 2020 20:42:37 -0700 Subject: [PATCH 04/11] Add UT tests for the namespace usecases --- tests/Makefile.am | 1 + tests/main.cpp | 24 +++++ .../database_config0.json | 82 ++++++++++++++++ .../database_config1.json | 82 ++++++++++++++++ .../database_global.json | 16 ++++ tests/redis_multi_ns_ut.cpp | 93 +++++++++++++++++++ 6 files changed, 298 insertions(+) create mode 100644 tests/redis_multi_db_ut_config/database_config0.json create mode 100644 tests/redis_multi_db_ut_config/database_config1.json create mode 100644 tests/redis_multi_db_ut_config/database_global.json create mode 100644 tests/redis_multi_ns_ut.cpp diff --git a/tests/Makefile.am b/tests/Makefile.am index a059ac792..9fbf48ead 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -28,6 +28,7 @@ tests_SOURCES = redis_ut.cpp \ warm_restart_ut.cpp \ redis_multi_db_ut.cpp \ logger_ut.cpp \ + redis_multi_ns_ut.cpp \ fdb_flush.cpp \ main.cpp diff --git a/tests/main.cpp b/tests/main.cpp index 89c4bdb29..0a0b910b1 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -7,6 +7,8 @@ using namespace swss; string existing_file = "./tests/redis_multi_db_ut_config/database_config.json"; string nonexisting_file = "./tests/redis_multi_db_ut_config/database_config_nonexisting.json"; +string global_existing_file = "./tests/redis_multi_db_ut_config/database_global.json"; +string global_nonexisting_file = "./tests/redis_multi_db_ut_config/database_global_nonexisting.json"; class SwsscommonEnvironment : public ::testing::Environment { public: @@ -32,6 +34,28 @@ class SwsscommonEnvironment : public ::testing::Environment { SonicDBConfig::initialize(existing_file); cout<<"INIT: load local db config file, isInit = "< +#include +#include "gtest/gtest.h" +#include "common/dbconnector.h" +#include "common/json.hpp" +#include + +using namespace std; +using namespace swss; +using json = nlohmann::json; + +extern string global_existing_file; + +TEST(DBConnector, multi_ns_test) +{ + std::string local_file, ns_name; + vector list; + vector ns_list; + + // load global config file again, should throw exception with init already done + try + { + cout<<"INIT: loading local config file again"<> g; + + for (auto& element : g["INCLUDES"]) + { + local_file = element["include"]; + + if(element["namespace"].empty()) + { + ns_name = EMPTY_NAMESPACE; + } + else + { + ns_name = element["namespace"]; + list.push_back(ns_name); + } + + // parse config file + ifstream i(local_file); + if (i.good()) + { + json j; + i >> j; + unordered_map m_inst_info; + for (auto it = j["INSTANCES"].begin(); it!= j["INSTANCES"].end(); it++) + { + string instName = it.key(); + string socket = it.value().at("unix_socket_path"); + string hostname = it.value().at("hostname"); + int port = it.value().at("port"); + m_inst_info[instName] = {socket, hostname, port}; + } + + for (auto it = j["DATABASES"].begin(); it!= j["DATABASES"].end(); it++) + { + string dbName = it.key(); + string instName = it.value().at("instance"); + int dbId = it.value().at("id"); + cout<<"testing "< Date: Sun, 19 Jul 2020 23:21:28 -0700 Subject: [PATCH 05/11] Updates to the unit tests --- tests/main.cpp | 6 +++--- tests/redis_multi_ns_ut.cpp | 25 +++++++++++++++++-------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/main.cpp b/tests/main.cpp index 0a0b910b1..2f8c6417f 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -35,7 +35,7 @@ class SwsscommonEnvironment : public ::testing::Environment { cout<<"INIT: load local db config file, isInit = "< list; - vector ns_list; + std::string local_file, dir_name, ns_name; + vector namespaces; + vector ns_names; // load global config file again, should throw exception with init already done try @@ -34,9 +34,14 @@ TEST(DBConnector, multi_ns_test) json g; f >> g; + // Get the directory name from the file path given as input. + std::string::size_type pos = global_existing_file.rfind("/"); + dir_name = global_existing_file.substr(0,pos+1); + for (auto& element : g["INCLUDES"]) { - local_file = element["include"]; + local_file.append(dir_name); + local_file.append(element["include"]); if(element["namespace"].empty()) { @@ -45,11 +50,12 @@ TEST(DBConnector, multi_ns_test) else { ns_name = element["namespace"]; - list.push_back(ns_name); + namespaces.push_back(ns_name); } // parse config file ifstream i(local_file); + if (i.good()) { json j; @@ -83,11 +89,14 @@ TEST(DBConnector, multi_ns_test) EXPECT_EQ(m_inst_info[instName].port, SonicDBConfig::getDbPort(dbName, ns_name)); } } + local_file.clear(); } } // Get the namespaces from the database_global.json file and compare with the list we created here. - ns_list = SonicDBConfig::getNamespaces(); - EXPECT_EQ(ns_list.size(), list.size()); - EXPECT_TRUE(list == ns_list); + ns_names = SonicDBConfig::getNamespaces(); + sort (namespaces.begin(), namespaces.end()); + sort (ns_names.begin(), ns_names.end()); + EXPECT_EQ(ns_names.size(), namespaces.size()); + EXPECT_TRUE(namespaces == ns_names); } From b39d0dc6615156e849045eea17dc5b2cef62c3d4 Mon Sep 17 00:00:00 2001 From: Judy Joseph Date: Sun, 19 Jul 2020 23:35:06 -0700 Subject: [PATCH 06/11] Derive the DIR of the config files from the file path itself. --- common/dbconnector.cpp | 10 +++++++--- common/dbconnector.h | 1 - 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index e5607a7d1..9f7ab0bde 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -69,7 +69,7 @@ void SonicDBConfig::parseDatabaseConfig(const string &file, void SonicDBConfig::initializeGlobalConfig(const string &file) { - std::string local_file, ns_name; + std::string local_file, dir_name, ns_name; std::unordered_map db_entry; std::unordered_map inst_entry; std::unordered_map separator_entry; @@ -82,9 +82,14 @@ void SonicDBConfig::initializeGlobalConfig(const string &file) return; } + ifstream i(file); if (i.good()) { + // Get the directory name from the file path given as input. + std::string::size_type pos = file.rfind("/"); + dir_name = file.substr(0,pos+1); + try { json j; @@ -92,7 +97,7 @@ void SonicDBConfig::initializeGlobalConfig(const string &file) for (auto& element : j["INCLUDES"]) { - local_file.append(DEFAULT_SONIC_DB_CONFIG_DIR); + local_file.append(dir_name); local_file.append(element["include"]); if(element["namespace"].empty()) @@ -253,7 +258,6 @@ vector SonicDBConfig::getNamespaces() constexpr const char *SonicDBConfig::DEFAULT_SONIC_DB_CONFIG_FILE; constexpr const char *SonicDBConfig::DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE; -constexpr const char *SonicDBConfig::DEFAULT_SONIC_DB_CONFIG_DIR; unordered_map> SonicDBConfig::m_inst_info; unordered_map> SonicDBConfig::m_db_info; unordered_map> SonicDBConfig::m_db_separator; diff --git a/common/dbconnector.h b/common/dbconnector.h index b54baa431..a2c029410 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -47,7 +47,6 @@ class SonicDBConfig static bool isGlobalInit() { return m_global_init; }; private: - static constexpr const char *DEFAULT_SONIC_DB_CONFIG_DIR = "/var/run/redis/sonic-db/"; static constexpr const char *DEFAULT_SONIC_DB_CONFIG_FILE = "/var/run/redis/sonic-db/database_config.json"; static constexpr const char *DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE = "/var/run/redis/sonic-db/database_global.json"; // { namespace { instName, { unix_socket_path, hostname, port } } } From ed202a1ed52641f64aa4fe093a4a557e86cb2d09 Mon Sep 17 00:00:00 2001 From: Abhishek Dosi Date: Mon, 20 Jul 2020 08:56:54 -0700 Subject: [PATCH 07/11] Fixes when testing and trying to try to connect multiple namespace DB. Signed-off-by: Abhishek Dosi --- common/dbconnector.cpp | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index 9f7ab0bde..e541667e5 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -140,6 +140,8 @@ void SonicDBConfig::initializeGlobalConfig(const string &file) // Set it as the global config file is already parsed and init done. m_global_init = true; + // Make regular init also done + m_init = true; } void SonicDBConfig::initialize(const string &file, const string &nameSpace) @@ -175,28 +177,28 @@ void SonicDBConfig::initialize(const string &file, const string &nameSpace) string SonicDBConfig::getDbInst(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(nameSpace); + initialize(DEFAULT_SONIC_DB_CONFIG_FILE, nameSpace); return m_db_info[nameSpace].at(dbName).instName; } int SonicDBConfig::getDbId(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(nameSpace); + initialize(DEFAULT_SONIC_DB_CONFIG_FILE, nameSpace); return m_db_info[nameSpace].at(dbName).dbId; } string SonicDBConfig::getSeparator(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(nameSpace); + initialize(DEFAULT_SONIC_DB_CONFIG_FILE, nameSpace); return m_db_info[nameSpace].at(dbName).separator; } string SonicDBConfig::getSeparator(int dbId, const string &nameSpace) { if (!m_init) - initialize(nameSpace); + initialize(DEFAULT_SONIC_DB_CONFIG_FILE, nameSpace); return m_db_separator[nameSpace].at(dbId); } @@ -222,21 +224,21 @@ string SonicDBConfig::getSeparator(const DBConnector* db) string SonicDBConfig::getDbSock(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(nameSpace); + initialize(DEFAULT_SONIC_DB_CONFIG_FILE, nameSpace); return m_inst_info[nameSpace].at(getDbInst(dbName)).unix_socket_path; } string SonicDBConfig::getDbHostname(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(nameSpace); + initialize(DEFAULT_SONIC_DB_CONFIG_FILE, nameSpace); return m_inst_info[nameSpace].at(getDbInst(dbName)).hostname; } int SonicDBConfig::getDbPort(const string &dbName, const string &nameSpace) { if (!m_init) - initialize(nameSpace); + initialize(DEFAULT_SONIC_DB_CONFIG_FILE, nameSpace); return m_inst_info[nameSpace].at(getDbInst(dbName)).port; } @@ -369,16 +371,7 @@ string DBConnector::getNamespace() const DBConnector *DBConnector::newConnector(unsigned int timeout) const { DBConnector *ret; - if (getContext()->connection_type == REDIS_CONN_TCP) - ret = new DBConnector(getDbId(), - getContext()->tcp.host, - getContext()->tcp.port, - timeout); - else - ret = new DBConnector(getDbId(), - getContext()->unix_sock.path, - timeout); - ret->m_dbName = m_dbName; + ret = new DBConnector(getDbName(), timeout, (getContext()->connection_type == REDIS_CONN_TCP), getNamespace()); return ret; } From d79bcd3f25f3bc2d5f6ec0a8d4944c19961ecbad Mon Sep 17 00:00:00 2001 From: Abhishek Dosi Date: Mon, 20 Jul 2020 09:09:43 -0700 Subject: [PATCH 08/11] Added Support to get DB Id and DB Namespace from Selectable Object. It is helpful in application that are doing select and need to the event is generated from which DB and namespace so that any parituclar action can be taken if needed. Signed-off-by: Abhishek Dosi --- common/redisselect.cpp | 9 +++++++++ common/redisselect.h | 2 ++ common/selectable.h | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/common/redisselect.cpp b/common/redisselect.cpp index 465428220..9d97dfef4 100644 --- a/common/redisselect.cpp +++ b/common/redisselect.cpp @@ -16,6 +16,15 @@ int RedisSelect::getFd() { return m_subscribe->getContext()->fd; } +int RedisSelect::getDbId() +{ + return m_subscribe->getDbId(); +} + +std::string RedisSelect::getDbNamespace() +{ + return m_subscribe->getNamespace(); +} uint64_t RedisSelect::readData() { diff --git a/common/redisselect.h b/common/redisselect.h index 97a70b12b..361570d0e 100644 --- a/common/redisselect.h +++ b/common/redisselect.h @@ -20,6 +20,8 @@ class RedisSelect : public Selectable bool hasCachedData() override; bool initializedWithData() override; void updateAfterRead() override; + int getDbId() override; + std::string getDbNamespace() override; /* Create a new redisContext, SELECT DB and SUBSCRIBE */ void subscribe(DBConnector* db, const std::string &channelName); diff --git a/common/selectable.h b/common/selectable.h index 268b933c8..531489a23 100644 --- a/common/selectable.h +++ b/common/selectable.h @@ -58,6 +58,16 @@ class Selectable return m_priority; } + virtual int getDbId() + { + return 0; + } + + virtual std::string getDbNamespace() + { + return std::string(); + } + private: friend class Select; From a7898c325a3a24b2417f9a1d00fb0697d4ef62d5 Mon Sep 17 00:00:00 2001 From: Judy Joseph Date: Mon, 20 Jul 2020 11:30:00 -0700 Subject: [PATCH 09/11] Review comments update, for the variable name. --- common/dbconnector.cpp | 2 +- common/dbconnector.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index e541667e5..215867be9 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -225,7 +225,7 @@ string SonicDBConfig::getDbSock(const string &dbName, const string &nameSpace) { if (!m_init) initialize(DEFAULT_SONIC_DB_CONFIG_FILE, nameSpace); - return m_inst_info[nameSpace].at(getDbInst(dbName)).unix_socket_path; + return m_inst_info[nameSpace].at(getDbInst(dbName)).unixSocketPath; } string SonicDBConfig::getDbHostname(const string &dbName, const string &nameSpace) diff --git a/common/dbconnector.h b/common/dbconnector.h index a2c029410..4bb6b8612 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -16,7 +16,7 @@ class DBConnector; class RedisInstInfo { public: - std::string unix_socket_path; + std::string unixSocketPath; std::string hostname; int port; }; From 92eacae0a605618be13df3700983f1f28f06fc29 Mon Sep 17 00:00:00 2001 From: Judy Joseph Date: Mon, 20 Jul 2020 23:32:56 -0700 Subject: [PATCH 10/11] Further fixes and updating unit tests --- common/dbconnector.cpp | 15 ++++++++++++++- tests/redis_multi_ns_ut.cpp | 4 ++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index 215867be9..5ad6d0c94 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -86,9 +86,14 @@ void SonicDBConfig::initializeGlobalConfig(const string &file) ifstream i(file); if (i.good()) { + local_file = dir_name = std::string(); + // Get the directory name from the file path given as input. std::string::size_type pos = file.rfind("/"); - dir_name = file.substr(0,pos+1); + if( pos != std::string::npos) + { + dir_name = file.substr(0,pos+1); + } try { @@ -102,6 +107,13 @@ void SonicDBConfig::initializeGlobalConfig(const string &file) if(element["namespace"].empty()) { + // If database_config.json is already initlized via SonicDBConfig::initialize + // skip initializing it here again. + if(m_init) + { + local_file.clear(); + continue; + } ns_name = EMPTY_NAMESPACE; } else @@ -140,6 +152,7 @@ void SonicDBConfig::initializeGlobalConfig(const string &file) // Set it as the global config file is already parsed and init done. m_global_init = true; + // Make regular init also done m_init = true; } diff --git a/tests/redis_multi_ns_ut.cpp b/tests/redis_multi_ns_ut.cpp index 7308a2b9a..ad90463f5 100644 --- a/tests/redis_multi_ns_ut.cpp +++ b/tests/redis_multi_ns_ut.cpp @@ -76,13 +76,13 @@ TEST(DBConnector, multi_ns_test) string instName = it.value().at("instance"); int dbId = it.value().at("id"); cout<<"testing "< Date: Thu, 23 Jul 2020 14:15:01 -0700 Subject: [PATCH 11/11] Renaming the newly added function in swss::RedisSelect to getDbConnectorId() as it was clashing with swss::TableBase::getDbId() --- common/redisselect.cpp | 2 +- common/redisselect.h | 2 +- common/selectable.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/redisselect.cpp b/common/redisselect.cpp index 9d97dfef4..4c76f19f8 100644 --- a/common/redisselect.cpp +++ b/common/redisselect.cpp @@ -16,7 +16,7 @@ int RedisSelect::getFd() { return m_subscribe->getContext()->fd; } -int RedisSelect::getDbId() +int RedisSelect::getDbConnectorId() { return m_subscribe->getDbId(); } diff --git a/common/redisselect.h b/common/redisselect.h index 361570d0e..6c291a4dd 100644 --- a/common/redisselect.h +++ b/common/redisselect.h @@ -20,7 +20,7 @@ class RedisSelect : public Selectable bool hasCachedData() override; bool initializedWithData() override; void updateAfterRead() override; - int getDbId() override; + int getDbConnectorId() override; std::string getDbNamespace() override; /* Create a new redisContext, SELECT DB and SUBSCRIBE */ diff --git a/common/selectable.h b/common/selectable.h index 531489a23..7de56685a 100644 --- a/common/selectable.h +++ b/common/selectable.h @@ -58,7 +58,7 @@ class Selectable return m_priority; } - virtual int getDbId() + virtual int getDbConnectorId() { return 0; }