[debug dump util] Module implementation Logic and Port Module (#1667)

* Port Module and UT Added
What I did
HLD for Dump Utility: HLD.
For More Info on adding new modules, Check this section in the HLD:
MatchInfra
Signed-off-by: Vivek Reddy Karri <[email protected]>

Author: vivekrnv

dump/match_infra.py

@@ -1,4 +1,5 @@
-import json, fnmatch
+import json
+import fnmatch
 from abc import ABC, abstractmethod
 from dump.helper import verbose_print
 from swsscommon.swsscommon import SonicV2Connector, SonicDBConfig
@@ -12,24 +13,25 @@
     "NO_SRC": "Either one of db or file in the request should be non-empty",
     "NO_TABLE": "No 'table' name provided",
     "NO_KEY": "'key_pattern' cannot be empty",
-    "NO_VALUE" : "Field is provided, but no value is provided to compare with", 
+    "NO_VALUE": "Field is provided, but no value is provided to compare with",
     "SRC_VAGUE": "Only one of db or file should be provided",
-    "CONN_ERR" : "Connection Error",
+    "CONN_ERR": "Connection Error",
     "JUST_KEYS_COMPAT": "When Just_keys is set to False, return_fields should be empty",
     "BAD_FORMAT_RE_FIELDS": "Return Fields should be of list type",
     "NO_ENTRIES": "No Keys found after applying the filtering criteria",
     "FILE_R_EXEP": "Exception Caught While Reading the json cfg file provided",
     "INV_NS": "Namespace is invalid"
 }
 
+
 class MatchRequest:
-    """ 
-    Request Object which should be passed to the MatchEngine  
-    
+    """
+    Request Object which should be passed to the MatchEngine
+
     Attributes:
     "table"             : A Valid Table Name
     "key_pattern"       : Pattern of the redis-key to match. Defaults to "*". Eg: "*" will match all the keys.
-                          Supports these glob style patterns. https://redis.io/commands/KEYS               
+                          Supports these glob style patterns. https://redis.io/commands/KEYS
     "field"             : Field to check for a match,Defaults to None
     "value"             : Value to match, Defaults to None
     "return_fields"     : An iterable type, where each element woudld imply a field to return from all the filtered keys
@@ -38,9 +40,10 @@ class MatchRequest:
                           Only one of the db/file fields should have a non-empty string.
     "just_keys"         : If true, Only Returns the keys matched. Does not return field-value pairs. Defaults to True
     "ns"                : namespace argument, if nothing is provided, default namespace is used
-    "match_entire_list" : When this arg is set to true, entire list is matched incluing the ",". 
+    "match_entire_list" : When this arg is set to true, entire list is matched incluing the ",".
                           When False, the values are split based on "," and individual items are matched with
     """
+
     def __init__(self, **kwargs):
         self.table = kwargs["table"] if "table" in kwargs else None
         self.key_pattern = kwargs["key_pattern"] if "key_pattern" in kwargs else "*"
@@ -56,16 +59,15 @@ def __init__(self, **kwargs):
         verbose_print(str(err))
         if err:
             raise Exception("Static Checks for the MatchRequest Failed, Reason: \n" + err)
-        
-        
+
     def __static_checks(self):
-        
+
         if not self.db and not self.file:
             return EXCEP_DICT["NO_SRC"]
-        
+
         if self.db and self.file:
             return EXCEP_DICT["SRC_VAGUE"]
-        
+
         if not self.db:
             try:
                 with open(self.file) as f:
@@ -75,32 +77,32 @@ def __static_checks(self):
 
         if not self.file and self.db not in SonicDBConfig.getDbList():
             return EXCEP_DICT["INV_DB"]
-        
+
         if not self.table:
             return EXCEP_DICT["NO_TABLE"]
-        
+
         if not isinstance(self.return_fields, list):
             return EXCEP_DICT["BAD_FORMAT_RE_FIELDS"]
-        
+
         if not self.just_keys and self.return_fields:
             return EXCEP_DICT["JUST_KEYS_COMPAT"]
-        
+
         if self.field and not self.value:
             return EXCEP_DICT["NO_VALUE"]
-        
+
         if self.ns != DEFAULT_NAMESPACE and self.ns not in multi_asic.get_namespace_list():
             return EXCEP_DICT["INV_NS"] + " Choose From {}".format(multi_asic.get_namespace_list())
-        
+
         verbose_print("MatchRequest Checks Passed")
-        
+
         return ""
-    
+
     def __str__(self):
         str = "----------------------- \n MatchRequest: \n"
         if self.db:
             str += "db:{} , ".format(self.db)
         if self.file:
-            str += "file:{} , ".format(self.file) 
+            str += "file:{} , ".format(self.file)
         if self.table:
             str += "table:{} , ".format(self.table)
         if self.key_pattern:
@@ -116,78 +118,76 @@ def __str__(self):
         if len(self.return_fields) > 0:
             str += "return_fields: " + ",".join(self.return_fields) + " "
         if self.ns:
-            str += "namespace: , " + self.ns 
+            str += "namespace: , " + self.ns
         if self.match_entire_list:
             str += "match_list: True , "
         else:
             str += "match_list: False , "
         return str
-    
+
+
 class SourceAdapter(ABC):
     """ Source Adaptor offers unified interface to Data Sources """
-    
+
     def __init__(self):
         pass
-    
+
     @abstractmethod
     def connect(self, db, ns):
         """ Return True for Success, False for failure """
         return False
-    
+
     @abstractmethod
     def getKeys(self, db, table, key_pattern):
         return []
-    
+
     @abstractmethod
     def get(self, db, key):
         return {}
-    
+
     @abstractmethod
     def hget(self, db, key, field):
         return ""
-    
+
     @abstractmethod
     def get_separator(self, db):
         return ""
-        
+
+
 class RedisSource(SourceAdapter):
     """ Concrete Adaptor Class for connecting to Redis Data Sources """
-    
-    def __init__(self):
-        self.conn = None 
-        
+
+    def __init__(self, conn_pool):
+        self.conn = None
+        self.pool = conn_pool
+
     def connect(self, db, ns):
         try:
-            if not SonicDBConfig.isInit():
-                if multi_asic.is_multi_asic():
-                    SonicDBConfig.load_sonic_global_db_config()
-                else:
-                    SonicDBConfig.load_sonic_db_config()
-            self.conn = SonicV2Connector(namespace=ns, use_unix_socket_path=True) 
-            self.conn.connect(db)
+            self.conn = self.pool.get(db, ns)
         except Exception as e:
             verbose_print("RedisSource: Connection Failed\n" + str(e))
             return False
         return True
-    
+
     def get_separator(self, db):
         return self.conn.get_db_separator(db)
-       
-    def getKeys(self, db, table, key_pattern):       
+
+    def getKeys(self, db, table, key_pattern):
         return self.conn.keys(db, table + self.get_separator(db) + key_pattern)
-    
+
     def get(self, db, key):
         return self.conn.get_all(db, key)
-     
+
     def hget(self, db, key, field):
         return self.conn.get(db, key, field)
 
+
 class JsonSource(SourceAdapter):
     """ Concrete Adaptor Class for connecting to JSON Data Sources """
-    
+
     def __init__(self):
         self.json_data = None
-    
+
     def connect(self, db, ns):
         try:
             with open(db) as f:
@@ -196,67 +196,114 @@ def connect(self, db, ns):
             verbose_print("JsonSource: Loading the JSON file failed" + str(e))
             return False
         return True
-    
+
     def get_separator(self, db):
         return SonicDBConfig.getSeparator("CONFIG_DB")
-    
+
     def getKeys(self, db, table, key_pattern):
         if table not in self.json_data:
             return []
         # https://docs.python.org/3.7/library/fnmatch.html
         kp = key_pattern.replace("[^", "[!")
         kys = fnmatch.filter(self.json_data[table].keys(), kp)
         return [table + self.get_separator(db) + ky for ky in kys]
-    
+
     def get(self, db, key):
         sep = self.get_separator(db)
         table, key = key.split(sep, 1)
         return self.json_data.get(table, {}).get(key, {})
-            
+
     def hget(self, db, key, field):
         sep = self.get_separator(db)
         table, key = key.split(sep, 1)
         return self.json_data.get(table, "").get(key, "").get(field, "")
-                        
+
+
+class ConnectionPool:
+    """ Caches SonicV2Connector objects for effective reuse """
+    def __init__(self):
+        self.cache = dict()  # Pool of SonicV2Connector objects
+
+    def initialize_connector(self, ns):
+        if not SonicDBConfig.isInit():
+            if multi_asic.is_multi_asic():
+                SonicDBConfig.load_sonic_global_db_config()
+            else:
+                SonicDBConfig.load_sonic_db_config()
+        return SonicV2Connector(namespace=ns, use_unix_socket_path=True)
+
+    def get(self, db_name, ns, update=False):
+        """ Returns a SonicV2Connector Object and caches it for further requests """
+        if ns not in self.cache:
+            self.cache[ns] = {}
+            self.cache[ns]["conn"] = self.initialize_connector(ns)
+            self.cache[ns]["connected_to"] = set()
+        if update or db_name not in self.cache[ns]["connected_to"]:
+            self.cache[ns]["conn"].connect(db_name)
+            self.cache[ns]["connected_to"].add(db_name)
+        return self.cache[ns]["conn"]
+
+    def clear(self, namespace=None):
+        if not namespace:
+            self.cache.clear()
+        elif namespace in self.cache:
+            del self.cache[namespace]
+
+
 class MatchEngine:
-    """ Pass in a MatchRequest, to fetch the Matched dump from the Data sources """
-    
+    """
+    Provide a MatchRequest to fetch the relevant keys/fv's from the data source
+    Usage Guidelines:
+    1) Instantiate the class once for the entire execution,
+                to effectively use the caching of redis connection objects
+    """
+    def __init__(self, pool=None):
+        if not isinstance(pool, ConnectionPool):
+            self.conn_pool = ConnectionPool()
+        else:
+            self.conn_pool = pool
+
+    def clear_cache(self, ns):
+        self.conn_pool(ns)
+
     def __get_source_adapter(self, req):
         src = None
         d_src = ""
         if req.db:
             d_src = req.db
-            src = RedisSource()
+            src = RedisSource(self.conn_pool)
         else:
             d_src = req.file
             src = JsonSource()
         return d_src, src
-    
+
     def __create_template(self):
-        return {"error" : "", "keys" : [], "return_values" : {}}
-    
+        return {"error": "", "keys": [], "return_values": {}}
+
     def __display_error(self, err):
         template = self.__create_template()
         template['error'] = err
         verbose_print("MatchEngine: \n" + template['error'])
         return template
-    
+
     def __filter_out_keys(self, src, req, all_matched_keys):
         # TODO: Custom Callbacks for Complex Matching Criteria
         if not req.field:
             return all_matched_keys
-        
+
         filtered_keys = []
         for key in all_matched_keys:
             f_values = src.hget(req.db, key, req.field)
+            if not f_values:
+                continue
             if "," in f_values and not req.match_entire_list:
                 f_value = f_values.split(",")
             else:
                 f_value = [f_values]
             if req.value in f_value:
                 filtered_keys.append(key)
         return filtered_keys
-        
+
     def __fill_template(self, src, req, filtered_keys, template):
         for key in filtered_keys:
             temp = {}
@@ -266,35 +313,34 @@ def __fill_template(self, src, req, filtered_keys, template):
             elif len(req.return_fields) > 0:
                 template["keys"].append(key)
                 template["return_values"][key] = {}
-                for field in req.return_fields: 
+                for field in req.return_fields:
                     template["return_values"][key][field] = src.hget(req.db, key, field)
             else:
                 template["keys"].append(key)
         verbose_print("Return Values:" + str(template["return_values"]))
         return template
-        
+
     def fetch(self, req):
         """ Given a request obj, find its match in the data source provided """
         if not isinstance(req, MatchRequest):
             return self.__display_error(EXCEP_DICT["INV_REQ"])
-        
+
         verbose_print(str(req))
-        
+
         if not req.key_pattern:
             return self.__display_error(EXCEP_DICT["NO_KEY"])
-        
+
         d_src, src = self.__get_source_adapter(req)
         if not src.connect(d_src, req.ns):
             return self.__display_error(EXCEP_DICT["CONN_ERR"])
-        
+
         template = self.__create_template()
         all_matched_keys = src.getKeys(req.db, req.table, req.key_pattern)
         if not all_matched_keys:
             return self.__display_error(EXCEP_DICT["NO_MATCHES"])
-        
+
         filtered_keys = self.__filter_out_keys(src, req, all_matched_keys)
         verbose_print("Filtered Keys:" + str(filtered_keys))
         if not filtered_keys:
             return self.__display_error(EXCEP_DICT["NO_ENTRIES"])
-        return self.__fill_template(src, req, filtered_keys, template)
-
+        return self.__fill_template(src, req, filtered_keys, template)