SONiC QoS MAPs and PORT_QOS_MAP Yang (sonic-net#7375)

Why I did it
Created SONiC Yang model for the following QOS MAPs and PORT QOS MAP:
DSCP_TO_TC_MAP
DOT1P_TO_TC_MAP
TC_TO_QUEUE_MAP
TC_TO_PRIORITY_GROUP_MAP
MAP_PFC_PRIORITY_TO_QUEUE
PORT_QOS_MAP

How I did it
Defined Yang models for QOS MAPs based on Guideline doc:
https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md
and
https://github.com/Azure/sonic-utilities/blob/master/doc/Command-Reference.md

How to verify it
sonic_yang_models package build
The infra code was modified to handle the nested list definition in YANG and translate it to a flat list in DB.

Author: ohu1

src/sonic-yang-mgmt/sonic_yang_ext.py

@@ -4,11 +4,17 @@
 from __future__ import print_function
 import yang as ly
 import syslog
-
 from json import dump, dumps, loads
 from xmltodict import parse
 from glob import glob
 
+qos_maps_model = ['DSCP_TO_TC_MAP_LIST',
+            'DOT1P_TO_TC_MAP_LIST',
+            'TC_TO_PRIORITY_GROUP_MAP_LIST',
+            'TC_TO_QUEUE_MAP_LIST',
+            'MAP_PFC_PRIORITY_TO_QUEUE_LIST',
+            'PFC_PRIORITY_TO_PRIORITY_GROUP_MAP_LIST']
+
 """
 This is the Exception thrown out of all public function of this class.
 """
@@ -406,6 +412,108 @@ def _yangConvert(val):
 
         return vValue
 
+    """
+    Xlate a Qos Maps list
+    This function will xlate from a dict in config DB to a Yang JSON list
+    using yang model. Output will be go in self.xlateJson
+
+    Note: Exceptions from this function are collected in exceptionList and
+    are displayed only when an entry is not xlated properly from ConfigDB
+    to sonic_yang.json.
+
+    QOS MAPS Yang has inner list, which is diffrent from config DB.
+    Each field value in config db should be converted to inner list with
+    key and value.
+    Example:
+
+    Config DB:
+    "DSCP_TO_TC_MAP": {
+       "Dscp_to_tc_map1": {
+          "1": "1",
+          "2": "2"
+       }
+    }
+
+    YANG Model:
+    module: sonic-dscp-tc-map
+     +--rw sonic-dscp-tc-map
+     +--rw DSCP_TO_TC_MAP
+        +--rw DSCP_TO_TC_MAP_LIST* [name]
+           +--rw name              string
+           +--rw DSCP_TO_TC_MAP* [dscp]
+              +--rw dscp    string
+              +--rw tc?     string
+
+    YANG JSON:
+    "sonic-dscp-tc-map:sonic-dscp-tc-map": {
+        "sonic-dscp-tc-map:DSCP_TO_TC_MAP": {
+             "DSCP_TO_TC_MAP_LIST": [
+                   {
+                        "name": "map3",
+                        "DSCP_TO_TC_MAP": [
+                            {
+                                "dscp": "64",
+                                "tc": "1"
+                            },
+                            {
+                                "dscp":"2",
+                                "tc":"2"
+                            }
+                        ]
+                    }
+                ]
+            }
+    }
+    """
+    def _xlateQosMapList(self, model, yang, config, table, exceptionList):
+
+        #create a dict to map each key under primary key with a dict yang model.
+        #This is done to improve performance of mapping from values of TABLEs in
+        #config DB to leaf in YANG LIST.
+        inner_clist = model.get('list')
+        if inner_clist:
+            inner_listKey = inner_clist['key']['@value']
+            inner_leafDict = self._createLeafDict(inner_clist, table)
+            for lkey in inner_leafDict:
+                if inner_listKey != lkey:
+                    inner_listVal = lkey
+
+        # get keys from YANG model list itself
+        listKeys = model['key']['@value']
+        self.sysLog(msg="xlateList keyList:{}".format(listKeys))
+        primaryKeys = list(config.keys())
+        for pkey in primaryKeys:
+            try:
+                vKey = None
+                self.sysLog(syslog.LOG_DEBUG, "xlateList Extract pkey:{}".\
+                    format(pkey))
+                # Find and extracts key from each dict in config
+                keyDict = self._extractKey(pkey, listKeys)
+
+                if inner_clist:
+                   inner_yang_list = list()
+                   for vKey in config[pkey]:
+                      inner_keyDict = dict()
+                      self.sysLog(syslog.LOG_DEBUG, "xlateList Key {} vkey {} Val {} vval {}".\
+                          format(inner_listKey, str(vKey), inner_listVal, str(config[pkey][vKey])))
+                      inner_keyDict[inner_listKey] = str(vKey)
+                      inner_keyDict[inner_listVal] = str(config[pkey][vKey])
+                      inner_yang_list.append(inner_keyDict)
+
+                keyDict[inner_clist['@name']] = inner_yang_list
+                yang.append(keyDict)
+                # delete pkey from config, done to match one key with one list
+                del config[pkey]
+
+            except Exception as e:
+                # log debug, because this exception may occur with multilists
+                self.sysLog(msg="xlateList Exception:{}".format(str(e)), \
+                    debug=syslog.LOG_DEBUG, doPrint=True)
+                exceptionList.append(str(e))
+                # with multilist, we continue matching other keys.
+                continue
+        return
+
     """
     Xlate a list
     This function will xlate from a dict in config DB to a Yang JSON list
@@ -416,16 +524,22 @@ def _yangConvert(val):
     to sonic_yang.json.
     """
     def _xlateList(self, model, yang, config, table, exceptionList):
+        
+        #Qos Map lists needs special handling because of inner yang list and
+        #config db format.
+        if model['@name'] in qos_maps_model:
+            self.sysLog(msg="_xlateQosMapList: {}".format(model['@name']))
+            self._xlateQosMapList(model, yang,config, table, exceptionList)
+            return
 
         #create a dict to map each key under primary key with a dict yang model.
         #This is done to improve performance of mapping from values of TABLEs in
         #config DB to leaf in YANG LIST.
-        leafDict = self._createLeafDict(model, table)
 
+        leafDict = self._createLeafDict(model, table)
         # get keys from YANG model list itself
         listKeys = model['key']['@value']
         self.sysLog(msg="xlateList keyList:{}".format(listKeys))
-
         primaryKeys = list(config.keys())
         for pkey in primaryKeys:
             try:
@@ -459,7 +573,6 @@ def _xlateList(self, model, yang, config, table, exceptionList):
     """
     def _xlateListInContainer(self, model, yang, configC, table, exceptionList):
         clist = model
-        #print(clist['@name'])
         yang[clist['@name']] = list()
         self.sysLog(msg="xlateProcessListOfContainer: {}".format(clist['@name']))
         self._xlateList(clist, yang[clist['@name']], configC, table, exceptionList)
@@ -629,10 +742,93 @@ def _revYangConvert(val):
 
         return vValue
 
+    """
+    Rev xlate from <TABLE>_LIST to table in config DB
+    QOS MAP Yang has inner list, each inner list key:val should
+    be mapped to field:value in Config DB.
+    Example:
+
+    YANG:
+    module: sonic-dscp-tc-map
+    +--rw sonic-dscp-tc-map
+     +--rw DSCP_TO_TC_MAP
+        +--rw DSCP_TO_TC_MAP_LIST* [name]
+           +--rw name              string
+           +--rw DSCP_TO_TC_MAP* [dscp]
+              +--rw dscp    string
+              +--rw tc?     string
+
+    YANG JSON:
+    "sonic-dscp-tc-map:sonic-dscp-tc-map": {
+            "sonic-dscp-tc-map:DSCP_TO_TC_MAP": {
+                "DSCP_TO_TC_MAP_LIST": [
+                    {
+                        "name": "map3",
+                        "DSCP_TO_TC_MAP": [
+                            {
+                                "dscp": "64",
+                                "tc": "1"
+                            },
+                            {
+                                "dscp":"2",
+                                "tc":"2"
+                            }
+                        ]
+                    }
+                ]
+            }
+        }
+
+    Config DB:
+    "DSCP_TO_TC_MAP": {
+       "Dscp_to_tc_map1": {
+          "1": "1",
+          "2": "2"
+       }
+    }
+    """
+
+    def _revQosMapXlateList(self, model, yang, config, table):
+        # get keys from YANG model list itself
+        listKeys = model['key']['@value']
+        # create a dict to map each key under primary key with a dict yang model.
+        # This is done to improve performance of mapping from values of TABLEs in
+        # config DB to leaf in YANG LIST.
+
+        # Gather inner list key and value from model  
+        inner_clist = model.get('list')
+        if inner_clist:
+            inner_listKey = inner_clist['key']['@value']
+            inner_leafDict = self._createLeafDict(inner_clist, table)
+            for lkey in inner_leafDict:
+                if inner_listKey != lkey:
+                    inner_listVal = lkey
+
+        # list with name <NAME>_LIST should be removed,
+        if "_LIST" in model['@name']:
+            for entry in yang:
+                # create key of config DB table
+                pkey, pkeydict = self._createKey(entry, listKeys)
+                self.sysLog(syslog.LOG_DEBUG, "revXlateList pkey:{}".format(pkey))
+                config[pkey]= dict()
+                # fill rest of the entries
+                inner_list = entry[inner_clist['@name']]
+                for index in range(len(inner_list)):
+                    self.sysLog(syslog.LOG_DEBUG, "revXlateList fkey:{} fval {}".\
+                         format(str(inner_list[index][inner_listKey]),\
+                             str(inner_list[index][inner_listVal])))
+                    config[pkey][str(inner_list[index][inner_listKey])] = str(inner_list[index][inner_listVal])
+        return
+
     """
     Rev xlate from <TABLE>_LIST to table in config DB
     """
     def _revXlateList(self, model, yang, config, table):
+        
+        # special processing for QOS Map table.
+        if model['@name'] in qos_maps_model:
+           self._revQosMapXlateList(model, yang, config, table)
+           return
 
         # get keys from YANG model list itself
         listKeys = model['key']['@value']

src/sonic-yang-models/setup.py

@@ -79,6 +79,13 @@
                          './yang-models/sonic-scheduler.yang',
                          './yang-models/sonic-wred-profile.yang',
                          './yang-models/sonic-queue.yang',
+                         './yang-models/sonic-dscp-tc-map.yang',
+                         './yang-models/sonic-dot1p-tc-map.yang',
+                         './yang-models/sonic-tc-priority-group-map.yang',
+                         './yang-models/sonic-tc-queue-map.yang',
+                         './yang-models/sonic-pfc-priority-queue-map.yang',
+                         './yang-models/sonic-pfc-priority-priority-group-map.yang',
+                         './yang-models/sonic-port-qos-map.yang',
                          './yang-models/sonic_yang_tree']),
     ],
     zip_safe=False,

src/sonic-yang-models/tests/files/sample_config_db.json

@@ -1188,6 +1188,93 @@
                 "scheduler": "TEST@1",
                 "wred_profile": "Wred1"
             }
+         },		
+
+        "DSCP_TO_TC_MAP": {
+           "Dscp_to_tc_map1": { 
+              "1": "1",
+              "2": "2"
+            }, 
+           "Dscp_to_tc_map2": { 
+              "3": "3",
+              "4": "4"
+            }
+        },
+
+        "DOT1P_TO_TC_MAP": {
+           "Dot1p_to_tc_map1": { 
+              "1": "1",
+              "2": "2"
+            }, 
+           "Dot1p_to_tc_map2": { 
+              "3": "3",
+              "4": "4"
+            }
+        },
+
+        "TC_TO_PRIORITY_GROUP_MAP": {
+           "tc_to_pg_map1": { 
+              "1": "1",
+              "2": "2"
+            }, 
+           "tc_to_pg_map2": { 
+              "3": "3",
+              "4": "4"
+            }
+        },
+
+        "TC_TO_QUEUE_MAP": {
+           "tc_to_q_map1": { 
+              "1": "1",
+              "2": "2"
+            }, 
+           "tc_to_q_map2": { 
+              "3": "3",
+              "4": "4"
+            }
+        },
+
+        "MAP_PFC_PRIORITY_TO_QUEUE": {
+           "pfc_prio_to_q_map1": { 
+              "1": "1",
+              "2": "2"
+           }, 
+           "pfc_prio_to_q_map2": { 
+              "3": "3",
+              "4": "4"
+            }
+        },
+
+        "PFC_PRIORITY_TO_PRIORITY_GROUP_MAP": {
+           "pfc_prio_to_pg_map1": { 
+              "1": "1",
+              "2": "2"
+            }, 
+            "pfc_prio_to_pg_map2": { 
+              "3": "3",
+              "4": "4"
+            }
+        },
+
+        "PORT_QOS_MAP": {
+           "Ethernet0": { 
+              "dot1p_to_tc_map" : "Dot1p_to_tc_map1",
+              "dscp_to_tc_map": "Dscp_to_tc_map1",
+	          "tc_to_queue_map": "tc_to_q_map1",
+	          "tc_to_pg_map": "tc_to_pg_map1",
+	          "pfc_to_queue_map": "pfc_prio_to_q_map1",
+	          "pfc_to_pg_map"   : "pfc_prio_to_pg_map1",
+	          "pfc_enable"      : "3,4"
+            }, 
+            "Ethernet4": { 
+              "dot1p_to_tc_map" : "Dot1p_to_tc_map2",
+              "dscp_to_tc_map": "Dscp_to_tc_map2",
+	          "tc_to_queue_map": "tc_to_q_map2",
+	          "tc_to_pg_map": "tc_to_pg_map2",
+	          "pfc_to_queue_map": "pfc_prio_to_q_map2",
+	          "pfc_to_pg_map"   : "pfc_prio_to_pg_map2",
+	          "pfc_enable"      : "3,4"
+            }
         }
     },