sonic-yang-mgmt: uses clause with leaf-list not honored (PR sonic-net#21907)

When a uses clause imports a grouping, it was only processing leaf
entries and ignoring leaf-list.  That means that for instance in
bgp route maps, route_map_in and route_map_out validations would
fail.

This fixes that behavior and adds test cases to ensure it doesn't
regress in the future.

Signed-off-by: Brad House (@bradh352)

Author: bradh352

src/sonic-yang-mgmt/sonic_yang_ext.py

@@ -70,6 +70,8 @@ def loadYangModel(self):
             self._loadJsonYangModel()
             # create a map from config DB table to yang container
             self._createDBTableToModuleMap()
+            # compile uses clause (embed into schema)
+            self._compileUsesClause()
         except Exception as e:
             self.sysLog(msg="Yang Models Load failed:{}".format(str(e)), \
                 debug=syslog.LOG_ERR, doPrint=True)
@@ -128,8 +130,9 @@ def _preProcessYangGrouping(self, moduleName, module):
 
             for grouping in groupings:
                 gName = grouping["@name"]
-                gLeaf = grouping["leaf"]
-                self.preProcessedYang['grouping'][moduleName][gName] = gLeaf
+                self.preProcessedYang['grouping'][moduleName][gName] = dict()
+                self.preProcessedYang['grouping'][moduleName][gName]["leaf"] = grouping.get('leaf')
+                self.preProcessedYang['grouping'][moduleName][gName]["leaf-list"] = grouping.get('leaf-list')
 
         except Exception as e:
             self.sysLog(msg="_preProcessYangGrouping failed:{}".format(str(e)), \
@@ -160,6 +163,78 @@ def _preProcessYang(self, moduleName, module):
             raise e
         return
 
+    def _compileUsesClauseList(self, model, group, name):
+        # If group doesn't have this entry, nothing to do.
+        if not group.get(name):
+            return
+
+        # model doesn't have this entry type, create it as a list
+        if not model.get(name):
+            model[name] = []
+
+        # model has this entry type, but its not a list, convert
+        if not isinstance(model.get(name), list):
+            model[name] = [ model[name] ]
+
+        if isinstance(group.get(name), list):
+            model[name].extend(group.get(name))
+        else:
+            model[name].append(group.get(name))
+
+    # Recursively process the yang schema looking for the "uses" clause under
+    # "container" and "list" nodes.  Merge in the "uses" dictionaries for leaf
+    # and leaf-list so callers don't need to try to do their own "uses"
+    # processing.  Remove the "uses" member when processed so anyone expecting
+    # it won't try to re-process.  It will just look like a yang model that
+    # doesn't use "uses" so shouldn't cause compatibility issues.
+    def _compileUsesClauseModel(self, module, model):
+        if isinstance(model, list):
+            for item in model:
+                self._compileUsesClauseModel(module, item)
+            return
+
+        node = model.get("container")
+        if node:
+            self._compileUsesClauseModel(module, node)
+
+        node = model.get("list")
+        if node:
+            self._compileUsesClauseModel(module, node)
+
+        uses_s = model.get("uses")
+        if not uses_s:
+            return
+
+        # Always make as a list
+        if isinstance(uses_s, dict):
+            uses_s = [uses_s]
+
+        # uses Example: "@name": "bgpcmn:sonic-bgp-cmn"
+        for uses in uses_s:
+            # Assume ':'  means reference to another module
+            if ':' in uses['@name']:
+                prefix = uses['@name'].split(':')[0].strip()
+                uses_module_name = self._findYangModuleFromPrefix(prefix, module)
+            else:
+                uses_module_name = module['@name']
+            grouping = uses['@name'].split(':')[-1].strip()
+            groupdata = self.preProcessedYang['grouping'][uses_module_name][grouping]
+
+            # Merge leaf from uses
+            self._compileUsesClauseList(model, groupdata, 'leaf')
+            self._compileUsesClauseList(model, groupdata, 'leaf-list')
+
+        # Delete the uses node so callers don't use it.
+        del model["uses"]
+
+    def _compileUsesClause(self):
+        try:
+            for module in self.yJson:
+                self._compileUsesClauseModel(module["module"], module["module"])
+        except Exception as e:
+            traceback.print_exc()
+            raise e
+
     """
     Create a map from config DB tables to container in yang model
     This module name and topLevelContainer are fetched considering YANG models are
@@ -320,44 +395,6 @@ def _findYangModuleFromPrefix(self, prefix, module):
             raise e
         return None
 
-    def _fillLeafDictUses(self, uses_s, table, leafDict):
-        '''
-            Find the leaf(s) in a grouping which maps to given uses statement,
-            then fill leafDict with leaf(s) information.
-
-            Parameters:
-                uses_s (str): uses statement in yang module.
-                table (str): config DB table, this table is being translated.
-                leafDict (dict): dict with leaf(s) information for List\Container
-                    corresponding to config DB table.
-
-            Returns:
-                 (void)
-        '''
-        try:
-            # make a list
-            if isinstance(uses_s, dict):
-                uses_s = [uses_s]
-            # find yang module for current table
-            table_module = self.confDbYangMap[table]['yangModule']
-            # uses Example: "@name": "bgpcmn:sonic-bgp-cmn"
-            for uses in uses_s:
-                # Assume ':'  means reference to another module
-                if ':' in uses['@name']:
-                    prefix = uses['@name'].split(':')[0].strip()
-                    uses_module_name = self._findYangModuleFromPrefix(prefix, table_module)
-                else:
-                    uses_module_name = table_module['@name']
-                grouping = uses['@name'].split(':')[-1].strip()
-                leafs = self.preProcessedYang['grouping'][uses_module_name][grouping]
-                self._fillLeafDict(leafs, leafDict)
-        except Exception as e:
-            self.sysLog(msg="_fillLeafDictUses failed:{}".format(str(e)), \
-                debug=syslog.LOG_ERR, doPrint=True)
-            raise e
-
-        return
-
     def _createLeafDict(self, model, table):
         '''
             create a dict to map each key under primary key with a leaf in yang model.
@@ -394,10 +431,6 @@ def _createLeafDict(self, model, table):
         # leaf-lists
         self._fillLeafDict(model.get('leaf-list'), leafDict, True)
 
-        # uses should map to grouping,
-        if model.get('uses') is not None:
-            self._fillLeafDictUses(model.get('uses'), table, leafDict)
-
         return leafDict
 
     """

src/sonic-yang-models/tests/files/sample_config_db.json

@@ -1874,6 +1874,8 @@
         },
         "BGP_PEER_GROUP_AF": {
             "default|PG1|ipv4_unicast": {
+                "route_map_in": [ "map1" ],
+                "route_map_out": [ "map1" ]
             }
         },
         "BGP_GLOBALS_LISTEN_PREFIX": {

src/sonic-yang-models/tests/yang_model_tests/tests/bgp.json

@@ -90,6 +90,9 @@
         "eStrKey" : "InvalidValue",
         "eStr": ["send_community"]
     },
+    "BGP_NEIGHBOR_AF_ROUTE_MAP_VALID": {
+        "desc": "Reference valid route map"
+    },
     "BGP_NEIGHBOR_AF_NEG_ROUTE_MAP_NOT_EXIST": {
         "desc": "Reference to non-existent route-map.",
         "eStrKey" : "LeafRef"
@@ -134,6 +137,9 @@
         "eStrKey" : "InvalidValue",
         "eStr": ["send_community"]
     },
+    "BGP_PEERGROUP_AF_ROUTE_MAP_VALID": {
+        "desc": "Validate route-map reference in peergroup AF."
+    },
     "BGP_PEERGROUP_AF_ROUTE_MAP_NOT_EXIST": {
         "desc": "Missing or non-existent route-map reference in peergroup AF.",
         "eStrKey" : "LeafRef"

src/sonic-yang-models/tests/yang_model_tests/tests_config/bgp.json

@@ -948,6 +948,62 @@
         }
     },
 
+    "BGP_NEIGHBOR_AF_ROUTE_MAP_VALID": {
+        "sonic-route-map:sonic-route-map": {
+            "sonic-route-map:ROUTE_MAP_SET": {
+                "ROUTE_MAP_SET_LIST": [
+                    {
+                        "name": "ALLOW"
+                    }
+                ]
+            },
+            "sonic-route-map:ROUTE_MAP": {
+                "ROUTE_MAP_LIST": [
+                {
+                    "name": "ALLOW",
+                    "stmt_name": 1,
+                    "route_operation": "permit"
+                }
+                ]
+            }
+        },
+        "sonic-bgp-global:sonic-bgp-global": {
+            "sonic-bgp-global:BGP_GLOBALS": {
+                "BGP_GLOBALS_LIST": [
+                    {
+                        "vrf_name": "default",
+                        "local_asn": 65001
+                    }
+                ]
+            }
+        },
+        "sonic-bgp-neighbor:sonic-bgp-neighbor": {
+            "sonic-bgp-neighbor:BGP_NEIGHBOR": {
+                "BGP_NEIGHBOR_LIST": [
+                    {
+                        "vrf_name": "default",
+                        "neighbor": "20.0.0.1",
+                        "local_asn": 1,
+                        "name": "BGP nbr 1",
+                        "asn": 65003
+                    }
+                ]
+            },
+            "sonic-bgp-neighbor:BGP_NEIGHBOR_AF": {
+                "BGP_NEIGHBOR_AF_LIST": [
+                    {
+                        "vrf_name": "default",
+                        "neighbor": "20.0.0.1",
+                        "afi_safi": "ipv4_unicast",
+                        "admin_status": "up",
+                        "route_map_in": [ "ALLOW" ],
+                        "route_map_out": [ "ALLOW" ]
+                    }
+                ]
+            }
+        }
+    },
+
     "BGP_NEIGHBOR_AF_NEG_INVALID_MAP_IN": {
         "sonic-route-map:sonic-route-map": {
             "sonic-route-map:ROUTE_MAP_SET": {
@@ -1306,6 +1362,62 @@
         }
     },
 
+    "BGP_PEERGROUP_AF_ROUTE_MAP_VALID": {
+         "sonic-route-map:sonic-route-map": {
+            "sonic-route-map:ROUTE_MAP_SET": {
+                "ROUTE_MAP_SET_LIST": [
+                    {
+                        "name": "ALLOW"
+                    }
+                ]
+            },
+            "sonic-route-map:ROUTE_MAP": {
+                "ROUTE_MAP_LIST": [
+                {
+                    "name": "ALLOW",
+                    "stmt_name": 1,
+                    "route_operation": "permit"
+                }
+                ]
+            }
+        },
+        "sonic-bgp-global:sonic-bgp-global": {
+            "sonic-bgp-global:BGP_GLOBALS": {
+                "BGP_GLOBALS_LIST": [
+                    {
+                        "vrf_name": "default",
+                        "local_asn": 65001
+                    }
+                ]
+            }
+        },
+        "sonic-bgp-peergroup:sonic-bgp-peergroup": {
+            "sonic-bgp-peergroup:BGP_PEER_GROUP": {
+                "BGP_PEER_GROUP_LIST": [
+                    {
+                        "vrf_name": "default",
+                        "peer_group_name": "PG1",
+                        "local_asn": 1,
+                        "name": "BGP PeerGroup 01",
+                        "asn": 65003
+                    }
+                ]
+            },
+            "sonic-bgp-peergroup:BGP_PEER_GROUP_AF": {
+                "BGP_PEER_GROUP_AF_LIST": [
+                    {
+                        "vrf_name": "default",
+                        "peer_group_name": "PG1",
+                        "afi_safi": "ipv4_unicast",
+                        "admin_status": "up",
+                        "route_map_in": [ "ALLOW" ],
+                        "route_map_out": [ "ALLOW" ]
+                    }
+                ]
+            }
+        }
+    },
+
     "BGP_PEERGROUP_AF_ROUTE_MAP_NOT_EXIST": {
         "sonic-bgp-global:sonic-bgp-global": {
             "sonic-bgp-global:BGP_GLOBALS": {