[chassis][voq] Add fabric capacity monitoring cmds (#3255)

What I did
Added fabric capacity monitoring related commands and the tests.

To config the warning threshold

~# config fabric
Usage: config fabric [OPTIONS] COMMAND [ARGS]...

  FABRIC-related configuration tasks

Options:
  -h, -?, --help  Show this message and exit.

Commands:
  monitor  FABRIC MONITOR configuration tasks
  port     FABRIC PORT configuration tasks
~# config fabric monitor
Usage: config fabric monitor [OPTIONS] COMMAND [ARGS]...

  FABRIC MONITOR configuration tasks

Options:
  -h, -?, --help  Show this message and exit.

Commands:
  capacity  FABRIC MONITOR CAPACITY configuration tasks
~# config fabric monitor capacity 
Usage: config fabric monitor capacity [OPTIONS] COMMAND [ARGS]...

  FABRIC MONITOR CAPACITY configuration tasks

Options:
  -?, -h, --help  Show this message and exit.

Commands:
  threshold  FABRIC CAPACITY MONITOR THRESHOLD configuration tasks
~# config fabric monitor capacity threshold 90
~#
To show the capacity :

~# show fabric 
Usage: show fabric [OPTIONS] COMMAND [ARGS]...

  Show fabric information

Options:
  -h, -?, --help  Show this message and exit.

Commands:
  counters      Show fabric port counters
  isolation     Show fabric isolation status
  monitor       Show fabric monitor
  reachability  Show fabric reachability
~# show fabric  monitor 
Usage: show fabric monitor [OPTIONS] COMMAND [ARGS]...

  Show fabric monitor

Options:
  -?, -h, --help  Show this message and exit.

Commands:
  capacity  Show fabric capacity
~# show fabric  monitor capacity
Monitored fabric capacity threshold:  90%

  ASIC    Operating     Total #        %    Last Event    Last Time
              Links    of Links
------  -----------  ----------  -------  ------------  -----------
 asic0           32         144  22.2222         Lower  0:19:17 ago
 asic1           28         144  19.4444         Lower  0:19:17 ago
 asic2           32         144  22.2222         Lower  0:19:17 ago
 asic3           28         144  19.4444         Lower  0:19:17 ago
 asic4           32         144  22.2222         Lower  0:19:17 ago
 asic5           28         144  19.4444         Lower  0:19:17 ago
 asic6           32         144  22.2222         Lower  0:19:19 ago
 asic7           28         144  19.4444         Lower  0:19:16 ago
 asic8           32         144  22.2222         Lower  0:19:16 ago
 asic9           28         144  19.4444         Lower  0:19:20 ago
asic10           31         144  21.5278         Lower  0:19:16 ago
asic11           28         144  19.4444         Lower  0:19:17 ago
~#
~# config fabric monitor capacity threshold 10
~# show fabric  monitor capacity
Monitored fabric capacity threshold:  10%

  ASIC    Operating     Total #        %    Last Event    Last Time
              Links    of Links
------  -----------  ----------  -------  ------------  -----------
 asic0           32         144  22.2222        Higher  0:00:03 ago
 asic1           28         144  19.4444        Higher  0:00:03 ago
 asic2           32         144  22.2222        Higher  0:00:03 ago
 asic3           28         144  19.4444        Higher  0:00:03 ago
 asic4           32         144  22.2222        Higher  0:00:03 ago
 asic5           28         144  19.4444        Higher  0:00:03 ago
 asic6           32         144  22.2222        Higher  0:00:05 ago
 asic7           28         144  19.4444        Higher  0:00:02 ago
 asic8           32         144  22.2222        Higher  0:00:02 ago
 asic9           28         144  19.4444        Higher  0:00:06 ago
asic10           31         144  21.5278        Higher  0:00:02 ago
asic11

Author: jfeng-arista

config/fabric.py

@@ -315,3 +315,45 @@ def recovery(pollcount, namespace):
         {"monPollThreshRecovery": pollcount})
 
 
+#
+# 'config fabric monitor ...'
+#
+@fabric.group(cls=clicommon.AbbreviationGroup, name='monitor')
+def capacity_monitor():
+    """FABRIC MONITOR configuration tasks"""
+    pass
+
+#
+# 'config fabric monitor capacity...'
+#
+@capacity_monitor.group(cls=clicommon.AbbreviationGroup)
+def capacity():
+    """FABRIC MONITOR CAPACITY configuration tasks"""
+    pass
+
+#
+# 'config fabric monitor capacity threshold <capcityThresh>'
+#
+@capacity.command()
+@click.argument('capacitythreshold', metavar='<capacityThreshold>', required=True, type=int)
+def threshold(capacitythreshold):
+    """FABRIC CAPACITY MONITOR THRESHOLD configuration tasks"""
+    ctx = click.get_current_context()
+
+    if capacitythreshold < 5 or capacitythreshold > 250:
+        ctx.fail("threshold must be in range 5...250")
+
+    namespaces = multi_asic.get_namespace_list()
+    for idx, namespace in enumerate(namespaces, start=1):
+        # Connect to config database
+        config_db = ConfigDBConnector(use_unix_socket_path=True, namespace=namespace)
+        config_db.connect()
+
+        # Make sure configuration data exists
+        monitorData = config_db.get_all(config_db.CONFIG_DB, "FABRIC_MONITOR|FABRIC_MONITOR_DATA")
+        if not bool(monitorData):
+            ctx.fail("Fabric monitor configuration data not present")
+
+        # Update entry
+        config_db.mod_entry("FABRIC_MONITOR", "FABRIC_MONITOR_DATA",
+            {"monCapacityThreshWarn": capacitythreshold})

scripts/fabricstat

@@ -1,19 +1,21 @@
 #!/usr/bin/env python3
 
 import argparse
-from collections import OrderedDict, namedtuple
+import click
 import json
 import os
 import sys
+import utilities_common.multi_asic as multi_asic_util
 
-from utilities_common import constants
-from utilities_common.cli import json_serial, UserCache
-from utilities_common.netstat import format_number_with_comma, table_as_json, ns_diff, format_prate
+from collections import OrderedDict, namedtuple
+from datetime import datetime, timezone, timedelta
 from natsort import natsorted
-from tabulate import tabulate
 from sonic_py_common import multi_asic
 from swsscommon.swsscommon import APP_FABRIC_PORT_TABLE_NAME, COUNTERS_TABLE, COUNTERS_FABRIC_PORT_NAME_MAP, COUNTERS_FABRIC_QUEUE_NAME_MAP
-import utilities_common.multi_asic as multi_asic_util
+from tabulate import tabulate
+from utilities_common import constants
+from utilities_common.cli import json_serial, UserCache
+from utilities_common.netstat import format_number_with_comma, table_as_json, ns_diff, format_prate
 
 # mock the redis for unit test purposes #
 try:
@@ -280,6 +282,62 @@ class FabricQueueStat(FabricStat):
         print(tabulate(table, queuestat_header, tablefmt='simple', stralign='right'))
         print()
 
+class FabricCapacity(FabricStat):
+    def __init__(self, namespace, table_cnt, threshold):
+        self.db = None
+        self.namespace = namespace
+        self.multi_asic = multi_asic_util.MultiAsic(constants.DISPLAY_ALL, namespace)
+        self.table_cnt = table_cnt
+        self.threshold = threshold
+
+    def capacity_print(self):
+        # Connect to database
+        self.db = multi_asic.connect_to_all_dbs_for_ns(self.namespace)
+        # Get fabric capacity data from STATE_DB table FABRIC_CAPACITY_TABLE
+        # and store them in fabric_capacity_data
+        fabric_capacity_data = self.db.get_all(self.db.STATE_DB, "FABRIC_CAPACITY_TABLE|FABRIC_CAPACITY_DATA")
+        operational_fap_capacity = 0
+        operational_fabric_capacity = 0
+        operational_fabric_links = 0;
+        total_fabric_links = 0;
+        ratio = 0
+        last_event = "None"
+        last_time = "Never"
+
+        # Get data from fabric_capacity_data
+        if "fabric_capacity" in fabric_capacity_data:
+            operational_fabric_capacity = int(fabric_capacity_data['fabric_capacity'])
+            operational_fabric_capacity = operational_fabric_capacity/1000.0
+        if "number_of_links" in fabric_capacity_data:
+            total_fabric_links = int(fabric_capacity_data['number_of_links'])
+        if "operating_links" in fabric_capacity_data:
+            operational_fabric_links = int(fabric_capacity_data['operating_links'])
+        if "warning_threshold" in fabric_capacity_data:
+            th = fabric_capacity_data['warning_threshold']
+            th = th + "%"
+            self.threshold.append(th)
+        if "last_event" in fabric_capacity_data:
+            last_event = fabric_capacity_data['last_event']
+        if "last_event_time" in fabric_capacity_data:
+            last_time = fabric_capacity_data['last_event_time']
+
+        # Calculate the ratio of number of operational links and all links
+        if total_fabric_links > 0:
+            ratio = operational_fabric_links/total_fabric_links*100
+
+        if last_time != "Never":
+            dt = datetime.fromtimestamp(int(last_time), timezone.utc)
+            td = datetime.now(timezone.utc) - dt
+            td_without_ms = timedelta(seconds=td.seconds)
+            last_time = str(td_without_ms) +" ago"
+
+        asic_name = "asic0"
+        if self.namespace:
+            asic_name = self.namespace
+
+        # Update the table to print
+        self.table_cnt.append((asic_name, operational_fabric_links, total_fabric_links, ratio, last_event, last_time))
+
 class FabricReachability(FabricStat):
     def reachability_print(self):
         # Connect to database
@@ -355,6 +413,8 @@ Examples:
     fabricstat -p -n asic0 -e
     fabricstat -q
     fabricstat -q -n asic0
+    fabricstat -c
+    fabricstat -c -n asic0
     fabricstat -C
     fabricstat -D
 """)
@@ -363,13 +423,15 @@ Examples:
     parser.add_argument('-r','--reachability', action='store_true', help='Display reachability, otherwise port stat')
     parser.add_argument('-n','--namespace', default=None, help='Display fabric ports counters for specific namespace')
     parser.add_argument('-e', '--errors', action='store_true', help='Display errors')
+    parser.add_argument('-c','--capacity',action='store_true', help='Display fabric capacity')
     parser.add_argument('-i','--isolation', action='store_true', help='Display fabric ports isolation status')
     parser.add_argument('-C','--clear', action='store_true', help='Copy & clear fabric counters')
     parser.add_argument('-D','--delete', action='store_true', help='Delete saved stats')
 
     args = parser.parse_args()
     queue = args.queue
     reachability = args.reachability
+    capacity_status = args.capacity
     isolation_status = args.isolation
     namespace = args.namespace
     errors_only = args.errors
@@ -410,14 +472,36 @@ Examples:
         else:
             stat.cnstat_print(cnstat_dict, errors_only)
 
-    if namespace is None:
-        # All asics or all fabric asics
-        multi_asic = multi_asic_util.MultiAsic()
-        for ns in multi_asic.get_ns_list_based_on_options():
-            nsStat(ns, errors_only)
+    if capacity_status:
+        # show fabric capacity command
+        capacity_header = []
+        table_cnt = []
+        threshold = []
+        capacity_header = ["ASIC", "Operating\nLinks", "Total #\nof Links", "%", "Last Event", "Last Time"]
+        if namespace is None:
+            # All asics or all fabric asics
+            multi_asic = multi_asic_util.MultiAsic()
+            for ns in multi_asic.get_ns_list_based_on_options():
+                stat = FabricCapacity(ns, table_cnt, threshold)
+                stat.capacity_print()
+        else:
+            # Asic with namespace
+            stat = FabricCapacity(namespace, table_cnt, threshold)
+            stat.capacity_print()
+
+        click.echo("Monitored fabric capacity threshold: {}".format(threshold[0]))
+        click.echo()
+        click.echo(tabulate(table_cnt, capacity_header, tablefmt='simple', stralign='right'))
     else:
-        # Asic with namespace
-        nsStat(namespace, errors_only)
+        # other show fabric commands
+        if namespace is None:
+            # All asics or all fabric asics
+            multi_asic = multi_asic_util.MultiAsic()
+            for ns in multi_asic.get_ns_list_based_on_options():
+                nsStat(ns, errors_only)
+        else:
+            # Asic with namespace
+            nsStat(namespace, errors_only)
 
 if __name__ == "__main__":
     main()

show/fabric.py

@@ -13,6 +13,23 @@ def counters():
     """Show fabric port counters"""
     pass
 
+@fabric.group(cls=clicommon.AliasedGroup)
+def monitor():
+    """Show fabric monitor"""
+    pass
+
+@monitor.group(invoke_without_command=True)
+@multi_asic_util.multi_asic_click_option_namespace
+@click.option('-e', '--errors', is_flag=True)
+def capacity(namespace, errors):
+    """Show fabric capacity"""
+    cmd = ['fabricstat', '-c']
+    if namespace is not None:
+        cmd += ['-n', str(namespace)]
+    if errors:
+        cmd += ['-e']
+    clicommon.run_command(cmd)
+
 @fabric.group(invoke_without_command=True)
 @multi_asic_util.multi_asic_click_option_namespace
 @click.option('-e', '--errors', is_flag=True)

tests/config_fabric_test.py

@@ -118,6 +118,18 @@ def test_config_fabric_monitor_state(self, ctx):
         expect_result = 0
         assert operator.eq(result.exit_code, expect_result)
 
+    def test_config_capacity(self, ctx):
+        # Issue command "config fabric monitor capacity threshold 90",
+        # check if the result is expected.
+        result = self.basic_check("monitor", ["capacity", "threshold", "90"], ctx)
+        expect_result=0
+        assert operator.eq(result.exit_code, expect_result)
+
+        # Issue command "config fabric monitor capacity threshold 3",
+        # check if the result has the warning message.
+        result = self.basic_check("monitor", ["capacity", "threshold", "3"], ctx)
+        assert "threshold must be in range 5...250" in result.output
+
     @classmethod
     def teardown_class(cls):
         print("TEARDOWN")
@@ -152,6 +164,18 @@ def test_multi_config_fabric_monitor_state(self, ctx):
         expect_result = 0
         assert operator.eq(result.exit_code, expect_result)
 
+    def test_config_capacity_multi(self, ctx):
+        # Issue command "config fabric monitor capacity threshold 80",
+        # check if the result is expected.
+        result = self.basic_check("monitor", ["capacity", "threshold", "80"], ctx)
+        expect_result=0
+        assert operator.eq(result.exit_code, expect_result)
+
+        # Issue command "config fabric monitor capacity threshold 4",
+        # check if the result has the warning message.
+        result = self.basic_check("monitor", ["capacity", "threshold", "4"], ctx)
+        assert "threshold must be in range 5...250" in result.output
+
     @classmethod
     def teardown_class(cls):
         print("TEARDOWN_TEST")

tests/fabricstat_test.py

@@ -151,6 +151,25 @@
            7                0             93        up
 """
 
+multi_asic_fabric_capacity = """\
+Monitored fabric capacity threshold: 100%
+
+  ASIC    Operating     Total #     %    Last Event    Last Time
+              Links    of Links
+------  -----------  ----------  ----  ------------  -----------
+ asic0            5           8  62.5          None        Never
+ asic1            2           8  25            None        Never
+"""
+
+multi_asic_fabric_capacity_asic0 = """\
+Monitored fabric capacity threshold: 100%
+
+  ASIC    Operating     Total #     %    Last Event    Last Time
+              Links    of Links
+------  -----------  ----------  ----  ------------  -----------
+ asic0            5           8  62.5          None        Never
+"""
+
 multi_asic_fabric_isolation = """\
 
 asic0
@@ -181,7 +200,6 @@
            7                0                  0           0
 """
 
-
 class TestFabricStat(object):
     @classmethod
     def setup_class(cls):
@@ -302,6 +320,20 @@ def test_multi_show_fabric_reachability_asic(self):
         assert return_code == 0
         assert result == multi_asic_fabric_reachability_asic0
 
+    def test_mutli_show_fabric_capacity(self):
+        return_code, result = get_result_and_return_code(['fabricstat', '-c'])
+        print("return_code: {}".format(return_code))
+        print("result = {}".format(result))
+        assert return_code == 0
+        assert result == multi_asic_fabric_capacity
+
+    def test_multi_show_fabric_capacity_asic(self):
+        return_code, result = get_result_and_return_code(['fabricstat', '-c', '-n', 'asic0'])
+        print("return_code: {}".format(return_code))
+        print("result = {}".format(result))
+        assert return_code == 0
+        assert result == multi_asic_fabric_capacity_asic0
+
     def test_multi_show_fabric_isolation(self):
         return_code, result = get_result_and_return_code(['fabricstat', '-i'])
         print("return_code: {}".format(return_code))

tests/mock_tables/asic0/state_db.json

@@ -304,5 +304,15 @@
     },
     "ACL_RULE_TABLE|DATAACL_5|RULE_1" : {
         "status": "Active"
+    },
+    "FABRIC_CAPACITY_TABLE|FABRIC_CAPACITY_DATA" : {
+        "fabric_capacity": "221580",
+        "missing_capacity": "132948",
+        "operating_links": "5",
+        "number_of_links": "8",
+        "warning_threshold": "100"
+    },
+    "PORT_CAPACITY_TABLE|PORT_CAPACITY_DATA" : {
+        "capacity": "80000"
     }
 }

tests/mock_tables/asic1/state_db.json

@@ -270,5 +270,15 @@
     },
     "FABRIC_PORT_TABLE|PORT7" : {
         "STATUS": "down"
+    },
+    "FABRIC_CAPACITY_TABLE|FABRIC_CAPACITY_DATA" : {
+        "fabric_capacity": "88632",
+        "missing_capacity": "265896",
+        "operating_links": "2",
+        "number_of_links": "8",
+        "warning_threshold": "100"
+    },
+    "PORT_CAPACITY_TABLE|PORT_CAPACITY_DATA" : {
+        "capacity": "80000"
     }
 }

tests/mock_tables/config_db.json

@@ -2703,6 +2703,7 @@
         "direction": "RX"
     },
     "FABRIC_MONITOR|FABRIC_MONITOR_DATA": {
+        "monCapacityThreshWarn": "100",
         "monErrThreshCrcCells": "1",
         "monErrThreshRxCells": "61035156",
         "monPollThreshIsolation": "1",

tests/mock_tables/state_db.json

@@ -1671,5 +1671,15 @@
         "minimum_current": "339",
         "timestamp": "20230704 17:38:04",
         "warning_status": "False"
+    },
+    "FABRIC_CAPACITY_TABLE|FABRIC_CAPACITY_DATA" : {
+        "fabric_capacity": "88632",
+        "missing_capacity": "265896",
+        "operating_links": "2",
+        "number_of_links": "8",
+        "warning_threshold": "100"
+    },
+    "PORT_CAPACITY_TABLE|PORT_CAPACITY_DATA" : {
+        "capacity": "80000"
     }
 }