Merge branch 'master' into dev-reset-local-users-password

Author: azmyali98

azure-pipelines.yml

@@ -43,14 +43,15 @@ stages:
       vmImage: ubuntu-20.04
 
     container:
-      image: sonicdev-microsoft.azurecr.io:443/sonic-slave-bullseye:$(BUILD_BRANCH)
+      image: sonicdev-microsoft.azurecr.io:443/sonic-slave-bookworm:$(BUILD_BRANCH)
 
     steps:
     - script: |
         set -ex
         sudo apt-get update
         sudo apt-get install -y python3-pip
         sudo pip3 install requests==2.31.0
+        sudo apt-get install -y python3-protobuf
       displayName: "Install dependencies"
 
     - script: |
@@ -84,15 +85,15 @@ stages:
         sudo dpkg -i libyang_1.0.73_amd64.deb
         sudo dpkg -i libyang-cpp_1.0.73_amd64.deb
         sudo dpkg -i python3-yang_1.0.73_amd64.deb
-      workingDirectory: $(Pipeline.Workspace)/target/debs/bullseye/
+      workingDirectory: $(Pipeline.Workspace)/target/debs/bookworm/
       displayName: 'Install Debian dependencies'
 
     - task: DownloadPipelineArtifact@2
       inputs:
         source: specific
         project: build
         pipeline: 9
-        artifact: sonic-swss-common
+        artifact: sonic-swss-common-bookworm
         runVersion: 'latestFromBranch'
         runBranch: 'refs/heads/$(sourceBranch)'
       displayName: "Download sonic swss common deb packages"
@@ -104,6 +105,27 @@ stages:
       workingDirectory: $(Pipeline.Workspace)/
       displayName: 'Install swss-common dependencies'
 
+
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        source: specific
+        project: build
+        pipeline: sonic-net.sonic-dash-api
+        artifact: sonic-dash-api
+        runVersion: 'latestFromBranch'
+        runBranch: 'refs/heads/$(BUILD_BRANCH)'
+        path: $(Build.ArtifactStagingDirectory)/download
+        patterns: |
+          libdashapi*.deb
+      displayName: "Download dash api"
+
+    - script: |
+        set -xe
+        sudo apt-get update
+        sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libdashapi_*.deb
+      workingDirectory: $(Pipeline.Workspace)/
+      displayName: 'Install libdashapi libraries'
+
     - script: |
         set -xe
         sudo pip3 install swsssdk-2.0.1-py3-none-any.whl
@@ -112,20 +134,22 @@ stages:
         sudo pip3 install sonic_yang_models-1.0-py3-none-any.whl
         sudo pip3 install sonic_config_engine-1.0-py3-none-any.whl
         sudo pip3 install sonic_platform_common-1.0-py3-none-any.whl
-      workingDirectory: $(Pipeline.Workspace)/target/python-wheels/bullseye/
+      workingDirectory: $(Pipeline.Workspace)/target/python-wheels/bookworm/
       displayName: 'Install Python dependencies'
 
     - script: |
         set -ex
         # Install .NET CORE
         curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
-        sudo apt-add-repository https://packages.microsoft.com/debian/11/prod
+        sudo apt-add-repository https://packages.microsoft.com/debian/12/prod
         sudo apt-get update
         sudo apt-get install -y dotnet-sdk-8.0
       displayName: "Install .NET CORE"
 
     - script: |
-        python3 setup.py test
+        pip3 install ".[testing]"
+        pip3 uninstall --yes sonic-utilities
+        pytest
       displayName: 'Test Python 3'
 
     - task: PublishTestResults@2
@@ -145,7 +169,7 @@ stages:
 
     - script: |
         set -e
-        python3 setup.py bdist_wheel
+        python3 -m build -n
       displayName: 'Build Python 3 wheel'
 
     - publish: '$(System.DefaultWorkingDirectory)/dist/'

config/main.py

@@ -34,7 +34,8 @@
 from sonic_yang_cfg_generator import SonicYangCfgDbGenerator
 from utilities_common import util_base
 from swsscommon import swsscommon
-from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, ConfigDBPipeConnector
+from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, ConfigDBPipeConnector, \
+                                isInterfaceNameValid, IFACE_NAME_MAX_LEN
 from utilities_common.db import Db
 from utilities_common.intf_filter import parse_interface_in_filter
 from utilities_common import bgp_util
@@ -106,7 +107,6 @@
 
 CFG_PORTCHANNEL_PREFIX = "PortChannel"
 CFG_PORTCHANNEL_PREFIX_LEN = 11
-CFG_PORTCHANNEL_NAME_TOTAL_LEN_MAX = 15
 CFG_PORTCHANNEL_MAX_VAL = 9999
 CFG_PORTCHANNEL_NO="<0-9999>"
 
@@ -439,7 +439,7 @@ def is_portchannel_name_valid(portchannel_name):
     if (portchannel_name[CFG_PORTCHANNEL_PREFIX_LEN:].isdigit() is False or
           int(portchannel_name[CFG_PORTCHANNEL_PREFIX_LEN:]) > CFG_PORTCHANNEL_MAX_VAL) :
         return False
-    if len(portchannel_name) > CFG_PORTCHANNEL_NAME_TOTAL_LEN_MAX:
+    if not isInterfaceNameValid(portchannel_name):
         return False
     return True
 
@@ -1830,7 +1830,7 @@ def reload(db, filename, yes, load_sysinfo, no_service_restart, force, file_form
         if multi_asic.is_multi_asic():
             # Multiasic has not 100% fully validated. Thus pass here.
             pass
-        else:
+        elif "golden" in filename.lower():
             config_file_yang_validation(filename)
 
     #Stop services before config push
@@ -2206,8 +2206,8 @@ def generate_sysinfo(cur_config, config_input, ns=None):
     if not platform:
         platform = device_info.get_platform()
 
-    device_metadata['localhost']['mac'] = mac
-    device_metadata['localhost']['platform'] = platform
+    device_metadata['localhost']['mac'] = mac.rstrip('\n')
+    device_metadata['localhost']['platform'] = platform.rstrip('\n')
 
     return
 
@@ -2484,8 +2484,9 @@ def add_portchannel(ctx, portchannel_name, min_links, fallback, fast_rate):
     db = ValidatedConfigDBConnector(ctx.obj['db'])
     if ADHOC_VALIDATION:
         if is_portchannel_name_valid(portchannel_name) != True:
-            ctx.fail("{} is invalid!, name should have prefix '{}' and suffix '{}'"
-                    .format(portchannel_name, CFG_PORTCHANNEL_PREFIX, CFG_PORTCHANNEL_NO))
+            ctx.fail("{} is invalid!, name should have prefix '{}' and suffix '{}' "
+                     "and its length should not exceed {} characters"
+                     .format(portchannel_name, CFG_PORTCHANNEL_PREFIX, CFG_PORTCHANNEL_NO, IFACE_NAME_MAX_LEN))
         if is_portchannel_present_in_db(db, portchannel_name):
             ctx.fail("{} already exists!".format(portchannel_name)) # TODO: MISSING CONSTRAINT IN YANG MODEL
 
@@ -6881,8 +6882,8 @@ def add_vrf(ctx, vrf_name):
     config_db = ValidatedConfigDBConnector(ctx.obj['config_db'])
     if not vrf_name.startswith("Vrf") and not (vrf_name == 'mgmt') and not (vrf_name == 'management'):
         ctx.fail("'vrf_name' must begin with 'Vrf' or named 'mgmt'/'management' in case of ManagementVRF.")
-    if len(vrf_name) > 15:
-        ctx.fail("'vrf_name' is too long!")
+    if not isInterfaceNameValid(vrf_name):
+        ctx.fail("'vrf_name' length should not exceed {} characters".format(IFACE_NAME_MAX_LEN))
     if is_vrf_exists(config_db, vrf_name):
         ctx.fail("VRF {} already exists!".format(vrf_name))
     elif (vrf_name == 'mgmt' or vrf_name == 'management'):
@@ -6901,8 +6902,8 @@ def del_vrf(ctx, vrf_name):
     config_db = ValidatedConfigDBConnector(ctx.obj['config_db'])
     if not vrf_name.startswith("Vrf") and not (vrf_name == 'mgmt') and not (vrf_name == 'management'):
         ctx.fail("'vrf_name' must begin with 'Vrf' or named 'mgmt'/'management' in case of ManagementVRF.")
-    if len(vrf_name) > 15:
-        ctx.fail("'vrf_name' is too long!")
+    if not isInterfaceNameValid(vrf_name):
+        ctx.fail("'vrf_name' length should not exceed {} characters".format((IFACE_NAME_MAX_LEN)))
     syslog_table = config_db.get_table("SYSLOG_SERVER")
     syslog_vrf_dev = "mgmt" if vrf_name == "management" else vrf_name
     for syslog_entry, syslog_data in syslog_table.items():
@@ -7932,8 +7933,8 @@ def add_loopback(ctx, loopback_name):
     config_db = ValidatedConfigDBConnector(ctx.obj['db'])
     if ADHOC_VALIDATION:
         if is_loopback_name_valid(loopback_name) is False:
-            ctx.fail("{} is invalid, name should have prefix '{}' and suffix '{}' "
-                    .format(loopback_name, CFG_LOOPBACK_PREFIX, CFG_LOOPBACK_NO))
+            ctx.fail("{} is invalid, name should have prefix '{}' and suffix '{}' and should not exceed {} characters"
+                    .format(loopback_name, CFG_LOOPBACK_PREFIX, CFG_LOOPBACK_NO, IFACE_NAME_MAX_LEN))
 
         lo_intfs = [k for k, v in config_db.get_table('LOOPBACK_INTERFACE').items() if type(k) != tuple]
         if loopback_name in lo_intfs:
@@ -8680,6 +8681,8 @@ def add_subinterface(ctx, subinterface_name, vid):
 
         if interface_alias is None:
             ctx.fail("{} invalid subinterface".format(interface_alias))
+        if not isInterfaceNameValid(interface_alias):
+            ctx.fail("Subinterface name length should not exceed {} characters".format(IFACE_NAME_MAX_LEN))
 
         if interface_alias.startswith("Po") is True:
             intf_table_name = CFG_PORTCHANNEL_PREFIX

config/vxlan.py

@@ -3,6 +3,7 @@
 
 from jsonpatch import JsonPatchConflict
 from .validated_config_db_connector import ValidatedConfigDBConnector
+from swsscommon.swsscommon import isInterfaceNameValid, IFACE_NAME_MAX_LEN
 
 ADHOC_VALIDATION = True
 #
@@ -24,6 +25,8 @@ def add_vxlan(db, vxlan_name, src_ip):
     if ADHOC_VALIDATION:
         if not clicommon.is_ipaddress(src_ip):
             ctx.fail("{} invalid src ip address".format(src_ip))  
+        if not isInterfaceNameValid(vxlan_name):
+            ctx.fail("'vxlan_name' length should not exceed {} characters".format(IFACE_NAME_MAX_LEN))
 
     vxlan_keys = db.cfgdb.get_keys('VXLAN_TUNNEL')
     if not vxlan_keys:
@@ -317,4 +320,3 @@ def del_vxlan_map_range(db, vxlan_name, vlan_start, vlan_end, vni_start):
            config_db.set_entry('VXLAN_TUNNEL_MAP', mapname, None)
        except JsonPatchConflict as e:
            ctx.fail("Invalid ConfigDB. Error: {}".format(e))
-

doc/Command-Reference.md

@@ -5137,6 +5137,37 @@ This command is to display the link-training status of the selected interfaces.
     Ethernet8      trained          on      up       up
   ```
 
+**show interfaces errors**
+
+The show interface errors command provides detailed statistics and error counters for MAC-level operations on an interface. It displays the status of various operational parameters, error counts, and timestamps for when these errors occurred.
+
+- Usage:
+  ```
+  show interfaces errors [<interface_name>]
+  ```
+
+- Example:
+  ```
+  admin@sonic:~$ show interfaces errors Ethernet4
+  Port Errors                        Count           Last timestamp(UTC)
+  ---------------------------------- -----           -------------------
+  oper_error_status                  5442            2024-11-02 04:00:05
+  mac_local_fault                    2               2024-11-02 04:00:05
+  fec_sync_loss                      2               2024-11-02 04:00:05
+  fec_alignment_loss                 2               2024-11-02 04:00:05
+  high_ser_error                     2               2024-11-02 04:00:05
+  high ber_error                     2               2024-11-02 04:00:05
+  data_unit_crc_error                2               2024-11-02 04:00:05
+  data_unit_misalignment_error       2               2024-11-02 04:00:05
+  signal_local_error                 2               2024-11-02 04:00:05
+  mac_remote_fault                   2               2024-11-02 04:00:50
+  crc_rate                           2               2024-11-02 04:00:50
+  data_unit_size                     2               2024-11-02 04:00:50
+  code_group_error                   0               Never
+  no_rx_reachability                 0               Never
+  ```
+
+
 **show interfaces mpls**
 
 This command is used to display the configured MPLS state for the list of configured interfaces.

dump/dash_util.py

@@ -0,0 +1,90 @@
+import base64
+import uuid
+import socket
+import ipaddress
+from google.protobuf.message import Message
+from dash_api.types_pb2 import Guid, IpAddress, IpPrefix
+from google.protobuf.json_format import MessageToDict
+
+
+def format_ip(node):
+    return str(ipaddress.IPv4Address(socket.ntohl(node)))
+
+
+def format_mac(node):
+    b64 = base64.b64decode(node)
+    return ':'.join(b64.hex()[i:i + 2] for i in range(0, 12, 2))
+
+
+def format_guid_dict(node):
+    b64 = base64.b64decode(node['value'])
+    return str(uuid.UUID(bytes=b64))
+
+
+def format_ip_address_dict(node):
+    if 'ipv4' in node:
+        return format_ip(node['ipv4'])
+
+
+def format_ip_prefix(node):
+    ip = format_ip_address_dict(node['ip'])
+    mask = format_ip_address_dict(node['mask'])
+    network = ipaddress.IPv4Network(f'{ip}/{mask}', strict=False)
+    return str(network)
+
+
+def get_decoded_value(pb, pb_data):
+    pb.ParseFromString(pb_data[b'pb'])
+    json_string = MessageToDict(pb, preserving_proto_field_name=True)
+    json_string = find_known_types_sec(pb, json_string)
+    return json_string
+
+
+decode_types = [IpAddress, Guid, IpPrefix]
+decode_types = [cls.__module__ + '.' + cls.__name__ for cls in decode_types]
+decode_fn = {'IpAddress': format_ip_address_dict,
+             'Guid': format_guid_dict,
+             'mac_address': format_mac,
+             'IpPrefix': format_ip_prefix}
+
+
+def find_known_types_sec(pb2_obj, pb2_dict):
+
+    def process_msg_field(obj, proto_dict, field_name):
+        class_name = type(obj).__name__
+        obj_type = f"{type(obj).__module__}.{type(obj).__name__}"
+        if obj_type in decode_types:
+            proto_dict[field_name] = decode_fn[class_name](proto_dict[field_name])
+        else:
+            find_index(obj, proto_dict[field_name])
+
+    def process_rep_field(obj, proto_dict, field_name):
+        final_list = []
+        requires_change = False
+        for ind, value in enumerate(obj):
+            if isinstance(value, Message):
+                obj_type = f"{type(value).__module__}.{type(value).__name__}"
+                if obj_type in decode_types:
+                    requires_change = True
+                    class_name = type(value).__name__
+                    final_list.append(decode_fn[class_name](proto_dict[field_name][ind]))
+                else:
+                    find_index(value, pb2_dict[field_name][ind])
+        if requires_change:
+            proto_dict[field_name] = final_list
+
+    def find_index(proto_obj, proto_dict=pb2_dict):
+        for field_descriptor, value in proto_obj.ListFields():
+            field_name = field_descriptor.name
+            field_type = field_descriptor.type
+            if field_type == field_descriptor.TYPE_MESSAGE:
+                obj = getattr(proto_obj, field_name)
+                if field_descriptor.label == field_descriptor.LABEL_REPEATED:
+                    process_rep_field(obj, proto_dict, field_name)
+                else:
+                    process_msg_field(obj, proto_dict, field_name)
+            elif field_name in decode_fn:
+                proto_dict[field_name] = decode_fn[field_name](proto_dict[field_name])
+
+    find_index(pb2_obj)
+    return pb2_dict

dump/main.py

@@ -93,7 +93,7 @@ def state(ctx, module, identifier, db, table, key_map, verbose, namespace):
     vidtorid = extract_rid(collected_info, namespace, ctx.obj.conn_pool)
 
     if not key_map:
-        collected_info = populate_fv(collected_info, module, namespace, ctx.obj.conn_pool)
+        collected_info = populate_fv(collected_info, module, namespace, ctx.obj.conn_pool, obj.return_pb2_obj())
 
     for id in vidtorid.keys():
         collected_info[id]["ASIC_DB"]["vidtorid"] = vidtorid[id]
@@ -145,7 +145,7 @@ def filter_out_dbs(db_list, collected_info):
     return collected_info
 
 
-def populate_fv(info, module, namespace, conn_pool):
+def populate_fv(info, module, namespace, conn_pool, dash_object):
     all_dbs = set()
     for id in info.keys():
         for db_name in info[id].keys():
@@ -157,7 +157,9 @@ def populate_fv(info, module, namespace, conn_pool):
             db_cfg_file.connect(plugins.dump_modules[module].CONFIG_FILE, namespace)
         else:
             conn_pool.get(db_name, namespace)
-    
+    if dash_object:
+        conn_pool.get_dash_conn(namespace)
+        redis_conn = conn_pool.cache.get(namespace, {}).get("DASH_"+CONN, None)
     db_conn = conn_pool.cache.get(namespace, {}).get(CONN, None)
 
     final_info = {}
@@ -170,10 +172,17 @@ def populate_fv(info, module, namespace, conn_pool):
             for key in info[id][db_name]["keys"]:
                 if db_name == "CONFIG_FILE":
                     fv = db_cfg_file.get(db_name, key)
+                elif dash_object and db_name == "APPL_DB":
+                    try:
+                        from dump.dash_util import get_decoded_value
+                        pb_data = redis_conn.hgetall(key)
+                        fv = get_decoded_value(dash_object, pb_data)
+                    except ModuleNotFoundError:
+                        print("Issue in importing dash module!")
+                        return final_info
                 else:
                     fv = db_conn.get_all(db_name, key)
                 final_info[id][db_name]["keys"].append({key: fv})
-
     return final_info