Enhanced NVMe disk support, added limited eUSB disk support (sonic-net#493)

* Enhanced NVMe storage disk attributes; 'Ssdtil' instantiation for devices with NVMe disks

* Added limited support for USB storage devices

* usb.py: removed logging, ssd.py: split nvme parsing to its own function

Author: assrinivasan

sonic_platform_base/sonic_storage/ssd.py

@@ -148,24 +148,41 @@ def _parse_vendor(self):
     def fetch_generic_ssd_info(self, diskdev):
         self.ssd_info = self._execute_shell(self.vendor_ssd_utility["Generic"]["utility"].format(diskdev))
 
+    def parse_nvme_ssd_info(self):
+        self.model = self._parse_re('Model Number:\s*(.+?)\n', self.ssd_info)
+
+        health_raw = self._parse_re('Percentage Used\s*(.+?)\n', self.ssd_info)
+        if health_raw == NOT_AVAILABLE:
+            self.health = NOT_AVAILABLE
+        else:
+            health_raw = health_raw.split()[-1]
+            self.health = 100 - float(health_raw.strip('%'))
+
+        temp_raw = self._parse_re('Temperature\s*(.+?)\n', self.ssd_info)
+        if temp_raw == NOT_AVAILABLE:
+            self.temperature = NOT_AVAILABLE
+        else:
+            temp_raw = temp_raw.split()[-2]
+            self.temperature = float(temp_raw)
+
+        spare_blocks_raw = self._parse_re('Available Spare\s*(.+?)\n', self.ssd_info)
+        if spare_blocks_raw != NOT_AVAILABLE:
+            spare_blocks_raw = spare_blocks_raw.split()[-1]
+            self.reserved_blocks = float(spare_blocks_raw.strip('%'))
+
+        disk_io_reads_raw = self._parse_re('Data Units Read\s*(.+?)\n', self.ssd_info)
+        if disk_io_reads_raw != NOT_AVAILABLE:
+            self.disk_io_reads = disk_io_reads_raw.split(':')[-1].strip()
+
+        disk_io_writes_raw = self._parse_re('Data Units Written\s*(.+?)\n', self.ssd_info)
+        if disk_io_writes_raw != NOT_AVAILABLE:
+            self.disk_io_writes = disk_io_writes_raw.split(':')[-1].strip()
+
     # Health and temperature values may be overwritten with vendor specific data
     def parse_generic_ssd_info(self):
-        if "nvme" in self.dev:
-            self.model = self._parse_re('Model Number:\s*(.+?)\n', self.ssd_info)
 
-            health_raw = self._parse_re('Percentage Used\s*(.+?)\n', self.ssd_info)
-            if health_raw == NOT_AVAILABLE:
-                self.health = NOT_AVAILABLE
-            else:
-                health_raw = health_raw.split()[-1]
-                self.health = 100 - float(health_raw.strip('%'))
-
-            temp_raw = self._parse_re('Temperature\s*(.+?)\n', self.ssd_info)
-            if temp_raw == NOT_AVAILABLE:
-                self.temperature = NOT_AVAILABLE
-            else:
-                temp_raw = temp_raw.split()[-2]
-                self.temperature = float(temp_raw)
+        if "nvme" in self.dev:
+            self.parse_nvme_ssd_info()
         else:
             self.model = self._parse_re('Device Model:\s*(.+?)\n', self.ssd_info)
 
@@ -184,21 +201,21 @@ def parse_generic_ssd_info(self):
             else:
                 self.temperature = temp_raw.split()[7].split()[0]
 
-        self.serial = self._parse_re('Serial Number:\s*(.+?)\n', self.ssd_info)
-        self.firmware = self._parse_re('Firmware Version:\s*(.+?)\n', self.ssd_info)
+            io_reads_raw = self.parse_id_number(GENERIC_IO_READS_ID, self.ssd_info)
+            self.disk_io_reads = NOT_AVAILABLE if io_reads_raw == NOT_AVAILABLE else io_reads_raw.split()[-1]
 
-        io_reads_raw = self.parse_id_number(GENERIC_IO_READS_ID, self.ssd_info)
-        self.disk_io_reads = NOT_AVAILABLE if io_reads_raw == NOT_AVAILABLE else io_reads_raw.split()[-1]
+            io_writes_raw = self.parse_id_number(GENERIC_IO_WRITES_ID, self.ssd_info)
+            self.disk_io_writes = NOT_AVAILABLE if io_writes_raw == NOT_AVAILABLE else io_writes_raw.split()[-1]
 
-        io_writes_raw = self.parse_id_number(GENERIC_IO_WRITES_ID, self.ssd_info)
-        self.disk_io_writes = NOT_AVAILABLE if io_writes_raw == NOT_AVAILABLE else io_writes_raw.split()[-1]
+            for ID in GENERIC_RESERVED_BLOCKS_ID:
+                rbc_raw = self.parse_id_number(ID, self.ssd_info)
+                if rbc_raw == NOT_AVAILABLE: self.reserved_blocks = NOT_AVAILABLE
+                else:
+                    self.reserved_blocks = rbc_raw.split()[-1]
+                    break
 
-        for ID in GENERIC_RESERVED_BLOCKS_ID:
-            rbc_raw = self.parse_id_number(ID, self.ssd_info)
-            if rbc_raw == NOT_AVAILABLE: self.reserved_blocks = NOT_AVAILABLE
-            else:
-                self.reserved_blocks = rbc_raw.split()[-1]
-                break
+        self.serial = self._parse_re('Serial Number:\s*(.+?)\n', self.ssd_info)
+        self.firmware = self._parse_re('Firmware Version:\s*(.+?)\n', self.ssd_info)
 
     def parse_innodisk_info(self):
         if self.vendor_ssd_info:

sonic_platform_base/sonic_storage/storage_devices.py

@@ -12,6 +12,8 @@
     from sonic_py_common import syslogger
     from sonic_platform_base.sonic_storage.ssd import SsdUtil
     from sonic_platform_base.sonic_storage.emmc import EmmcUtil
+    from sonic_platform_base.sonic_storage.usb import UsbUtil
+
 except ImportError as e:
     raise ImportError (str(e) + "- required module not found")
 
@@ -23,6 +25,8 @@
 
 BASE_PATH = "/sys/block"
 BLKDEV_BASE_PATH = "/dev"
+ssdutil_compatible_keys = ('sd', 'nvme')
+ssdutil_compatible_paths_strings = ('ata', 'nvme')
 
 class StorageDevices:
     def __init__(self):
@@ -65,9 +69,9 @@ def _storage_device_object_factory(self, key):
         blkdev = os.path.join(BLKDEV_BASE_PATH, key)
         diskdev = os.path.join(BASE_PATH, key)
 
-        if key.startswith('sd'):
+        if key.startswith(ssdutil_compatible_keys):
             path = os.path.join(diskdev, "device")
-            if "ata" in os.path.realpath(path):
+            if any(substring in os.path.realpath(path) for substring in ssdutil_compatible_paths_strings):
                 try:
                     return SsdUtil(blkdev)
                 except Exception as e:

sonic_platform_base/sonic_storage/usb.py

@@ -0,0 +1,136 @@
+#
+# usb.py
+#
+# Implementation of SSD Utility API for eUSB.
+# It reads eUSB health, model, firmware, and serial from /sys/block/*.
+#
+
+try:
+    import os
+    from .storage_common import StorageCommon
+    from blkinfo import BlkDiskInfo
+except ImportError as e:
+    raise ImportError(str(e) + "- required module not found")
+
+
+NOT_AVAILABLE = "N/A"
+
+class UsbUtil(StorageCommon):
+
+    model = NOT_AVAILABLE
+    serial = NOT_AVAILABLE
+    firmware = NOT_AVAILABLE
+    temperature = NOT_AVAILABLE
+    health = NOT_AVAILABLE
+    vendor = NOT_AVAILABLE
+    disk_io_reads = NOT_AVAILABLE
+    disk_io_writes = NOT_AVAILABLE
+    reserved_blocks = NOT_AVAILABLE
+    blkd = {}
+
+
+    def __init__(self, diskdev):
+
+        self.diskdev = diskdev
+        self.path = os.path.join('/sys/block', os.path.basename(diskdev))
+        StorageCommon.__init__(self, diskdev)
+
+        self.fetch_parse_info()
+
+    def fetch_parse_info(self, diskdev=None):
+        self.fetch_blkinfo()
+        self.parse_blkinfo()
+
+    def fetch_blkinfo(self):
+        filters = {}
+        filters['name'] = '{}'.format(os.path.basename(self.diskdev))
+        self.blkd = BlkDiskInfo().get_disks(filters)[0]
+
+    def parse_blkinfo(self):
+        if 'dict' in str(type(self.blkd)) and self.blkd:
+            self.model = self.blkd["model"]
+            self.serial = self.blkd["serial"]
+            self.vendor = self.blkd["vendor"]
+
+    def get_health(self):
+        """
+        Retrieves current disk health in percentages
+
+        Returns:
+            A float number of current ssd health
+            e.g. 83.5
+        """
+        return self.health
+
+    def get_temperature(self):
+        """
+        Retrieves current disk temperature in Celsius
+
+        Returns:
+            A float number of current temperature in Celsius
+            e.g. 40.1
+        """
+        return self.temperature
+
+    def get_model(self):
+        """
+        Retrieves model for the given disk device
+
+        Returns:
+            A string holding disk model as provided by the manufacturer
+        """
+        return self.model
+
+    def get_firmware(self):
+        """
+        Retrieves firmware version for the given disk device
+
+        Returns:
+            A string holding disk firmware version as provided by the manufacturer
+        """
+        return self.firmware
+
+    def get_serial(self):
+        """
+        Retrieves serial number for the given disk device
+
+        Returns:
+            A string holding disk serial number as provided by the manufacturer
+        """
+        return self.serial
+
+    def get_disk_io_reads(self):
+        """
+        Retrieves the total number of Input/Output (I/O) reads done on an SSD
+
+        Returns:
+            An integer value of the total number of I/O reads
+        """
+        return self.disk_io_reads
+
+    def get_disk_io_writes(self):
+        """
+        Retrieves the total number of Input/Output (I/O) writes done on an SSD
+
+        Returns:
+            An integer value of the total number of I/O writes
+        """
+        return self.disk_io_writes
+
+    def get_reserved_blocks(self):
+        """
+        Retrieves the total number of reserved blocks in an SSD
+
+        Returns:
+            An integer value of the total number of reserved blocks
+        """
+        return self.reserved_blocks
+
+    def get_vendor_output(self):
+        """
+        Retrieves vendor specific data for the given disk device
+
+        Returns:
+            A string holding some vendor specific disk information
+        """
+        return self.vendor

tests/mocked_libs/__init__.py

@@ -0,0 +1,90 @@
+mock_json_op = \
+    [
+        {
+            "name": "sdx",
+            "kname": "sdx",
+            "fstype": "",
+            "label": "",
+            "mountpoint": "",
+            "size": "3965714432",
+            "maj:min": "8:0",
+            "rm": "0",
+            "model": "SMART EUSB",
+            "vendor": "SMART EUSB",
+            "serial": "SPG200807J1",
+            "hctl": "2:0:0:0",
+            "tran": "usb",
+            "rota": "1",
+            "type": "disk",
+            "ro": "0",
+            "owner": "",
+            "group": "",
+            "mode": "brw-rw----",
+            "children": [
+                {
+                    "name": "sdx1",
+                    "kname": "sdx1",
+                    "fstype": "ext4",
+                    "label": "",
+                    "mountpoint": "/host",
+                    "size": "3964665856",
+                    "maj:min": "8:1",
+                    "rm": "0",
+                    "model": " ",
+                    "vendor": " ",
+                    "serial": "",
+                    "hctl": "",
+                    "tran": "",
+                    "rota": "1",
+                    "type": "part",
+                    "ro": "0",
+                    "owner": "",
+                    "group": "",
+                    "mode": "brw-rw----",
+                    "children": [],
+                    "parents": ["sdx"],
+                    "statistics": {
+                        "major": "8",
+                        "minor": "1",
+                        "kname": "sdx1",
+                        "reads_completed": "22104",
+                        "reads_merged": "5299",
+                        "sectors_read": "1091502",
+                        "time_spent_reading_ms": "51711",
+                        "writes_completed": "11283",
+                        "writes_merged": "13401",
+                        "sectors_written": "443784",
+                        "time_spent_ writing": "133398",
+                        "ios_in_progress": "0",
+                        "time_spent_doing_ios_ms": "112040",
+                        "weighted_time_ios_ms": "112040",
+                    },
+                }
+            ],
+            "parents": [],
+            "statistics": {
+                "major": "8",
+                "minor": "0",
+                "kname": "sdx",
+                "reads_completed": "22151",
+                "reads_merged": "5299",
+                "sectors_read": "1093606",
+                "time_spent_reading_ms": "52005",
+                "writes_completed": "11283",
+                "writes_merged": "13401",
+                "sectors_written": "443784",
+                "time_spent_ writing": "133398",
+                "ios_in_progress": "0",
+                "time_spent_doing_ios_ms": "112220",
+                "weighted_time_ios_ms": "112220",
+            },
+        }
+    ]
+
+
+class BlkDiskInfo:
+    def __init__(self):
+        return
+
+    def get_disks(self, filters={}):
+        return mock_json_op

tests/mocked_libs/blkinfo.py

@@ -0,0 +1,6 @@
+from collections import namedtuple
+
+
+def disk_partitions():
+    sdiskio = namedtuple('sdiskio', ['read_count', 'write_count'])
+    return sdiskio(read_count=42444, write_count=210141)

tests/mocked_libs/psutil.py

@@ -1341,6 +1341,9 @@ def test_nvme_ssd(self):
         assert(nvme_ssd.get_firmware() == "COT6OQ")
         assert(nvme_ssd.get_temperature() == 37)
         assert(nvme_ssd.get_serial() == "A0221030722410000027")
+        assert(nvme_ssd.get_disk_io_reads() == "1,546,369 [791 GB]")
+        assert(nvme_ssd.get_disk_io_writes() == "7,118,163 [3.64 TB]")
+        assert(nvme_ssd.get_reserved_blocks() == 100.0)
 
     @mock.patch('sonic_platform_base.sonic_storage.ssd.SsdUtil._execute_shell', mock.MagicMock(return_value=output_lack_info_ssd))
     def test_nvme_ssd_with_na_path(self):