Add show reboot-cause history cli (sonic-net#1210)

What I did
To make the reboot cause history information available for telemetry service, during processing the reboot-cause on boot, the information is saved in state-db.
The cli command show reboot-cause is changed to support the new file format and file name.
New command show reboot-cause history is added to display the reboot-cause information which is stored in state-db.

How I did it
Add the 'show reboot-cause history' command to read up to the10 previous reboot-cause information from state-db and display them.
Read the last reboot-cause from the new file "previous-reboot-cause.json" and process the data to display for show reboot-cause

New command output (if the output of a command-line utility has changed)
admin@sonic:~$ show reboot-cause history
name                 cause        time                          user    comment
-------------------  -----------  ----------------------------  ------  ---------
2020_10_09_04_53_58  warm-reboot  Fri Oct  9 04:51:47 UTC 2020  admin
2020_10_09_02_33_06  reboot       Fri Oct  9 02:29:44 UTC 2020  admin
2020_10_09_02_00_53  fast-reboot  Fri Oct  9 01:58:04 UTC 2020  admin
2020_10_09_01_56_59  reboot       Fri Oct  9 01:53:49 UTC 2020  admin

Author: sujinmkang

doc/Command-Reference.md

@@ -552,6 +552,26 @@ This command displays the cause of the previous reboot
   User issued reboot command [User: admin, Time: Mon Mar 25 01:02:03 UTC 2019]
   ```
 
+**show reboot-cause history**
+
+This command displays the history of the previous reboots up to 10 entry
+
+- Usage:
+  ```
+  show reboot-cause history
+  ```
+
+- Example:
+  ```
+  admin@sonic:~$ show reboot-cause history
+  Name                 Cause        Time                          User    Comment
+  -------------------  -----------  ----------------------------  ------  ---------
+  2020_10_09_02_33_06  reboot       Fri Oct  9 02:29:44 UTC 2020  admin
+  2020_10_09_01_56_59  reboot       Fri Oct  9 01:53:49 UTC 2020  admin
+  2020_10_09_02_00_53  fast-reboot  Fri Oct  9 01:58:04 UTC 2020  admin
+  2020_10_09_04_53_58  warm-reboot  Fri Oct  9 04:51:47 UTC 2020  admin
+  ```
+
 **show uptime**
 
 This command displays the current system uptime

show/main.py

@@ -24,6 +24,7 @@
 from . import kube
 from . import mlnx
 from . import muxcable
+from . import reboot_cause
 from . import vlan
 from . import system_health
 
@@ -134,6 +135,7 @@ def cli(ctx):
 cli.add_command(interfaces.interfaces)
 cli.add_command(kube.kubernetes)
 cli.add_command(muxcable.muxcable)
+cli.add_command(reboot_cause.reboot_cause)
 cli.add_command(vlan.vlan)
 cli.add_command(system_health.system_health)
 
@@ -1812,27 +1814,6 @@ def mmu():
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, text=True)
     click.echo(proc.stdout.read())
 
-
-#
-# 'reboot-cause' command ("show reboot-cause")
-#
-@cli.command('reboot-cause')
-def reboot_cause():
-    """Show cause of most recent reboot"""
-    PREVIOUS_REBOOT_CAUSE_FILE = "/host/reboot-cause/previous-reboot-cause.txt"
-
-    # At boot time, PREVIOUS_REBOOT_CAUSE_FILE is generated based on
-    # the contents of the 'reboot cause' file as it was left when the device
-    # went down for reboot. This file should always be created at boot,
-    # but check first just in case it's not present.
-    if not os.path.isfile(PREVIOUS_REBOOT_CAUSE_FILE):
-        click.echo("Unable to determine cause of previous reboot\n")
-    else:
-        cmd = "cat {}".format(PREVIOUS_REBOOT_CAUSE_FILE)
-        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, text=True)
-        click.echo(proc.stdout.read())
-
-
 #
 # 'line' command ("show line")
 #

show/reboot_cause.py

@@ -0,0 +1,68 @@
+import json
+import os
+import sys
+
+import click
+from tabulate import tabulate
+from swsssdk import SonicV2Connector
+import utilities_common.cli as clicommon
+
+
+PREVIOUS_REBOOT_CAUSE_FILE = "/host/reboot-cause/previous-reboot-cause.json"
+USER_ISSUED_REBOOT_CAUSE_REGEX ="User issued \'{}\' command [User: {}, Time: {}]"
+
+def read_reboot_cause_file():
+    result = ""
+    if os.path.exists(PREVIOUS_REBOOT_CAUSE_FILE):
+        with open(PREVIOUS_REBOOT_CAUSE_FILE) as f:
+            result = json.load(f)
+    return result
+
+#
+# 'reboot-cause' group ("show reboot-cause")
+#
+@click.group(cls=clicommon.AliasedGroup, invoke_without_command=True)
+@click.pass_context
+def reboot_cause(ctx):
+    """Show cause of most recent reboot"""
+    if ctx.invoked_subcommand is None:
+        reboot_cause = ""
+        # Read the previous reboot cause
+        data = read_reboot_cause_file()
+        if data['user'] == "N/A":
+            reboot_cause = "{}".format(data['cause'])
+        else:
+            reboot_cause = USER_ISSUED_REBOOT_CAUSE_REGEX.format(data['cause'], data['user'], data['time'])
+
+        click.echo(reboot_cause)
+
+# 'history' subcommand ("show reboot-cause history")
+@reboot_cause.command()
+def history():
+    """Show history of reboot-cause"""
+    REBOOT_CAUSE_TABLE_NAME = "REBOOT_CAUSE"
+    TABLE_NAME_SEPARATOR = '|'
+    db = SonicV2Connector(host='127.0.0.1')
+    db.connect(db.STATE_DB, False)   # Make one attempt only
+    prefix = REBOOT_CAUSE_TABLE_NAME + TABLE_NAME_SEPARATOR
+    _hash = '{}{}'.format(prefix, '*')
+    table_keys = db.keys(db.STATE_DB, _hash)
+    if table_keys is not None:
+        table_keys.sort(reverse=True)
+
+        table = []
+        for tk in table_keys:
+            entry = db.get_all(db.STATE_DB, tk)
+            r = []
+            r.append(tk.replace(prefix,""))
+            r.append(entry['cause'] if 'cause' in entry else "")
+            r.append(entry['time'] if 'time' in entry else "")
+            r.append(entry['user'] if 'user' in entry else "")
+            r.append(entry['comment'] if 'comment' in entry else "")
+            table.append(r)
+
+        header = ['Name', 'Cause', 'Time', 'User', 'Comment']
+        click.echo(tabulate(table, header, numalign="left"))
+    else:
+        click.echo("Reboot-cause history is not yet available in StateDB")
+        sys.exit(1)

tests/mock_tables/state_db.json

@@ -273,6 +273,18 @@
 	"30" : "200.200.200.5@Vlan1000",
 	"31" : "200.200.200.5@Vlan1000"	
     },
+    "REBOOT_CAUSE|2020_10_09_04_53_58": {
+        "cause": "warm-reboot",
+        "time": "Fri Oct  9 04:51:47 UTC 2020",
+        "user": "admin",
+        "comment": "N/A"
+    },
+    "REBOOT_CAUSE|2020_10_09_02_33_06": {
+        "cause": "reboot",
+        "time": "Fri Oct  9 02:29:44 UTC 2020",
+        "user": "admin",
+        "comment": "N/A"
+    },
     "CHASSIS_TABLE|CHASSIS 1": {
         "module_num": "5"
     },

tests/reboot_cause_test.py

@@ -0,0 +1,73 @@
+import os
+import sys
+import textwrap
+import mock
+from click.testing import CliRunner
+from .mock_tables import dbconnector
+
+test_path = os.path.dirname(os.path.abspath(__file__))
+modules_path = os.path.dirname(test_path)
+sys.path.insert(0, modules_path)
+
+
+import show.main as show
+
+"""
+    Note: The following 'show reboot-cause' commands simply call other SONiC
+    CLI utilities, so the unit tests for the other utilities are expected
+    to cover testing their functionality:
+
+        show reboot-cause
+        show reboot-cause history
+"""
+
+class TestShowRebootCause(object):
+    @classmethod
+    def setup_class(cls):
+        print("SETUP")
+        os.environ["UTILITIES_UNIT_TESTING"] = "1"
+
+    # Test 'show reboot-cause' without previous-reboot-cause.json 
+    def test_reboot_cause_no_history_file(self):
+        expected_output = ""
+        runner = CliRunner()
+        result = runner.invoke(show.cli.commands["reboot-cause"], [])
+        assert result.output == expected_output
+
+    # Test 'show reboot-cause' with user issued reboot
+    def test_reboot_cause_user(self):
+        expected_output = "User issued 'reboot' command [User: admin, Time: Thu Oct 22 03:11:08 UTC 2020]\n"
+
+        with mock.patch("show.reboot_cause.read_reboot_cause_file", return_value={"comment": "", "gen_time": "2020_10_22_03_14_07", "cause": "reboot", "user": "admin", "time": "Thu Oct 22 03:11:08 UTC 2020"}):
+            runner = CliRunner()
+            result = runner.invoke(show.cli.commands["reboot-cause"], [])
+            assert result.output == expected_output
+
+
+    # Test 'show reboot-cause' with non-user issue reboot (hardware reboot-cause or unknown reboot-cause)
+    def test_reboot_cause_non_user(self):
+        expected_output = "Watchdog\n"
+
+        with mock.patch("show.reboot_cause.read_reboot_cause_file", return_value={"comment": "N/A", "gen_time": "2020_10_22_03_15_08", "cause": "Watchdog", "user": "N/A", "time": "N/A"}):
+            runner = CliRunner()
+            result = runner.invoke(show.cli.commands["reboot-cause"], [])
+            assert result.output == expected_output
+
+    # Test 'show reboot-cause history'
+    def test_reboot_cause_history(self):
+        expected_output = """\
+Name                 Cause        Time                          User    Comment
+-------------------  -----------  ----------------------------  ------  ---------
+2020_10_09_04_53_58  warm-reboot  Fri Oct  9 04:51:47 UTC 2020  admin   N/A
+2020_10_09_02_33_06  reboot       Fri Oct  9 02:29:44 UTC 2020  admin   N/A
+"""
+        runner = CliRunner()
+        result = runner.invoke(show.cli.commands["reboot-cause"].commands["history"], [])
+        print(result.output)
+        assert result.output == expected_output
+
+    @classmethod
+    def teardown_class(cls):
+        print("TEARDOWN")
+        os.environ["PATH"] = os.pathsep.join(os.environ["PATH"].split(os.pathsep)[:-1])
+        os.environ["UTILITIES_UNIT_TESTING"] = "0"