Merge branch 'master' into storm_control

Author: mohan-selvaraj

config/main.py

@@ -15,7 +15,7 @@
 
 from collections import OrderedDict
 from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat
-from minigraph import parse_device_desc_xml
+from minigraph import parse_device_desc_xml, minigraph_encoder
 from natsort import natsorted
 from portconfig import get_child_ports
 from socket import AF_INET, AF_INET6
@@ -27,7 +27,7 @@
 from utilities_common.intf_filter import parse_interface_in_filter
 from utilities_common import bgp_util
 import utilities_common.cli as clicommon
-from utilities_common.general import load_db_config
+from utilities_common.general import load_db_config, load_module_from_source
 import utilities_common.multi_asic as multi_asic_util
 
 from .utils import log
@@ -72,6 +72,7 @@
 DEFAULT_CONFIG_YANG_FILE = '/etc/sonic/config_yang.json'
 NAMESPACE_PREFIX = 'asic'
 INTF_KEY = "interfaces"
+DEFAULT_GOLDEN_CONFIG_DB_FILE = '/etc/sonic/golden_config_db.json'
 
 INIT_CFG_FILE = '/etc/sonic/init_cfg.json'
 
@@ -100,6 +101,9 @@
 QUEUE_RANGE = click.IntRange(min=0, max=255)
 GRE_TYPE_RANGE = click.IntRange(min=0, max=65535)
 
+# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension.
+sonic_cfggen = load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen')
+
 #
 # Helper functions
 #
@@ -1692,6 +1696,10 @@ def load_minigraph(db, no_service_restart):
                 cfggen_namespace_option = " -n {}".format(namespace)
             clicommon.run_command(db_migrator + ' -o set_version' + cfggen_namespace_option)
 
+    # Load golden_config_db.json
+    if os.path.isfile(DEFAULT_GOLDEN_CONFIG_DB_FILE):
+        override_config_by(DEFAULT_GOLDEN_CONFIG_DB_FILE)
+
     # We first run "systemctl reset-failed" to remove the "failed"
     # status from all services before we attempt to restart them
     if not no_service_restart:
@@ -1740,6 +1748,65 @@ def load_port_config(config_db, port_config_path):
                 port_name), display_cmd=True)
     return
 
+
+def override_config_by(golden_config_path):
+    # Override configDB with golden config
+    clicommon.run_command('config override-config-table {}'.format(
+        golden_config_path), display_cmd=True)
+    return
+
+
+#
+# 'override-config-table' command ('config override-config-table ...')
+#
+@config.command('override-config-table')
+@click.argument('input-config-db', required=True)
+@click.option(
+    '--dry-run', is_flag=True, default=False,
+    help='test out the command without affecting config state'
+)
+@clicommon.pass_db
+def override_config_table(db, input_config_db, dry_run):
+    """Override current configDB with input config."""
+
+    try:
+        # Load golden config json
+        config_input = read_json_file(input_config_db)
+    except Exception as e:
+        click.secho("Bad format: json file broken. {}".format(str(e)),
+                    fg='magenta')
+        sys.exit(1)
+
+    # Validate if the input is dict
+    if not isinstance(config_input, dict):
+        click.secho("Bad format: input_config_db is not a dict",
+                    fg='magenta')
+        sys.exit(1)
+
+    config_db = db.cfgdb
+
+    if dry_run:
+        # Read config from configDB
+        current_config = config_db.get_config()
+        # Serialize to the same format as json input
+        sonic_cfggen.FormatConverter.to_serialized(current_config)
+        # Override current config with golden config
+        for table in config_input:
+            current_config[table] = config_input[table]
+        print(json.dumps(current_config, sort_keys=True,
+                         indent=4, cls=minigraph_encoder))
+    else:
+        # Deserialized golden config to DB recognized format
+        sonic_cfggen.FormatConverter.to_deserialized(config_input)
+        # Delete table from DB then mod_config to apply golden config
+        click.echo("Removing configDB overriden table first ...")
+        for table in config_input:
+            config_db.delete_table(table)
+        click.echo("Overriding input config to configDB ...")
+        data = sonic_cfggen.FormatConverter.output_to_db(config_input)
+        config_db.mod_config(data)
+        click.echo("Overriding completed. No service is restarted.")
+
 #
 # 'hostname' command
 #

scripts/sonic-kdump-config

@@ -17,12 +17,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
-import sys
 import argparse
-import shlex
+import json
 import os
+import shlex
+import sys
+import syslog
 import subprocess
-import json
+
 from swsscommon.swsscommon import ConfigDBConnector
 
 aboot_cfg_template ="/host/image-%s/kernel-cmdline"
@@ -437,7 +439,7 @@ def get_kdump_config_json(config_param):
 #  @param verbose If True, the function will display a few additinal information
 #  @param image   The image on which kdump settings are changed
 #  @return        True if the grub/cmdline cfg has changed, and False if it has not
-def cmd_kdump_enable(verbose, image=get_current_image()):
+def cmd_kdump_enable(verbose, image):
 
     kdump_enabled = get_kdump_administrative_mode()
     memory = get_kdump_memory()
@@ -460,75 +462,76 @@ def cmd_kdump_enable(verbose, image=get_current_image()):
 #  @param image   The image on which kdump settings are changed
 #  @return        True if the grub/cmdline cfg has changed, and False if it has not
 def cmd_kdump_config_next(verbose):
-    return cmd_kdump_enable(verbose, image=get_next_image())
+    image = get_next_image()
+    return cmd_kdump_enable(verbose, image)
 
-## Disable kdump
-#
-#  @param verbose If True, the function will display a few additional information
-#  @return        True if the grub/cmdline cfg has changed, and False if it has not
-def kdump_disable(verbose, kdump_enabled, memory, num_dumps, image, cmdline_file):
+
+def kdump_disable(verbose, image, booter_config_file_path):
+    """Unloads Kdump kernel and remove parameter `crashkernel=*` from configuration file of 
+    kernel boot loader.
+
+    Args:
+        image: A string represents SONiC image version.
+        booter_config_file_path: A string represents path of kernel boot loader configuration file.
+
+    Returns:
+        changes: If kernel booter configuration file was changed, returns True; Otherwise, returns
+        False.
+    """
     write_use_kdump(0)
 
     if verbose:
         print("Disabling kdump for image=[%s]\n" % image)
-    lines = [line.rstrip('\n') for line in open(cmdline_file)]
+
+    try:
+        with open(booter_config_file_path) as config_file_handler:
+            lines = [line.rstrip('\n') for line in config_file_handler]
+    except OSError as sys_err:
+        syslog.syslog(syslog.LOG_ERR, "Failed to open kernel booter configuration file: '{}', Error is: '{}'!"
+                      .format(booter_config_file_path, sys_err))
+        return False
+
     img_index = locate_image(lines, "loop=image-"+image)
 
     changed = False
+
     crash_kernel_mem = search_for_crash_kernel(lines[img_index])
     if crash_kernel_mem is None:
         print("kdump is already disabled")
     else:
         lines[img_index] = lines[img_index].replace("crashkernel="+crash_kernel_mem, "")
         changed = True
         if verbose:
-            print("Removed [%s] in %s" % ("crashkernel="+crash_kernel_mem, cmdline_file))
+            print("Removed [%s] in %s" % ("crashkernel="+crash_kernel_mem, booter_config_file_path))
 
     if changed:
-        rewrite_cfg(lines, cmdline_file)
+        rewrite_cfg(lines, booter_config_file_path)
 
     if not os.path.exists('/etc/sonic/config_db.json'):
         print_err("Startup configuration not found, Kdump configuration is not saved")
         return False
 
-    current_img = get_current_image();
-    if verbose:
-        print("Current image=[%s]\n" % current_img)
-    lines = [line.rstrip('\n') for line in open(grub_cfg)]
-    current_img_index = locate_image(lines, "loop=image-"+current_img)
-
-    changed = False
-    curr_crash_kernel_mem = search_for_crash_kernel(lines[current_img_index])
-    if curr_crash_kernel_mem is None:
-        print("Kdump is already disabled")
-    else:
-        lines[current_img_index] = lines[current_img_index].replace("crashkernel="+curr_crash_kernel_mem, "")
-        changed = True
-        if verbose:
-            print("Removed [%s] in grub.cfg" % ("crashkernel="+curr_crash_kernel_mem))
-
-    if changed:
-        rewrite_grub_cfg(lines, grub_cfg)
-
     return changed
 
+
 ## Command: Disable kdump
 #
 #  @param verbose If True, the function will display a few additional information
-#  @param image   The image on which kdump settings are changed
-def cmd_kdump_disable(verbose, image=get_current_image()):
+def cmd_kdump_disable(verbose):
 
+    image = get_current_image()
     kdump_enabled = get_kdump_administrative_mode()
     memory = get_kdump_memory()
     num_dumps = get_kdump_num_dumps()
+
     if verbose:
         print("configDB: kdump_enabled=%d memory=[%s] num_nums=%d" % (kdump_enabled, memory, num_dumps))
 
     if os.path.exists(grub_cfg):
-        return kdump_disable(verbose, kdump_enabled, memory, num_dumps, image, grub_cfg)
+        return kdump_disable(verbose, image, grub_cfg)
     elif open(machine_cfg, 'r').read().find('aboot_platform') >= 0:
         aboot_cfg = aboot_cfg_template % image
-        return kdump_disable(verbose, kdump_enabled, memory, num_dumps, image, aboot_cfg)
+        return kdump_disable(verbose, image, aboot_cfg)
     else:
         print("Feature not supported on this platform")
         return False
@@ -549,7 +552,8 @@ def cmd_kdump_memory(verbose, memory):
             memory_in_db = get_kdump_memory()
             memory_in_json = get_kdump_config_json("memory")
             if memory != crash_kernel_in_cmdline or memory != memory_in_db or memory != memory_in_json:
-                cmd_kdump_enable(verbose)
+                image = get_current_image()
+                cmd_kdump_enable(verbose, image)
                 print("Kdump updated memory will be only operational after the system reboots")
         else:
             num_dumps = get_kdump_num_dumps()
@@ -635,7 +639,8 @@ def main():
     changed = False
     try:
         if options.enable:
-            changed = cmd_kdump_enable(options.verbose)
+            image = get_current_image()
+            changed = cmd_kdump_enable(options.verbose, image)
         elif options.config_next:
             changed = cmd_kdump_config_next(options.verbose)
         elif options.disable:

tests/config_override_input/empty_input.json

@@ -0,0 +1,113 @@
+{
+    "running_config": {
+        "ACL_TABLE": {
+            "DATAACL": {
+                "policy_desc": "DATAACL",
+                "ports": [
+                    "Ethernet4"
+                ],
+                "stage": "ingress",
+                "type": "L3"
+            },
+            "NTP_ACL": {
+                "policy_desc": "NTP_ACL",
+                "services": [
+                    "NTP"
+                ],
+                "stage": "ingress",
+                "type": "CTRLPLANE"
+            }
+        },
+        "AUTO_TECHSUPPORT_FEATURE": {
+            "bgp": {
+                "rate_limit_interval": "600",
+                "state": "enabled"
+            },
+            "database": {
+                "rate_limit_interval": "600",
+                "state": "enabled"
+            }
+        },
+        "PORT": {
+            "Ethernet4": {
+                "admin_status": "up",
+                "alias": "fortyGigE0/4",
+                "description": "Servers0:eth0",
+                "index": "1",
+                "lanes": "29,30,31,32",
+                "mtu": "9100",
+                "pfc_asym": "off",
+                "speed": "40000",
+                "tpid": "0x8100"
+            },
+            "Ethernet8": {
+                "admin_status": "up",
+                "alias": "fortyGigE0/8",
+                "description": "Servers1:eth0",
+                "index": "2",
+                "lanes": "33,34,35,36",
+                "mtu": "9100",
+                "pfc_asym": "off",
+                "speed": "40000",
+                "tpid": "0x8100"
+            }
+        }
+    },
+    "golden_config": {
+
+    },
+    "expected_config": {
+        "ACL_TABLE": {
+            "DATAACL": {
+                "policy_desc": "DATAACL",
+                "ports": [
+                    "Ethernet4"
+                ],
+                "stage": "ingress",
+                "type": "L3"
+            },
+            "NTP_ACL": {
+                "policy_desc": "NTP_ACL",
+                "services": [
+                    "NTP"
+                ],
+                "stage": "ingress",
+                "type": "CTRLPLANE"
+            }
+        },
+        "AUTO_TECHSUPPORT_FEATURE": {
+            "bgp": {
+                "rate_limit_interval": "600",
+                "state": "enabled"
+            },
+            "database": {
+                "rate_limit_interval": "600",
+                "state": "enabled"
+            }
+        },
+        "PORT": {
+            "Ethernet4": {
+                "admin_status": "up",
+                "alias": "fortyGigE0/4",
+                "description": "Servers0:eth0",
+                "index": "1",
+                "lanes": "29,30,31,32",
+                "mtu": "9100",
+                "pfc_asym": "off",
+                "speed": "40000",
+                "tpid": "0x8100"
+            },
+            "Ethernet8": {
+                "admin_status": "up",
+                "alias": "fortyGigE0/8",
+                "description": "Servers1:eth0",
+                "index": "2",
+                "lanes": "33,34,35,36",
+                "mtu": "9100",
+                "pfc_asym": "off",
+                "speed": "40000",
+                "tpid": "0x8100"
+            }
+        }
+    }
+}