[sonic-package-manager] support sonic-cli-gen and packages with YANG model (#1650)

- What I did
This PR brings in support for packages with YANG models and CLI auto generation capabilities for 3rd party packages.

- How I did it
Packages can set two new flags in manifest - "auto-generate-show" and "auto-generate-config" in addition to YANG module recorded in package image label "com.azure.sonic.yang-module".

- How to verify it
Build and run. Prepare some package with YANG model and test CLI is generated for it.

Signed-off-by: Stepan Blyshchak <[email protected]>
Co-authored-by: Vadym Hlushko <[email protected]>

Author: stepanblyschak

config/config_mgmt.py

@@ -2,8 +2,13 @@
 config_mgmt.py provides classes for configuration validation and for Dynamic
 Port Breakout.
 '''
+
+import os
 import re
+import shutil
 import syslog
+import tempfile
+import yang as ly
 from json import load
 from sys import flags
 from time import sleep as tsleep
@@ -46,34 +51,38 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True):
         try:
             self.configdbJsonIn = None
             self.configdbJsonOut = None
+            self.source = source
             self.allowTablesWithoutYang = allowTablesWithoutYang
 
             # logging vars
             self.SYSLOG_IDENTIFIER = "ConfigMgmt"
             self.DEBUG = debug
 
-            self.sy = sonic_yang.SonicYang(YANG_DIR, debug=debug)
-            # load yang models
-            self.sy.loadYangModel()
-            # load jIn from config DB or from config DB json file.
-            if source.lower() == 'configdb':
-                self.readConfigDB()
-            # treat any other source as file input
-            else:
-                self.readConfigDBJson(source)
-            # this will crop config, xlate and load.
-            self.sy.loadData(self.configdbJsonIn)
-
-            # Raise if tables without YANG models are not allowed but exist.
-            if not allowTablesWithoutYang and len(self.sy.tablesWithOutYang):
-                raise Exception('Config has tables without YANG models')
+            self.__init_sonic_yang()
 
         except Exception as e:
             self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e))
             raise Exception('ConfigMgmt Class creation failed')
 
         return
 
+    def __init_sonic_yang(self):
+        self.sy = sonic_yang.SonicYang(YANG_DIR, debug=self.DEBUG)
+        # load yang models
+        self.sy.loadYangModel()
+        # load jIn from config DB or from config DB json file.
+        if self.source.lower() == 'configdb':
+            self.readConfigDB()
+        # treat any other source as file input
+        else:
+            self.readConfigDBJson(self.source)
+        # this will crop config, xlate and load.
+        self.sy.loadData(self.configdbJsonIn)
+
+        # Raise if tables without YANG models are not allowed but exist.
+        if not self.allowTablesWithoutYang and len(self.sy.tablesWithOutYang):
+            raise Exception('Config has tables without YANG models')
+
     def __del__(self):
         pass
 
@@ -213,6 +222,69 @@ def writeConfigDB(self, jDiff):
 
         return
 
+    def add_module(self, yang_module_str):
+        """
+        Validate and add new YANG module to the system.
+
+        Parameters:
+            yang_module_str (str): YANG module in string representation.
+
+        Returns:
+            None
+        """
+
+        module_name = self.get_module_name(yang_module_str)
+        module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name))
+        if os.path.exists(module_path):
+            raise Exception('{} already exists'.format(module_name))
+        with open(module_path, 'w') as module_file:
+            module_file.write(yang_module_str)
+        try:
+            self.__init_sonic_yang()
+        except Exception:
+            os.remove(module_path)
+            raise
+
+    def remove_module(self, module_name):
+        """
+        Remove YANG module from the system and validate.
+
+        Parameters:
+            module_name (str): YANG module name.
+
+        Returns:
+            None
+        """
+
+        module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name))
+        if not os.path.exists(module_path):
+            return
+        temp = tempfile.NamedTemporaryFile(delete=False)
+        try:
+            shutil.move(module_path, temp.name)
+            self.__init_sonic_yang()
+        except Exception:
+            shutil.move(temp.name, module_path)
+            raise
+
+    @staticmethod
+    def get_module_name(yang_module_str):
+        """
+        Read yangs module name from yang_module_str
+
+        Parameters:
+            yang_module_str(str): YANG module string.
+
+        Returns:
+            str: Module name
+        """
+
+        # Instantiate new context since parse_module_mem() loads the module into context.
+        sy = sonic_yang.SonicYang(YANG_DIR)
+        module = sy.ctx.parse_module_mem(yang_module_str, ly.LYS_IN_YANG)
+        return module.name()
+
+
 # End of Class ConfigMgmt
 
 class ConfigMgmtDPB(ConfigMgmt):
@@ -417,8 +489,8 @@ def _deletePorts(self, ports=list(), force=False):
                     deps.extend(dep)
 
             # No further action with no force and deps exist
-            if force == False and deps:
-                return configToLoad, deps, False;
+            if not force and deps:
+                return configToLoad, deps, False
 
             # delets all deps, No topological sort is needed as of now, if deletion
             # of deps fails, return immediately
@@ -436,8 +508,8 @@ def _deletePorts(self, ports=list(), force=False):
                 self.sy.deleteNode(str(xPathPort))
 
             # Let`s Validate the tree now
-            if self.validateConfigData()==False:
-                return configToLoad, deps, False;
+            if not self.validateConfigData():
+                return configToLoad, deps, False
 
             # All great if we are here, Lets get the diff
             self.configdbJsonOut = self.sy.getData()

sonic_package_manager/main.py

@@ -414,10 +414,11 @@ def reset(ctx, name, force, yes, skip_host_plugins):
 
 @cli.command()
 @add_options(PACKAGE_COMMON_OPERATION_OPTIONS)
+@click.option('--keep-config', is_flag=True, help='Keep features configuration in CONFIG DB.')
 @click.argument('name')
 @click.pass_context
 @root_privileges_required
-def uninstall(ctx, name, force, yes):
+def uninstall(ctx, name, force, yes, keep_config):
     """ Uninstall package. """
 
     manager: PackageManager = ctx.obj
@@ -428,6 +429,7 @@ def uninstall(ctx, name, force, yes):
 
     uninstall_opts = {
         'force': force,
+        'keep_config': keep_config,
     }
 
     try:

sonic_package_manager/manager.py

@@ -10,8 +10,11 @@
 
 import docker
 import filelock
+from config import config_mgmt
 from sonic_py_common import device_info
 
+from sonic_cli_gen.generator import CliGenerator
+
 from sonic_package_manager import utils
 from sonic_package_manager.constraint import (
     VersionConstraint,
@@ -45,7 +48,10 @@
     run_command
 )
 from sonic_package_manager.service_creator.feature import FeatureRegistry
-from sonic_package_manager.service_creator.sonic_db import SonicDB
+from sonic_package_manager.service_creator.sonic_db import (
+    INIT_CFG_JSON,
+    SonicDB
+)
 from sonic_package_manager.service_creator.utils import in_chroot
 from sonic_package_manager.source import (
     PackageSource,
@@ -435,13 +441,16 @@ def install_from_source(self,
 
     @under_lock
     @opt_check
-    def uninstall(self, name: str, force=False):
+    def uninstall(self, name: str,
+                  force: bool = False,
+                  keep_config: bool = False):
         """ Uninstall SONiC Package referenced by name. The uninstallation
         can be forced if force argument is True.
 
         Args:
             name: SONiC Package name.
             force: Force the installation.
+            keep_config: Keep feature configuration in databases.
         Raises:
             PackageManagerError
         """
@@ -482,7 +491,7 @@ def uninstall(self, name: str, force=False):
             self._systemctl_action(package, 'stop')
             self._systemctl_action(package, 'disable')
             self._uninstall_cli_plugins(package)
-            self.service_creator.remove(package)
+            self.service_creator.remove(package, keep_config=keep_config)
             self.service_creator.generate_shutdown_sequence_files(
                 self._get_installed_packages_except(package)
             )
@@ -1000,9 +1009,13 @@ def get_manager() -> 'PackageManager':
         docker_api = DockerApi(docker.from_env(), ProgressManager())
         registry_resolver = RegistryResolver()
         metadata_resolver = MetadataResolver(docker_api, registry_resolver)
+        cfg_mgmt = config_mgmt.ConfigMgmt(source=INIT_CFG_JSON)
+        cli_generator = CliGenerator(log)
         feature_registry = FeatureRegistry(SonicDB)
         service_creator = ServiceCreator(feature_registry,
-                                         SonicDB)
+                                         SonicDB,
+                                         cli_generator,
+                                         cfg_mgmt)
 
         return PackageManager(docker_api,
                               registry_resolver,

sonic_package_manager/manifest.py

@@ -205,7 +205,9 @@ def unmarshal(self, value):
             ManifestField('mandatory', DefaultMarshaller(bool), False),
             ManifestField('show', DefaultMarshaller(str), ''),
             ManifestField('config', DefaultMarshaller(str), ''),
-            ManifestField('clear', DefaultMarshaller(str), '')
+            ManifestField('clear', DefaultMarshaller(str), ''),
+            ManifestField('auto-generate-show', DefaultMarshaller(bool), False),
+            ManifestField('auto-generate-config', DefaultMarshaller(bool), False),
         ])
     ])
 

sonic_package_manager/metadata.py

@@ -4,7 +4,7 @@
 
 import json
 import tarfile
-from typing import Dict
+from typing import Dict, Optional
 
 from sonic_package_manager import utils
 from sonic_package_manager.errors import MetadataError
@@ -54,6 +54,7 @@ class Metadata:
 
     manifest: Manifest
     components: Dict[str, Version] = field(default_factory=dict)
+    yang_module_str: Optional[str] = None
 
 
 class MetadataResolver:
@@ -163,5 +164,6 @@ def from_labels(cls, labels: Dict[str, str]) -> Metadata:
                 except ValueError as err:
                     raise MetadataError(f'Failed to parse component version: {err}')
 
-        return Metadata(Manifest.marshal(manifest_dict), components)
+        yang_module_str = sonic_metadata.get('yang-module')
 
+        return Metadata(Manifest.marshal(manifest_dict), components, yang_module_str)