[sonic-py-common] Add 'general' module with load_module_from_source() function (#7167)

#### Why I did it

To eliminate the need to write duplicate code in order to import a Python module from a source file.

#### How I did it

Add `general` module to sonic-py-common, which contains a `load_module_from_source()` function which supports both Python 2 and 3.

Call this new function in:
- sonic-ctrmgrd/tests/container_test.py
- sonic-ctrmgrd/tests/ctrmgr_tools_test.py
- sonic-host-services/tests/determine-reboot-cause_test.py
- sonic-host-services/tests/hostcfgd/hostcfgd_test.py
- sonic-host-services/tests/procdockerstatsd_test.py
- sonic-py-common/sonic_py_common/daemon_base.py

Author: jleveque

src/sonic-ctrmgrd/setup.py

@@ -24,6 +24,7 @@
     tests_require=[
         'pytest',
         'pytest-cov',
+        'sonic-py-common',
     ],
     install_requires=['netaddr', 'pyyaml'],
     license="GNU General Public License v3",

src/sonic-ctrmgrd/tests/common_test.py

@@ -1,6 +1,4 @@
 import copy
-import importlib.machinery
-import importlib.util
 import json
 import os
 import subprocess
@@ -655,13 +653,3 @@ def create_remote_ctr_config_json():
         s.write(str_conf)
 
     return fname
-
-
-def load_mod_from_file(modname, fpath):
-    spec = importlib.util.spec_from_loader(modname,
-            importlib.machinery.SourceFileLoader(modname, fpath))
-    mod = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(mod)
-    sys.modules[modname] = mod
-    return mod
-

src/sonic-ctrmgrd/tests/container_test.py

@@ -2,12 +2,14 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
+from sonic_py_common.general import load_module_from_source
 
 from . import common_test
 
-common_test.load_mod_from_file("docker",
+
+load_module_from_source("docker",
         os.path.join(os.path.dirname(os.path.realpath(__file__)), "mock_docker.py"))
-container = common_test.load_mod_from_file("container",
+container = load_module_from_source("container",
         os.path.join(os.path.dirname(os.path.realpath(__file__)), "../ctrmgr/container"))
 
 

src/sonic-ctrmgrd/tests/ctrmgr_tools_test.py

@@ -3,10 +3,11 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
+from sonic_py_common.general import load_module_from_source
 
 from . import common_test
 
-common_test.load_mod_from_file("docker",
+load_module_from_source("docker",
         os.path.join(os.path.dirname(os.path.realpath(__file__)), "mock_docker.py"))
 
 sys.path.append("ctrmgr")

src/sonic-host-services/setup.py

@@ -35,6 +35,7 @@
     ],
     tests_require = [
         'pytest',
+        'sonic-py-common'
     ],
     classifiers = [
         'Development Status :: 3 - Alpha',

src/sonic-host-services/tests/determine-reboot-cause_test.py

@@ -1,10 +1,9 @@
-import importlib.machinery
-import importlib.util
 import sys
 import os
 import pytest
 
 import swsssdk
+from sonic_py_common.general import load_module_from_source
 
 # TODO: Remove this if/else block once we no longer support Python 2
 if sys.version_info.major == 3:
@@ -31,11 +30,7 @@
 
 # Load the file under test
 determine_reboot_cause_path = os.path.join(scripts_path, 'determine-reboot-cause')
-loader = importlib.machinery.SourceFileLoader('determine_reboot_cause', determine_reboot_cause_path)
-spec = importlib.util.spec_from_loader(loader.name, loader)
-determine_reboot_cause = importlib.util.module_from_spec(spec)
-loader.exec_module(determine_reboot_cause)
-sys.modules['determine_reboot_cause'] = determine_reboot_cause
+determine_reboot_cause = load_module_from_source('determine_reboot_cause', determine_reboot_cause_path)
 
 
 PROC_CMDLINE_CONTENTS = """\

src/sonic-host-services/tests/hostcfgd/hostcfgd_test.py

@@ -1,13 +1,13 @@
-import importlib.machinery
-import importlib.util
 import os
 import sys
 import swsssdk
 
 from parameterized import parameterized
+from sonic_py_common.general import load_module_from_source
 from unittest import TestCase, mock
-from tests.hostcfgd.test_vectors import HOSTCFGD_TEST_VECTOR
-from tests.hostcfgd.mock_configdb import MockConfigDb
+
+from .test_vectors import HOSTCFGD_TEST_VECTOR
+from .mock_configdb import MockConfigDb
 
 
 swsssdk.ConfigDBConnector = MockConfigDb
@@ -18,11 +18,7 @@
 
 # Load the file under test
 hostcfgd_path = os.path.join(scripts_path, 'hostcfgd')
-loader = importlib.machinery.SourceFileLoader('hostcfgd', hostcfgd_path)
-spec = importlib.util.spec_from_loader(loader.name, loader)
-hostcfgd = importlib.util.module_from_spec(spec)
-loader.exec_module(hostcfgd)
-sys.modules['hostcfgd'] = hostcfgd
+hostcfgd = load_module_from_source('hostcfgd', hostcfgd_path)
 
 
 class TestHostcfgd(TestCase):

src/sonic-host-services/tests/procdockerstatsd_test.py

@@ -1,10 +1,9 @@
-import importlib.machinery
-import importlib.util
 import sys
 import os
 import pytest
 
 import swsssdk
+from sonic_py_common.general import load_module_from_source
 
 from .mock_connector import MockConnector
 
@@ -17,11 +16,7 @@
 
 # Load the file under test
 procdockerstatsd_path = os.path.join(scripts_path, 'procdockerstatsd')
-loader = importlib.machinery.SourceFileLoader('procdockerstatsd', procdockerstatsd_path)
-spec = importlib.util.spec_from_loader(loader.name, loader)
-procdockerstatsd = importlib.util.module_from_spec(spec)
-loader.exec_module(procdockerstatsd)
-sys.modules['procdockerstatsd'] = procdockerstatsd
+procdockerstatsd = load_module_from_source('procdockerstatsd', procdockerstatsd_path)
 
 class TestProcDockerStatsDaemon(object):
     def test_convert_to_bytes(self):

src/sonic-py-common/sonic_py_common/daemon_base.py

@@ -2,6 +2,7 @@
 import sys
 
 from . import device_info
+from .general import load_module_from_source
 from .logger import Logger
 
 #
@@ -25,29 +26,6 @@ def db_connect(db_name, namespace=EMPTY_NAMESPACE):
     return swsscommon.DBConnector(db_name, REDIS_TIMEOUT_MSECS, True, namespace)
 
 
-# TODO: Consider moving this logic out of daemon_base and into antoher file
-# so that it can be used by non-daemons. We can simply call that function here
-# to retain backward compatibility.
-def _load_module_from_file(module_name, file_path):
-    module = None
-
-    # TODO: Remove this check once we no longer support Python 2
-    if sys.version_info.major == 3:
-        import importlib.machinery
-        import importlib.util
-        loader = importlib.machinery.SourceFileLoader(module_name, file_path)
-        spec = importlib.util.spec_from_loader(loader.name, loader)
-        module = importlib.util.module_from_spec(spec)
-        loader.exec_module(module)
-    else:
-        import imp
-        module = imp.load_source(module_name, file_path)
-
-    sys.modules[module_name] = module
-
-    return module
-
-
 #
 # DaemonBase ===================================================================
 #
@@ -92,7 +70,7 @@ def load_platform_util(self, module_name, class_name):
 
         try:
             module_file = "/".join([platform_path, "plugins", module_name + ".py"])
-            module = _load_module_from_file(module_name, module_file)
+            module = load_module_from_source(module_name, module_file)
         except IOError as e:
             raise IOError("Failed to load platform module '%s': %s" % (module_name, str(e)))
 

src/sonic-py-common/sonic_py_common/general.py

@@ -0,0 +1,25 @@
+import sys
+
+
+def load_module_from_source(module_name, file_path):
+    """
+    This function will load the Python source file specified by <file_path>
+    as a module named <module_name> and return an instance of the module
+    """
+    module = None
+
+    # TODO: Remove this check once we no longer support Python 2
+    if sys.version_info.major == 3:
+        import importlib.machinery
+        import importlib.util
+        loader = importlib.machinery.SourceFileLoader(module_name, file_path)
+        spec = importlib.util.spec_from_loader(loader.name, loader)
+        module = importlib.util.module_from_spec(spec)
+        loader.exec_module(module)
+    else:
+        import imp
+        module = imp.load_source(module_name, file_path)
+
+    sys.modules[module_name] = module
+
+    return module