Add possibility to add kernel drivers to initramdisk

repos/system_upgrade/common/actors/commonleappdracutmodules/libraries/modscan.py

@@ -31,6 +31,7 @@
     'kernel-core',
     'kernel-modules',
     'keyutils',
+    'kmod',
     'lldpad',
     'lvm2',
     'mdadm',

repos/system_upgrade/common/actors/initramfs/checkinitramfstasks/libraries/checkinitramfstasks.py

@@ -6,11 +6,11 @@
 from leapp.models import TargetInitramfsTasks, UpgradeInitramfsTasks
 
 DRACUT_MOD_DIR = '/usr/lib/dracut/modules.d/'
-SUMMARY_DRACUT_FMT = (
-    'The requested dracut modules for the initramfs are in conflict.'
-    ' At least one dracut module is specified to be installed from'
-    ' multiple paths. The list of conflicting dracut module names'
-    ' with paths is listed below: {}'
+SUMMARY_FMT = (
+    'The requested {kind} modules for the initramfs are in conflict.'
+    ' At least one {kind} module is specified to be installed from'
+    ' multiple paths. The list of conflicting {kind} module names'
+    ' with paths is listed below: {conflicts}'
 )
 
 
@@ -22,51 +22,72 @@ def _printable_modules(conflicts):
     return ''.join(output)
 
 
-def _treat_path(dmodule):
+def _treat_path_dracut(dmodule):
     """
     In case the path is not set, set the expected path of the dracut module.
     """
+
     if not dmodule.module_path:
         return os.path.join(DRACUT_MOD_DIR, dmodule.name)
     return dmodule.module_path
 
 
-def _detect_dracut_modules_conflicts(msgtype):
+def _treat_path_kernel(kmodule):
+    """
+    In case the path of a kernel module is not set, indicate that the module is
+    taken from the current system.
+    """
+
+    if not kmodule.module_path:
+        return kmodule.name + ' (system)'
+    return kmodule.module_path
+
+
+def _detect_modules_conflicts(msgtype, kind):
     """
     Return dict of modules with conflicting tasks
 
-    In this case when a dracut module should be applied but different
-    sources are specified. E.g.:
-       include dracut modules X where,
+    In this case when a module should be applied but different sources are
+    specified. E.g.:
+       include modules X where,
          msg A)  X
          msg B)  X from custom path
     """
-    dracut_modules = defaultdict(set)
+
+    modules_map = {
+        'dracut': {
+            'msgattr': 'include_dracut_modules',
+            'treat_path_fn': _treat_path_dracut,
+        },
+        'kernel': {
+            'msgattr': 'include_kernel_modules',
+            'treat_path_fn': _treat_path_kernel
+        },
+    }
+
+    modules = defaultdict(set)
     for msg in api.consume(msgtype):
-        for dmodule in msg.include_dracut_modules:
-            dracut_modules[dmodule.name].add(_treat_path(dmodule))
-    return {key: val for key, val in dracut_modules.items() if len(val) > 1}
+        for module in getattr(msg, modules_map[kind]['msgattr']):
+            treat_path_fn = modules_map[kind]['treat_path_fn']
+            modules[module.name].add(treat_path_fn(module))
+    return {key: val for key, val in modules.items() if len(val) > 1}
+
+
+def report_conflicts(msgname, kind, msgtype):
+    conflicts = _detect_modules_conflicts(msgtype, kind)
+    if not conflicts:
+        return
+    report = [
+        reporting.Title('Conflicting requirements of {kind} modules for the {msgname} initramfs'.format(
+            kind=kind, msgname=msgname)),
+        reporting.Summary(SUMMARY_FMT.format(kind=kind, conflicts=_printable_modules(conflicts))),
+        reporting.Severity(reporting.Severity.HIGH),
+        reporting.Groups([reporting.Groups.SANITY, reporting.Groups.INHIBITOR]),
+    ]
+    reporting.create_report(report)
 
 
 def process():
-    conflicts = _detect_dracut_modules_conflicts(UpgradeInitramfsTasks)
-    if conflicts:
-        report = [
-            reporting.Title('Conflicting requirements of dracut modules for the upgrade initramfs'),
-            reporting.Summary(SUMMARY_DRACUT_FMT.format(_printable_modules(conflicts))),
-            reporting.Severity(reporting.Severity.HIGH),
-            reporting.Groups([reporting.Groups.SANITY]),
-            reporting.Groups([reporting.Groups.INHIBITOR]),
-        ]
-        reporting.create_report(report)
-
-    conflicts = _detect_dracut_modules_conflicts(TargetInitramfsTasks)
-    if conflicts:
-        report = [
-            reporting.Title('Conflicting requirements of dracut modules for the target initramfs'),
-            reporting.Summary(SUMMARY_DRACUT_FMT.format(_printable_modules(conflicts))),
-            reporting.Severity(reporting.Severity.HIGH),
-            reporting.Groups([reporting.Groups.SANITY]),
-            reporting.Groups([reporting.Groups.INHIBITOR]),
-        ]
-        reporting.create_report(report)
+    report_conflicts('upgrade', 'kernel', UpgradeInitramfsTasks)
+    report_conflicts('upgrade', 'dracut', UpgradeInitramfsTasks)
+    report_conflicts('target', 'dracut', TargetInitramfsTasks)

repos/system_upgrade/common/actors/initramfs/checkinitramfstasks/tests/unit_test_checkinitramfstasks.py

@@ -6,15 +6,16 @@
 from leapp.libraries.actor import checkinitramfstasks
 from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked
 from leapp.libraries.stdlib import api
-from leapp.models import DracutModule, Report, TargetInitramfsTasks, UpgradeInitramfsTasks
+from leapp.models import DracutModule, KernelModule, TargetInitramfsTasks, UpgradeInitramfsTasks
 from leapp.utils.report import is_inhibitor
 
 
 def gen_UIT(modules):
     if not isinstance(modules, list):
         modules = [modules]
     dracut_modules = [DracutModule(name=i[0], module_path=i[1]) for i in modules]
-    return UpgradeInitramfsTasks(include_dracut_modules=dracut_modules)
+    kernel_modules = [KernelModule(name=i[0], module_path=i[1]) for i in modules]
+    return UpgradeInitramfsTasks(include_dracut_modules=dracut_modules, include_kernel_modules=kernel_modules)
 
 
 def gen_TIT(modules):
@@ -71,9 +72,57 @@ def gen_TIT(modules):
         TargetInitramfsTasks,
     ),
 ])
-def test_conflict_detection(monkeypatch, expected_res, input_msgs, test_msg_type):
+def test_dracut_conflict_detection(monkeypatch, expected_res, input_msgs, test_msg_type):
     monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=input_msgs))
-    res = checkinitramfstasks._detect_dracut_modules_conflicts(test_msg_type)
+    res = checkinitramfstasks._detect_modules_conflicts(test_msg_type, 'dracut')
+    assert res == expected_res
+
+
+@pytest.mark.parametrize('expected_res,input_msgs,test_msg_type', [
+    (
+        {},
+        [],
+        UpgradeInitramfsTasks,
+    ),
+    (
+        {},
+        [gen_UIT([('modA', 'pathA'), ('modB', 'pathB')])],
+        UpgradeInitramfsTasks,
+    ),
+    (
+        {},
+        [gen_UIT([('modA', 'pathA'), ('modA', 'pathA')])],
+        UpgradeInitramfsTasks,
+    ),
+    (
+        {'modA': {'pathA', 'pathB'}},
+        [gen_UIT([('modA', 'pathA'), ('modA', 'pathB')])],
+        UpgradeInitramfsTasks,
+    ),
+    (
+        {'modA': {'pathA', 'pathB'}},
+        [gen_UIT(('modA', 'pathA')), gen_UIT(('modA', 'pathB'))],
+        UpgradeInitramfsTasks,
+    ),
+    (
+        {'modA': {'pathA', 'pathB'}},
+        [gen_UIT([('modA', 'pathA'), ('modA', 'pathB'), ('modB', 'pathC')])],
+        UpgradeInitramfsTasks,
+    ),
+    (
+        {'modA': {'modA (system)', 'pathB'}},
+        [gen_UIT([('modA', None), ('modA', 'pathB')])],
+        UpgradeInitramfsTasks,
+    ),
+    (
+        {},
+        [gen_UIT([('modA', 'pathA'), ('modA', 'pathB')])],
+        TargetInitramfsTasks,
+    ),
+])
+def test_kernel_conflict_detection(monkeypatch, expected_res, input_msgs, test_msg_type):
+    monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=input_msgs))
+    res = checkinitramfstasks._detect_modules_conflicts(test_msg_type, 'kernel')
     assert res == expected_res
 
 

repos/system_upgrade/common/actors/initramfs/targetinitramfsgenerator/libraries/targetinitramfsgenerator.py

@@ -1,3 +1,7 @@
+import errno
+import os
+import shutil
+
 from leapp.exceptions import StopActorExecutionError
 from leapp.libraries.stdlib import api, CalledProcessError, run
 from leapp.models import InitrdIncludes  # deprecated
@@ -7,25 +11,66 @@
 DRACUT_DIR = '/usr/lib/dracut/modules.d/'
 
 
-def copy_dracut_modules(modules):
+def _get_target_kernel_modules_dir(kernel_version):
+    """
+    Return the path where the custom kernel modules should be copied.
+    """
+
+    modules_dir = os.path.join('/', 'lib', 'modules', kernel_version, 'extra', 'leapp')
+
+    return modules_dir
+
+
+def _copy_modules(modules, dst_dir, kind):
     """
-    Copy every dracut module with specified path into the expected directory.
+    Copy modules of given kind to the specified destination directory.
+
+    Attempts to remove an cleanup by removing the existing destination
+    directory. If the directory does not exist, it is created anew. Then, for
+    each module message, it checks if the module has a module path specified. If
+    the module already exists in the destination directory, a debug message is
+    logged, and the operation is skipped. Otherwise, the module is copied to the
+    destination directory.
 
-    original content is overwritten if exists
     """
-    # FIXME: use just python functions instead of shell cmds
+
+    try:
+        os.makedirs(dst_dir)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST and os.path.isdir(dst_dir):
+            pass
+        else:
+            raise
+
     for module in modules:
         if not module.module_path:
             continue
+
+        dst_path = os.path.join(dst_dir, os.path.basename(module.module_path))
+        if os.path.exists(dst_path):
+            api.current_logger().debug(
+                'The {name} {kind} module has been already installed. Skipping.'
+                .format(name=module.name, kind=kind))
+            continue
+
+        copy_fn = shutil.copytree
+        if os.path.isfile(module.module_path):
+            copy_fn = shutil.copy2
+
         try:
-            # context.copytree_to(module.module_path, os.path.join(DRACUT_DIR, os.path.basename(module.module_path)))
-            run(['cp', '-f', '-a', module.module_path, DRACUT_DIR])
-        except CalledProcessError as e:
-            api.current_logger().error('Failed to copy dracut module "{name}" from "{source}" to "{target}"'.format(
-                name=module.name, source=module.module_path, target=DRACUT_DIR), exc_info=True)
-            # FIXME: really do we want to raise the error and stop execution completely??....
+            api.current_logger().debug(
+                'Copying {kind} module "{name}" to "{path}".'
+                .format(kind=kind, name=module.name, path=dst_path))
+
+            copy_fn(module.module_path, dst_path)
+        except shutil.Error as e:
+            api.current_logger().error(
+                    'Failed to copy {kind} module "{name}" from "{source}" to "{target}"'.format(
+                        kind=kind, name=module.name, source=module.module_path, target=dst_dir),
+                    exc_info=True)
             raise StopActorExecutionError(
-                message='Failed to install dracut modules required in the target initramfs. Error: {}'.format(str(e))
+                message='Failed to install {kind} modules required in the initram. Error: {error}'.format(
+                    kind=kind, error=str(e))
             )
 
 
@@ -43,17 +88,19 @@ def _get_modules():
     # supposed to create any such tasks before the reporting phase, so we
     # are able to check it.
     #
-    modules = []
+    modules = {'dracut': [], 'kernel': []}
     for task in api.consume(TargetInitramfsTasks):
-        modules.extend(task.include_dracut_modules)
+        modules['dracut'].extend(task.include_dracut_modules)
+        modules['kernel'].extend(task.include_kernel_modules)
+
     return modules
 
 
 def process():
     files = _get_files()
     modules = _get_modules()
 
-    if not files and not modules:
+    if not files and not modules['kernel'] and not modules['dracut']:
         api.current_logger().debug(
             'No additional files or modules required to add into the target initramfs.')
         return
@@ -65,15 +112,29 @@ def process():
             details={'Problem': 'Did not receive a message with installed RHEL-8 kernel version'
                                 ' (InstalledTargetKernelVersion)'})
 
-    copy_dracut_modules(modules)
+    _copy_modules(modules['dracut'], DRACUT_DIR, 'dracut')
+    _copy_modules(modules['kernel'], _get_target_kernel_modules_dir(target_kernel.version), 'kernel')
+
+    # Discover any new modules and regenerate modules.dep
+    should_regenerate = any(module.module_path is not None for module in modules['kernel'])
+    if should_regenerate:
+        try:
+            run(['depmod', target_kernel.version, '-a'])
+        except CalledProcessError as e:
+            raise StopActorExecutionError('Failed to generate modules.dep and map files.', details={'details': str(e)})
+
     try:
         # multiple files|modules need to be quoted, see --install | --add in dracut(8)
-        module_names = list({module.name for module in modules})
+        dracut_module_names = list({module.name for module in modules['dracut']})
+        kernel_module_names = list({module.name for module in modules['kernel']})
         cmd = ['dracut', '-f', '--kver', target_kernel.version]
         if files:
             cmd += ['--install', '{}'.format(' '.join(files))]
-        if modules:
-            cmd += ['--add', '{}'.format(' '.join(module_names))]
+        if modules['dracut']:
+            cmd += ['--add', '{}'.format(' '.join(dracut_module_names))]
+        if modules['kernel']:
+            cmd += ['--add-drivers', '{}'.format(' '.join(kernel_module_names))]
+
         run(cmd)
     except CalledProcessError as e:
         # just hypothetic check, it should not die