repair pipeline

Signed-off-by: Ronan Abhamon <[email protected]>

Author: Wescoeur

drivers/LVMSR.py

@@ -941,6 +941,7 @@ def _handleInterruptedCloneOp(self, origUuid, jval, forceUndo=False):
             raise util.SMException("base copy %s not present, " \
                     "but no original %s found" % (baseUuid, origUuid))
 
+        vdis = LvmCowUtil.getVDIInfo(self.lvmCache)
         base = vdis[baseUuid]
         cowutil = getCowUtil(base.vdiType)
 
@@ -954,7 +955,6 @@ def _handleInterruptedCloneOp(self, origUuid, jval, forceUndo=False):
             self._undoCloneOp(cowutil, lvs, origUuid, baseUuid, clonUuid)
             return
 
-        vdis = LvmCowUtil.getVDIInfo(self.lvmCache)
         if vdis[origUuid].scanError or (clonUuid and vdis[clonUuid].scanError):
             util.SMlog("One or both leaves invalid => revert")
             self._undoCloneOp(cowutil, lvs, origUuid, baseUuid, clonUuid)

drivers/LinstorSR.py

@@ -56,7 +56,7 @@
 import xmlrpc.client
 import xs_errors
 
-from cowutil import getCowUtil, getVdiTypeFromImageFormat
+from cowutil import getVdiTypeFromImageFormat
 from srmetadata import \
     NAME_LABEL_TAG, NAME_DESCRIPTION_TAG, IS_A_SNAPSHOT_TAG, SNAPSHOT_OF_TAG, \
     TYPE_TAG, VDI_TYPE_TAG, READ_ONLY_TAG, SNAPSHOT_TIME_TAG, \
@@ -156,7 +156,7 @@ def attach_thin(session, journaler, linstor, sr_uuid, vdi_uuid):
 
     device_path = linstor.get_device_path(vdi_uuid)
 
-    linstorcowutil = LinstorCowUtil(session, linstor, getCowUtil(vdi_type))
+    linstorcowutil = LinstorCowUtil(session, linstor, vdi_type)
 
     # If the virtual COW size is lower than the LINSTOR volume size,
     # there is nothing to do.
@@ -201,7 +201,7 @@ def check_vbd_count():
     util.retry(check_vbd_count, maxretry=10, period=1)
 
     device_path = linstor.get_device_path(vdi_uuid)
-    linstorcowutil = LinstorCowUtil(session, linstor, getCowUtil(vdi_type))
+    linstorcowutil = LinstorCowUtil(session, linstor, vdi_type)
     new_volume_size = LinstorVolumeManager.round_up_volume_size(
         # TODO: Replace pylint comment with this feature when possible:
         # https://github.com/PyCQA/pylint/pull/2926
@@ -1162,7 +1162,7 @@ def _load_vdis_ex(self):
                 if not VdiType.isCowImage(vdi_type):
                     managed = not volume_metadata.get(HIDDEN_TAG)
                 else:
-                    image_info = LinstorCowUtil(self.session, self._linstor, getCowUtil(vdi_type)).get_info(vdi_uuid)
+                    image_info = LinstorCowUtil(self.session, self._linstor, vdi_type).get_info(vdi_uuid)
                     managed = not image_info.hidden
                     if image_info.parentUuid:
                         sm_config['vhd-parent'] = image_info.parentUuid
@@ -1303,7 +1303,7 @@ def _get_vdi_path_and_parent(self, vdi_uuid, volume_name):
                 return (device_path, None)
 
             # Otherwise it's a COW and a parent can exist.
-            linstorcowutil = LinstorCowUtil(self.session, self._linstor, getCowUtil(vdi_type))
+            linstorcowutil = LinstorCowUtil(self.session, self._linstor, vdi_type)
             if linstorcowutil.check(vdi_uuid) != cowutil.CheckResult.Success:
                 return (None, None)
 
@@ -1419,7 +1419,7 @@ def _undo_clone(self, volume_names, vdi_uuid, base_uuid, snap_uuid):
             util.SMlog('*** INTERRUPTED CLONE OP: rollback fail')
             return
 
-        linstorcowutil = LinstorCowUtil(self.session, self._linstor, getCowUtil(base_type))
+        linstorcowutil = LinstorCowUtil(self.session, self._linstor, base_type)
 
         # Un-hide the parent.
         self._linstor.update_volume_metadata(base_uuid, {READ_ONLY_TAG: False})
@@ -1468,7 +1468,8 @@ def _undo_clone(self, volume_names, vdi_uuid, base_uuid, snap_uuid):
         # Inflate to the right size.
         if VdiType.isCowImage(base_type):
             vdi = self.vdi(vdi_uuid)
-            volume_size = linstorcowutil.compute_volume_size(vdi.size, vdi.vdi_type)
+            linstorcowutil = LinstorCowUtil(self.session, self._linstor, vdi.vdi_type)
+            volume_size = linstorcowutil.compute_volume_size(vdi.size)
             linstorcowutil.inflate(
                 self._journaler, vdi_uuid, vdi.path,
                 volume_size, vdi.capacity
@@ -1667,7 +1668,7 @@ def create(self, sr_uuid, vdi_uuid, size) -> str:
 
         # 2. Compute size and check space available.
         size = self.linstorcowutil.cowutil.validateAndRoundImageSize(int(size))
-        volume_size = self.linstorcowutil.compute_volume_size(size, self.vdi_type)
+        volume_size = self.linstorcowutil.compute_volume_size(size)
         util.SMlog(
             'LinstorVDI.create: type={}, cow-size={}, volume-size={}'
             .format(self.vdi_type, size, volume_size)
@@ -1831,7 +1832,7 @@ def attach(self, sr_uuid, vdi_uuid) -> str:
             if (
                 not VdiType.isCowImage(self.vdi_type) or
                 not writable or
-                self.capacity >= self.linstorcowutil.compute_volume_size(self.size, self.vdi_type)
+                self.capacity >= self.linstorcowutil.compute_volume_size(self.size)
             ):
                 need_inflate = False
 
@@ -1876,7 +1877,7 @@ def detach(self, sr_uuid, vdi_uuid) -> None:
 
         # The VDI is already deflated if the COW image size + metadata is
         # equal to the LINSTOR volume size.
-        volume_size = self.linstorcowutil.compute_volume_size(self.size, self.vdi_type)
+        volume_size = self.linstorcowutil.compute_volume_size(self.size)
         already_deflated = self.capacity <= volume_size
 
         if already_deflated:
@@ -1939,7 +1940,7 @@ def resize(self, sr_uuid, vdi_uuid, size) -> str:
 
         # Compute the virtual COW and DRBD volume size.
         size = self.linstorcowutil.cowutil.validateAndRoundImageSize(int(size))
-        volume_size = self.linstorcowutil.compute_volume_size(size, self.vdi_type)
+        volume_size = self.linstorcowutil.compute_volume_size(size)
         util.SMlog(
             'LinstorVDI.resize: type={}, cow-size={}, volume-size={}'
             .format(self.vdi_type, size, volume_size)
@@ -1964,7 +1965,7 @@ def resize(self, sr_uuid, vdi_uuid, size) -> str:
                 # VDI is currently deflated, so keep it deflated.
                 new_volume_size = old_volume_size
             else:
-                new_volume_size = self.linstorcowutil.compute_volume_size(size, self.vdi_type)
+                new_volume_size = self.linstorcowutil.compute_volume_size(size)
         assert new_volume_size >= old_volume_size
 
         space_needed = new_volume_size - old_volume_size

drivers/blktap2.py

@@ -51,6 +51,8 @@
 
 import VDI as sm
 
+from cowutil import getCowUtil
+
 # For RRDD Plugin Registration
 from xmlrpc.client import ServerProxy, Transport
 from socket import socket, AF_UNIX, SOCK_STREAM
@@ -1930,8 +1932,11 @@ def _setup_cache(self, session, sr_uuid, vdi_uuid, local_sr_uuid,
         from lock import Lock
         from FileSR import FileVDI
 
-        parent_uuid = self._cowutil.getParent(self.target.vdi.path,
-                FileVDI.extractUuid)
+        vdi_type = self.target.get_vdi_type()
+        tap_type = VDI._tap_type(vdi_type)
+        cowutil = getCowUtil(vdi_type)
+
+        parent_uuid = cowutil.getParent(self.target.vdi.path, FileVDI.extractUuid)
         if not parent_uuid:
             util.SMlog("ERROR: VDI %s has no parent, not enabling" % \
                     self.target.vdi.uuid)
@@ -1959,35 +1964,33 @@ def _setup_cache(self, session, sr_uuid, vdi_uuid, local_sr_uuid,
                     read_cache_path)
         else:
             try:
-                self._cowutil.snapshot(read_cache_path, shared_target.path, False)
+                cowutil.snapshot(read_cache_path, shared_target.path, False)
             except util.CommandException as e:
                 util.SMlog("Error creating parent cache: %s" % e)
                 self.alert_no_cache(session, vdi_uuid, local_sr_uuid, e.code)
                 return None
 
         # local write node
-        leaf_size = self._cowutil.getSizeVirt(self.target.vdi.path)
+        leaf_size = cowutil.getSizeVirt(self.target.vdi.path)
         local_leaf_path = "%s/%s.vhdcache" % \
                 (local_sr.path, self.target.vdi.uuid)
         if util.pathexists(local_leaf_path):
             util.SMlog("Local leaf node (%s) already exists, deleting" % \
                     local_leaf_path)
             os.unlink(local_leaf_path)
         try:
-            self._cowutil.snapshot(local_leaf_path, read_cache_path, False,
+            cowutil.snapshot(local_leaf_path, read_cache_path, False,
                     msize=leaf_size, checkEmpty=False)
         except util.CommandException as e:
             util.SMlog("Error creating leaf cache: %s" % e)
             self.alert_no_cache(session, vdi_uuid, local_sr_uuid, e.code)
             return None
 
-        local_leaf_size = self._cowutil.getSizeVirt(local_leaf_path)
+        local_leaf_size = cowutil.getSizeVirt(local_leaf_path)
         if leaf_size > local_leaf_size:
             util.SMlog("Leaf size %d > local leaf cache size %d, resizing" %
                     (leaf_size, local_leaf_size))
-            self._cowutil.setSizeVirtFast(local_leaf_path, leaf_size)
-
-        vdi_type = self.target.get_vdi_type()
+            cowutil.setSizeVirtFast(local_leaf_path, leaf_size)
 
         prt_tapdisk = Tapdisk.find_by_path(read_cache_path)
         if not prt_tapdisk:
@@ -2000,15 +2003,12 @@ def _setup_cache(self, session, sr_uuid, vdi_uuid, local_sr_uuid,
                 blktap.set_pool_name("lcache-parent-pool-%s" % blktap.minor)
                 # no need to change pool_size since each parent tapdisk is in
                 # its own pool
-                prt_tapdisk = \
-                    Tapdisk.launch_on_tap(blktap, read_cache_path,
-                            'vhd', parent_options)
+                prt_tapdisk = Tapdisk.launch_on_tap(blktap, read_cache_path, tap_type, parent_options)
             except:
                 blktap.free()
                 raise
 
-        secondary = "%s:%s" % (self.target.get_vdi_type(),
-                self.PhyLink.from_uuid(sr_uuid, vdi_uuid).readlink())
+        secondary = "%s:%s" % (vdi_type, self.PhyLink.from_uuid(sr_uuid, vdi_uuid).readlink())
 
         util.SMlog("Parent tapdisk: %s" % prt_tapdisk)
         leaf_tapdisk = Tapdisk.find_by_path(local_leaf_path)
@@ -2021,9 +2021,7 @@ def _setup_cache(self, session, sr_uuid, vdi_uuid, local_sr_uuid,
             child_options["secondary"] = secondary
             child_options["standby"] = scratch_mode
             try:
-                leaf_tapdisk = \
-                    Tapdisk.launch_on_tap(blktap, local_leaf_path,
-                            'vhd', child_options)
+                leaf_tapdisk = Tapdisk.launch_on_tap(blktap, local_leaf_path, tap_type, child_options)
             except:
                 blktap.free()
                 raise
@@ -2077,8 +2075,8 @@ def _remove_cache(self, session, local_sr_uuid):
         from lock import Lock
         from FileSR import FileVDI
 
-        parent_uuid = self._cowutil.getParent(self.target.vdi.path,
-                FileVDI.extractUuid)
+        vdi_type = self.target.get_vdi_type()
+        parent_uuid = getCowUtil(vdi_type).getParent(self.target.vdi.path, FileVDI.extractUuid)
         if not parent_uuid:
             util.SMlog("ERROR: No parent for VDI %s, ignore" % \
                     self.target.vdi.uuid)

drivers/cleanup.py

@@ -18,7 +18,7 @@
 # Script to coalesce and garbage collect COW-based SR's in the background
 #
 
-from sm_typing import Optional, override
+from sm_typing import Any, Optional, override
 
 import os
 import os.path
@@ -802,7 +802,6 @@ def __str__(self) -> str:
 
     def validate(self, fast=False) -> None:
         if self.cowutil.check(self.path, fast=fast) != CowUtil.CheckResult.Success:
-            print(f"TO_REMOVE 5 => '{str(self.cowutil.check(self.path, fast=fast))}' : '{CowUtil.CheckResult.Success}' ")
             raise util.SMException("COW image %s corrupted" % self)
 
     def _clear(self):
@@ -932,8 +931,6 @@ def _coalesceCowImage(self, timeOut):
         Util.log("  Running COW coalesce on %s" % self)
         abortTest = lambda: IPCFlag(self.sr.uuid).test(FLAG_TYPE_ABORT)
         try:
-            print(f"TO_REMOVE 2")
-
             util.fistpoint.activate_custom_fn(
                 "cleanup_coalesceVHD_inject_failure",
                 util.inject_failure)
@@ -946,7 +943,6 @@ def _coalesceCowImage(self, timeOut):
             parent = ""
             try:
                 parent = self.getParent()
-                print(f"TO_REMOVE 1 {parent} {self._vdi_is_raw(parent)}")
                 if not self._vdi_is_raw(parent):
                     # Repair error is logged and ignored. Error reraised later
                     util.SMlog('Coalesce failed on %s, attempting repair on ' \
@@ -1528,7 +1524,7 @@ def load(self, info=None) -> None:
 
         self.fileName = self.sr._linstor.get_volume_name(self.uuid)
         self.path = self.sr._linstor.build_device_path(self.fileName)
-        self.linstorcowutil = LinstorCowUtil(self.xapi.session, self.sr._linstor, self.cowutil)
+        self.linstorcowutil = LinstorCowUtil(self.sr.xapi.session, self.sr._linstor, self.cowutil)
 
         if not info:
             try:
@@ -1597,7 +1593,7 @@ def deflate(self):
 
     def inflateFully(self):
         if VdiType.isCowImage(self.vdi_type):
-            self.inflate(linstorcowutil.compute_volume_size(self.sizeVirt, self.vdi_type))
+            self.inflate(self.linstorcowutil.compute_volume_size(self.sizeVirt))
 
     @override
     def rename(self, uuid) -> None:
@@ -1786,7 +1782,7 @@ def _setSizeVirt(self, size) -> None:
             jfile, self.cowutil.getResizeJournalSize(), persistent=False, volume_name=jfile
         )
         try:
-            self.inflate(linstorcowutil.compute_volume_size(size, self.vdi_type))
+            self.inflate(self.linstorcowutil.compute_volume_size(size))
             self.linstorcowutil.set_size_virt(size, jfile)
         finally:
             try:
@@ -1810,9 +1806,7 @@ def _inflateParentForCoalesce(self):
     def _calcExtraSpaceForCoalescing(self) -> int:
         if not VdiType.isCowImage(self.parent.vdi_type):
             return 0
-        size_coalesced = linstorcowutil.compute_volume_size(
-            self._getCoalescedSizeData(), self.vdi_type
-        )
+        size_coalesced = self.linstorcowutil.compute_volume_size(self._getCoalescedSizeData())
         Util.log("Coalesced size = %s" % Util.num2str(size_coalesced))
         return size_coalesced - self.parent.getDrbdSize()
 
@@ -2122,7 +2116,6 @@ def coalesce(self, vdi, dryRun=False):
                 self.cleanup()
                 raise
             else:
-                print(f"TO_REMOVE 3 {e}")
                 self._failedCoalesceTargets.append(vdi)
                 Util.logException("coalesce")
                 Util.log("Coalesce failed, skipping")
@@ -3447,7 +3440,9 @@ def _finishCoalesceLeaf(self, parent) -> None:
 
     @override
     def _calcExtraSpaceNeeded(self, child, parent) -> int:
-        return linstorcowutil.compute_volume_size(parent.sizeVirt, parent.vdi_type) - parent.getDrbdSize()
+        return LinstorCowUtil(
+            self.xapi.session, self._linstor, parent.vdi_type
+        ).compute_volume_size(parent.sizeVirt) - parent.getDrbdSize()
 
     def _hasValidDevicePath(self, uuid):
         try:

drivers/cowutil.py

@@ -243,7 +243,7 @@ def check(
         path: str,
         ignoreMissingFooter: Optional[bool] = False,
         fast: Optional[bool] = False
-    ) -> CheckResult:
+    ) -> CowUtil.CheckResult:
         pass
 
     @abstractmethod

drivers/linstorcowutil.py

@@ -404,15 +404,17 @@ def _force_deflate(self, cowutil_inst, path, newSize, oldSize, zeroize):
     # Helpers.
     # --------------------------------------------------------------------------
 
-    def compute_volume_size(cls, virtual_size: int) -> int:
-        if VdiType.isCowImage(vdi_type):
+    def compute_volume_size(self, virtual_size: int) -> int:
+        if VdiType.isCowImage(self._vdi_type):
             # All LINSTOR VDIs have the metadata area preallocated for
             # the maximum possible virtual size (for fast online VDI.resize).
-            meta_overhead = self._cowutil.calcOverheadEmpty(self.MAX_SIZE) # TODO: getDefaultPreallocaqatio... + max
+            meta_overhead = self._cowutil.calcOverheadEmpty(
+                max(virtual_size, self._cowutil.getDefaultPreallocationSizeVirt())
+            )
             bitmap_overhead = self._cowutil.calcOverheadBitmap(virtual_size)
             virtual_size += meta_overhead + bitmap_overhead
         else:
-            raise Exception('Invalid image type: {}'.format(vdi_type))
+            raise Exception('Invalid image type: {}'.format(self._vdi_type))
 
         return LinstorVolumeManager.round_up_volume_size(virtual_size)
 
@@ -585,7 +587,7 @@ def remote_call():
             )
         return util.retry(remote_call, 5, 2)
 
-    def _zeroize(path, size):
+    def _zeroize(self, path, size):
         if not util.zeroOut(path, size, self._cowutil.getFooterSize()):
             raise xs_errors.XenError(
                 'EIO',

drivers/lvmcowutil.py

@@ -27,7 +27,7 @@
 
 import lock
 import util
-import XenAPI
+import XenAPI # pylint: disable=import-error
 
 from constants import NS_PREFIX_LVM, VG_LOCATION, VG_PREFIX
 from cowutil import CowImageInfo, CowUtil, getCowUtil

drivers/verifyVHDsOnSR.py

@@ -24,13 +24,12 @@
 import sys
 import util
 import lvutil
-import vhdutil
 
 import VDI
 
 from constants import NS_PREFIX_LVM, VG_LOCATION, VG_PREFIX
 from lock import Lock
-from lvmcowutil import LV_PREFIX
+from lvmcowutil import LV_PREFIX, LvmCowUtil
 from refcounter import RefCounter
 from vditype import VdiType
 
@@ -116,10 +115,11 @@ def checkAllVHD(sr_uuid):
     VHDs_total = 0
 
     vg_name = VG_PREFIX + sr_uuid
-    pattern = "%s*" % LV_PREFIX[VdiType.VHD]
+    vdi_type = VdiType.VHD
+    pattern = "%s*" % LV_PREFIX[vdi_type]
 
     # Do a vhd scan and gets all the VHDs
-    vhds = CowUtil.getAllInfoFromVG(pattern, LvmCowUtil.extractUuid, vg_name)
+    vhds = getCowUtil(vdi_type).getAllInfoFromVG(pattern, LvmCowUtil.extractUuid, vg_name)
     VHDs_total = len(vhds)
 
     # Build VHD chain, that way it will be easier to activate all the VHDs