CP-54207: Move VBD_attach outside of VM migrate downtime

VBDs can be attached to multiple VMs, so now that VBD_plug has been
split into VBD_attach and VBD_activate, the attach can happen outside of
the VM migrate downtime. This doesn't change the overall duration of the
migration but can reduce the downtime by several seconds.

This new functionality is dependent on two flags:
firstly, xenopsd_vbd_plug_unplug_legacy must be false so that the
VBD_attach and VBD_activate are separate atoms. This is off by default.
Then there is another flag can_attach_early which is currently true iff
the VBD's SM has required_api_version >= 3.0

Signed-off-by: Steven Woods <[email protected]>

Author: snwoods

ocaml/xapi-idl/xen/xenops_interface.ml

@@ -303,6 +303,7 @@ module Vbd = struct
     ; extra_private_keys: (string * string) list [@default []]
     ; qos: qos option [@default None]
     ; persistent: bool [@default true]
+    ; can_attach_early: bool [@default false]
   }
   [@@deriving rpcty]
 

ocaml/xapi/xapi_sr.ml

@@ -1080,3 +1080,15 @@ let get_live_hosts ~__context ~sr =
     Xapi_vm_helpers.assert_can_see_specified_SRs ~__context ~reqd_srs:[sr] ~host
   in
   Xapi_vm_helpers.possible_hosts ~__context ~choose_fn ()
+
+let required_api_version_of_sr ~__context ~sr =
+  let sr_type = Db.SR.get_type ~__context ~self:sr in
+  let expr =
+    Xapi_database.Db_filter_types.(Eq (Field "type", Literal sr_type))
+  in
+  match Db.SM.get_records_where ~__context ~expr with
+  | (_, sm) :: _ ->
+      Some sm.API.sM_required_api_version
+  | [] ->
+      warn "Couldn't find SM with type %s" sr_type ;
+      None

ocaml/xapi/xapi_xenops.ml

@@ -553,6 +553,10 @@ let list_net_sriov_vf_pcis ~__context ~vm =
              None
      )
 
+module StringMap = Map.Make (String)
+
+let sr_version_cache = ref StringMap.empty
+
 module MD = struct
   (** Convert between xapi DB records and xenopsd records *)
 
@@ -684,6 +688,28 @@ module MD = struct
       ) else
         disk_of_vdi ~__context ~self:vbd.API.vBD_VDI
     in
+    let can_attach_early =
+      let vdi = vbd.API.vBD_VDI in
+      try
+        let sr = Db.VDI.get_SR ~__context ~self:vdi in
+        let sr_key = Ref.string_of sr in
+        match StringMap.find_opt sr_key !sr_version_cache with
+        | Some cached_api_version ->
+            Version.String.ge cached_api_version "3.0"
+        | None -> (
+          match Xapi_sr.required_api_version_of_sr ~__context ~sr with
+          | Some api_version ->
+              sr_version_cache :=
+                StringMap.add sr_key api_version !sr_version_cache ;
+              Version.String.ge api_version "3.0"
+          | None ->
+              false
+        )
+      with Db_exn.DBCache_NotFound (_, _, _) as e ->
+        info "Caught error %s; Defaulting can_attach_early to false"
+          (Printexc.to_string e) ;
+        false
+    in
     {
       id= (vm.API.vM_uuid, Device_number.to_linux_device device_number)
     ; position= Some device_number
@@ -707,6 +733,7 @@ module MD = struct
         ( try Db.VDI.get_on_boot ~__context ~self:vbd.API.vBD_VDI = `persist
           with _ -> true
         )
+    ; can_attach_early
     }
 
   let of_pvs_proxy ~__context vif proxy =

ocaml/xenopsd/cli/xn.ml

@@ -278,6 +278,7 @@ let vbd_of_disk_info vm_id info =
   ; extra_private_keys= []
   ; qos= None
   ; persistent= true
+  ; can_attach_early= false
   }
 
 let print_disk vbd =

ocaml/xenopsd/lib/xenops_server.ml

@@ -1763,7 +1763,8 @@ let rec atomics_of_operation = function
           serial "VIF.activate_and_plug" ~id
             [VIF_set_active (vif.Vif.id, true); VIF_plug vif.Vif.id]
       )
-  | VM_restore_devices (id, restore_vifs) ->
+  | VM_restore_devices (id, migration) ->
+      let restore_vifs = not migration in
       let vbds_rw, vbds_ro = VBD_DB.vbds id |> vbd_plug_sets in
       let vgpus = VGPU_DB.vgpus id in
       let pcis = PCI_DB.pcis id |> pci_plug_order in
@@ -1773,8 +1774,22 @@ let rec atomics_of_operation = function
         let name_multi = pf "VBDs.activate_and_plug %s" typ in
         let name_one = pf "VBD.activate_and_plug %s" typ in
         parallel_map name_multi ~id vbds (fun vbd ->
-            serial name_one ~id
-              [VBD_set_active (vbd.Vbd.id, true); vbd_plug vbd.Vbd.id]
+            (* When migrating, attach early if the vbd's SM allows it.
+               Note: there is a bug here for SxM if migrating between API
+               versions as the Vbd's new SR won't have propagated to xenopsd
+               yet. This means can_attach_early will be based on the origin SR.
+               This is a non-issue as v1 <-> v3 migration is still experimental
+               and v1 is already early-attaching in SxM through mirroring.
+            *)
+            if
+              migration
+              && (not !xenopsd_vbd_plug_unplug_legacy)
+              && vbd.Vbd.can_attach_early
+            then
+              [VBD_activate vbd.Vbd.id]
+            else
+              serial name_one ~id
+                [VBD_set_active (vbd.Vbd.id, true); vbd_plug vbd.Vbd.id]
         )
       in
       [
@@ -1897,7 +1912,7 @@ let rec atomics_of_operation = function
         ]
       ; vgpu_start_operations
       ; [VM_restore (id, data, vgpu_data)]
-      ; atomics_of_operation (VM_restore_devices (id, true))
+      ; atomics_of_operation (VM_restore_devices (id, false))
       ; [
           (* At this point the domain is considered survivable. *)
           VM_set_domain_action_request (id, None)
@@ -2696,9 +2711,9 @@ and perform_exn ?result (op : operation) (t : Xenops_task.task_handle) : unit =
   | VM_restore_vifs id ->
       debug "VM_restore_vifs %s" id ;
       perform_atomics (atomics_of_operation op) t
-  | VM_restore_devices (id, restore_vifs) ->
+  | VM_restore_devices (id, migration) ->
       (* XXX: this is delayed due to the 'attach'/'activate' behaviour *)
-      debug "VM_restore_devices %s %b" id restore_vifs ;
+      debug "VM_restore_devices %s %b" id migration ;
       perform_atomics (atomics_of_operation op) t
   | VM_resume (id, _data) ->
       debug "VM.resume %s" id ;
@@ -3022,11 +3037,31 @@ and perform_exn ?result (op : operation) (t : Xenops_task.task_handle) : unit =
           ( try
               let no_sharept = VGPU_DB.vgpus id |> List.exists is_no_sharept in
               debug "VM %s no_sharept=%b (%s)" id no_sharept __LOC__ ;
+              (* If plug is split into activate and attach, we could attach
+                 early so that it is outside of the VM downtime (if the SM
+                 supports this) *)
+              let early_attach =
+                parallel_map "VBDs.set_active_and_attach" ~id (VBD_DB.vbds id)
+                  (fun vbd ->
+                    if
+                      (not !xenopsd_vbd_plug_unplug_legacy)
+                      && vbd.Vbd.can_attach_early
+                    then
+                      serial "VBD.set_active_and_attach" ~id
+                        [
+                          VBD_set_active (vbd.Vbd.id, true)
+                        ; VBD_attach vbd.Vbd.id
+                        ]
+                    else
+                      []
+                )
+              in
               perform_atomics
                 ([VM_create (id, Some memory_limit, Some final_id, no_sharept)]
-                @ (* Perform as many operations as possible on the destination
-                     domain before pausing the original domain *)
-                atomics_of_operation (VM_restore_vifs id)
+                (* Perform as many operations as possible on the destination
+                   domain before pausing the original domain *)
+                @ atomics_of_operation (VM_restore_vifs id)
+                @ early_attach
                 )
                 t ;
               Handshake.send s Handshake.Success
@@ -3142,7 +3177,7 @@ and perform_exn ?result (op : operation) (t : Xenops_task.task_handle) : unit =
         ) ;
         debug "VM.receive_memory: restoring remaining devices and unpausing" ;
         perform_atomics
-          (atomics_of_operation (VM_restore_devices (final_id, false))
+          (atomics_of_operation (VM_restore_devices (final_id, true))
           @ [
               VM_unpause final_id
             ; VM_set_domain_action_request (final_id, None)