Update worker thread sender on x86

On x86, use the same sender as we would for macOS. Additionally, rather
than using an EventFd to determine when the thread work is done, use a
response sender/receiver like macOS.

Signed-off-by: Jake Correnti <[email protected]>

Author: jakecorrenti

src/devices/src/legacy/ioapic.rs

@@ -1,3 +1,4 @@
+use crossbeam_channel::unbounded;
 use kvm_bindings::{
     kvm_enable_cap, kvm_irq_routing, kvm_irq_routing_entry, kvm_irq_routing_entry__bindgen_ty_1,
     kvm_irq_routing_msi, KVM_CAP_SPLIT_IRQCHIP, KVM_IRQ_ROUTING_MSI,
@@ -96,14 +97,13 @@ pub struct IoApic {
     version: u8,
     irq_eoi: [i32; IOAPIC_NUM_PINS],
     irq_routes: Vec<kvm_irq_routing_entry>,
-    irq_sender: crossbeam_channel::Sender<(WorkerMessage, EventFd)>,
-    event_fd: EventFd,
+    irq_sender: crossbeam_channel::Sender<WorkerMessage>,
 }
 
 impl IoApic {
     pub fn new(
         vm: &VmFd,
-        _irq_sender: crossbeam_channel::Sender<(WorkerMessage, EventFd)>,
+        _irq_sender: crossbeam_channel::Sender<WorkerMessage>,
     ) -> Result<Self, Error> {
         let mut cap = kvm_enable_cap {
             cap: KVM_CAP_SPLIT_IRQCHIP,
@@ -121,7 +121,6 @@ impl IoApic {
             irq_eoi: [0; IOAPIC_NUM_PINS],
             irq_routes: Vec::with_capacity(IOAPIC_NUM_PINS),
             irq_sender: _irq_sender,
-            event_fd: EventFd::new(libc::EFD_SEMAPHORE).unwrap(),
         };
 
         (0..IOAPIC_NUM_PINS).for_each(|i| ioapic.add_msi_route(i));
@@ -176,11 +175,7 @@ impl IoApic {
     }
 
     fn send_irq_worker_message(&self, msg: WorkerMessage) {
-        self.irq_sender
-            .send((msg, self.event_fd.try_clone().unwrap()))
-            .unwrap();
-
-        self.event_fd.read().unwrap();
+        self.irq_sender.send(msg).unwrap();
     }
 
     fn parse_entry(&self, entry: &RedirectionTableEntry) -> IoApicEntryInfo {
@@ -249,7 +244,12 @@ impl IoApic {
             }
         }
 
-        self.send_irq_worker_message(WorkerMessage::GsiRoute(self.irq_routes.clone()));
+        let (response_sender, response_receiver) = unbounded();
+        self.send_irq_worker_message(WorkerMessage::GsiRoute(
+            response_sender.clone(),
+            self.irq_routes.clone(),
+        ));
+        response_receiver.recv().unwrap();
     }
 
     fn service(&mut self) {
@@ -273,11 +273,27 @@ impl IoApic {
                         continue;
                     }
 
+                    let (response_sender, response_receiver) = unbounded();
                     if info.trig_mode as u64 == IOAPIC_TRIGGER_EDGE {
-                        self.send_irq_worker_message(WorkerMessage::IrqLine(i as u32, true));
-                        self.send_irq_worker_message(WorkerMessage::IrqLine(i as u32, false));
+                        self.send_irq_worker_message(WorkerMessage::IrqLine(
+                            response_sender.clone(),
+                            i as u32,
+                            true,
+                        ));
+                        let _ = response_receiver.recv().unwrap();
+                        self.send_irq_worker_message(WorkerMessage::IrqLine(
+                            response_sender.clone(),
+                            i as u32,
+                            false,
+                        ));
+                        let _ = response_receiver.recv().unwrap();
                     } else {
-                        self.send_irq_worker_message(WorkerMessage::IrqLine(i as u32, true));
+                        self.send_irq_worker_message(WorkerMessage::IrqLine(
+                            response_sender.clone(),
+                            i as u32,
+                            true,
+                        ));
+                        let _ = response_receiver.recv().unwrap();
                     }
                 }
             }

src/utils/src/worker_message.rs

@@ -1,9 +1,12 @@
 #[derive(Debug)]
 pub enum WorkerMessage {
     #[cfg(target_arch = "x86_64")]
-    GsiRoute(Vec<kvm_bindings::kvm_irq_routing_entry>),
+    GsiRoute(
+        crossbeam_channel::Sender<bool>,
+        Vec<kvm_bindings::kvm_irq_routing_entry>,
+    ),
     #[cfg(target_arch = "x86_64")]
-    IrqLine(u32, bool),
+    IrqLine(crossbeam_channel::Sender<bool>, u32, bool),
     #[cfg(target_os = "macos")]
     GpuAddMapping(crossbeam_channel::Sender<bool>, u64, u64, u64),
     #[cfg(target_os = "macos")]

src/vmm/src/builder.rs

@@ -511,8 +511,7 @@ pub fn build_microvm(
     event_manager: &mut EventManager,
     _shutdown_efd: Option<EventFd>,
     #[cfg(feature = "tee")] pm_sender: (Sender<MemoryProperties>, EventFd),
-    #[cfg(target_os = "macos")] _sender: Sender<WorkerMessage>,
-    #[cfg(target_arch = "x86_64")] _sender: Sender<(WorkerMessage, EventFd)>,
+    _sender: Sender<WorkerMessage>,
 ) -> std::result::Result<Arc<Mutex<Vmm>>, StartMicrovmError> {
     let payload = choose_payload(vm_resources)?;
 

src/vmm/src/lib.rs

@@ -384,7 +384,7 @@ impl Vmm {
     pub fn start_worker_thread(
         vmm: Arc<Mutex<Vmm>>,
         #[cfg(target_os = "macos")] receiver: Receiver<WorkerMessage>,
-        #[cfg(not(target_os = "macos"))] receiver: Receiver<(WorkerMessage, EventFd)>,
+        #[cfg(not(target_os = "macos"))] receiver: Receiver<WorkerMessage>,
     ) -> io::Result<()> {
         std::thread::Builder::new()
             .name("vmm worker".into())
@@ -394,26 +394,20 @@ impl Vmm {
                     #[cfg(target_os = "macos")]
                     Ok(message) => vmm.lock().unwrap().match_worker_message(message),
                     #[cfg(target_os = "linux")]
-                    Ok((message, evt_fd)) => {
-                        vmm.lock().unwrap().match_worker_message(message, evt_fd)
-                    }
+                    Ok(message) => vmm.lock().unwrap().match_worker_message(message),
                 }
             })?;
         Ok(())
     }
 
-    fn match_worker_message(
-        &self,
-        msg: WorkerMessage,
-        #[cfg(target_os = "linux")] evt_fd: EventFd,
-    ) {
+    fn match_worker_message(&self, msg: WorkerMessage) {
         match msg {
             #[cfg(target_os = "macos")]
             WorkerMessage::GpuAddMapping(s, h, g, l) => self.add_mapping(s, h, g, l),
             #[cfg(target_os = "macos")]
             WorkerMessage::GpuRemoveMapping(s, g, l) => self.remove_mapping(s, g, l),
             #[cfg(target_arch = "x86_64")]
-            WorkerMessage::GsiRoute(entries) => {
+            WorkerMessage::GsiRoute(sender, entries) => {
                 let mut irq_routing = utils::sized_vec::vec_with_array_field::<
                     kvm_bindings::kvm_irq_routing,
                     kvm_bindings::kvm_irq_routing_entry,
@@ -428,13 +422,12 @@ impl Vmm {
                 }
 
                 self.vm.fd().set_gsi_routing(&irq_routing[0]).unwrap();
-
-                evt_fd.write(1).unwrap();
+                sender.send(true).unwrap();
             }
             #[cfg(target_arch = "x86_64")]
-            WorkerMessage::IrqLine(irq, active) => {
+            WorkerMessage::IrqLine(sender, irq, active) => {
                 self.vm.fd().set_irq_line(irq, active).unwrap();
-                evt_fd.write(1).unwrap();
+                sender.send(true).unwrap();
             }
         }
     }