vmm: if IRQCHIP is split, create IoApic device

If the user calls `krun_split_irqchip` with `enabled` set to `true`,
then create an `IoApic` device in userspace rather than creating an
IOAPIC in the guest with KVM.

Signed-off-by: Jake Correnti <[email protected]>

Author: jakecorrenti

src/vmm/src/builder.rs

@@ -32,6 +32,8 @@ use devices::legacy::Serial;
 use devices::legacy::VcpuList;
 #[cfg(target_os = "macos")]
 use devices::legacy::{GicV3, HvfGicV3};
+#[cfg(target_arch = "x86_64")]
+use devices::legacy::{IoApic, IrqChipT};
 use devices::legacy::{IrqChip, IrqChipDevice};
 #[cfg(feature = "net")]
 use devices::virtio::Net;
@@ -664,10 +666,20 @@ pub fn build_microvm(
     // while on aarch64 we need to do it the other way around.
     #[cfg(target_arch = "x86_64")]
     {
-        let kvmioapic = KvmIoapic::new(vm.fd()).map_err(StartMicrovmError::CreateKvmIrqChip)?;
-        intc = Arc::new(Mutex::new(IrqChipDevice::new(Box::new(kvmioapic))));
+        let ioapic: Box<dyn IrqChipT> = if vm_resources.split_irqchip {
+            Box::new(IoApic::new(vm.fd(), irq_sender).map_err(StartMicrovmError::CreateKvmIrqChip)?)
+        } else {
+            Box::new(KvmIoapic::new(vm.fd()).map_err(StartMicrovmError::CreateKvmIrqChip)?)
+        };
+        intc = Arc::new(Mutex::new(IrqChipDevice::new(ioapic)));
 
-        attach_legacy_devices(&vm, &mut pio_device_manager)?;
+        attach_legacy_devices(
+            &vm,
+            vm_resources.split_irqchip,
+            &mut pio_device_manager,
+            &mut mmio_device_manager,
+            Some(intc.clone()),
+        )?;
 
         vcpus = create_vcpus_x86_64(
             &vm,
@@ -1333,13 +1345,23 @@ pub fn setup_serial_device(
 #[cfg(target_arch = "x86_64")]
 fn attach_legacy_devices(
     vm: &Vm,
+    split_irqchip: bool,
     pio_device_manager: &mut PortIODeviceManager,
+    mmio_device_manager: &mut MMIODeviceManager,
+    intc: Option<Arc<Mutex<IrqChipDevice>>>,
 ) -> std::result::Result<(), StartMicrovmError> {
     pio_device_manager
         .register_devices()
         .map_err(Error::LegacyIOBus)
         .map_err(StartMicrovmError::Internal)?;
 
+    if split_irqchip {
+        mmio_device_manager
+            .register_mmio_ioapic(intc)
+            .map_err(Error::RegisterMMIODevice)
+            .map_err(StartMicrovmError::Internal)?;
+    }
+
     macro_rules! register_irqfd_evt {
         ($evt: ident, $index: expr) => {{
             vm.fd()

src/vmm/src/device_manager/kvm/mmio.rs

@@ -88,6 +88,23 @@ impl MMIODeviceManager {
         }
     }
 
+    /// Register a MMIO IOAPIC device.
+    #[cfg(target_arch = "x86_64")]
+    pub fn register_mmio_ioapic(
+        &mut self,
+        intc: Option<Arc<Mutex<devices::legacy::IrqChipDevice>>>,
+    ) -> Result<()> {
+        if let Some(intc) = intc {
+            let (addr, size) = {
+                let intc = intc.lock().unwrap();
+                (intc.get_mmio_addr(), intc.get_mmio_size())
+            };
+            self.bus.insert(intc, addr, size).map_err(Error::BusError)?;
+        }
+
+        Ok(())
+    }
+
     /// Register an already created MMIO device to be used via MMIO transport.
     pub fn register_mmio_device(
         &mut self,
@@ -117,6 +134,8 @@ impl MMIODeviceManager {
         vm.register_irqfd(mmio_device.locked_device().interrupt_evt(), self.irq)
             .map_err(Error::RegisterIrqFd)?;
 
+        mmio_device.locked_device().set_irq_line(self.irq);
+
         self.bus
             .insert(Arc::new(Mutex::new(mmio_device)), self.mmio_base, MMIO_LEN)
             .map_err(Error::BusError)?;