devices: implement support for in-kernel HVF GICv3

macOS 15 extended Hypervisor.framework with an in-kernel HVF GICv3
device. Using it reduces the cost of GIC operations and enables us to
use nested virt.

Add support for it and enable it automatically when the functions are
found in HVF.

Signed-off-by: Sergio Lopez <[email protected]>

Author: slp

Cargo.lock

@@ -19,6 +19,7 @@ bitflags = "1.2.0"
 crossbeam-channel = "0.5"
 env_logger = "0.9.0"
 libc = ">=0.2.39"
+libloading = "0.8"
 log = "0.4.0"
 nix = { version = "0.24.1", features = ["poll"] }
 pw = { package = "pipewire", version = "0.8.0", optional = true }

src/devices/Cargo.toml

@@ -0,0 +1,184 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::io;
+use std::sync::LazyLock;
+
+use crate::bus::BusDevice;
+use crate::legacy::gic::GICDevice;
+use crate::legacy::irqchip::IrqChipT;
+use crate::Error as DeviceError;
+
+use hvf::bindings::{hv_gic_config_t, hv_ipa_t, hv_return_t, HV_SUCCESS};
+use hvf::Error;
+use utils::eventfd::EventFd;
+
+// Device trees specific constants
+const ARCH_GIC_V3_MAINT_IRQ: u32 = 9;
+
+pub struct HvfGicBindings {
+    hv_gic_create:
+        libloading::Symbol<'static, unsafe extern "C" fn(hv_gic_config_t) -> hv_return_t>,
+    hv_gic_config_create: libloading::Symbol<'static, unsafe extern "C" fn() -> hv_gic_config_t>,
+    hv_gic_config_set_distributor_base:
+        libloading::Symbol<'static, unsafe extern "C" fn(hv_gic_config_t, hv_ipa_t) -> hv_return_t>,
+    hv_gic_config_set_redistributor_base:
+        libloading::Symbol<'static, unsafe extern "C" fn(hv_gic_config_t, hv_ipa_t) -> hv_return_t>,
+    hv_gic_get_distributor_size:
+        libloading::Symbol<'static, unsafe extern "C" fn(*mut usize) -> hv_return_t>,
+    hv_gic_get_redistributor_size:
+        libloading::Symbol<'static, unsafe extern "C" fn(*mut usize) -> hv_return_t>,
+    hv_gic_set_spi: libloading::Symbol<'static, unsafe extern "C" fn(u32, bool) -> hv_return_t>,
+}
+
+pub struct HvfGicV3 {
+    bindings: HvfGicBindings,
+
+    /// GIC device properties, to be used for setting up the fdt entry
+    properties: [u64; 4],
+
+    /// Number of CPUs handled by the device
+    vcpu_count: u64,
+}
+
+static HVF: LazyLock<libloading::Library> = LazyLock::new(|| unsafe {
+    libloading::Library::new(
+        "/System/Library/Frameworks/Hypervisor.framework/Versions/A/Hypervisor",
+    )
+    .unwrap()
+});
+
+impl HvfGicV3 {
+    pub fn new(vcpu_count: u64) -> Result<Self, Error> {
+        let bindings = unsafe {
+            HvfGicBindings {
+                hv_gic_create: HVF.get(b"hv_gic_create").map_err(Error::FindSymbol)?,
+                hv_gic_config_create: HVF
+                    .get(b"hv_gic_config_create")
+                    .map_err(Error::FindSymbol)?,
+                hv_gic_config_set_distributor_base: HVF
+                    .get(b"hv_gic_config_set_distributor_base")
+                    .map_err(Error::FindSymbol)?,
+                hv_gic_config_set_redistributor_base: HVF
+                    .get(b"hv_gic_config_set_redistributor_base")
+                    .map_err(Error::FindSymbol)?,
+                hv_gic_get_distributor_size: HVF
+                    .get(b"hv_gic_get_distributor_size")
+                    .map_err(Error::FindSymbol)?,
+                hv_gic_get_redistributor_size: HVF
+                    .get(b"hv_gic_get_redistributor_size")
+                    .map_err(Error::FindSymbol)?,
+                hv_gic_set_spi: HVF.get(b"hv_gic_set_spi").map_err(Error::FindSymbol)?,
+            }
+        };
+
+        let mut dist_size: usize = 0;
+        let ret = unsafe { (bindings.hv_gic_get_distributor_size)(&mut dist_size) };
+        if ret != HV_SUCCESS {
+            return Err(Error::VmCreate);
+        }
+        let dist_size = dist_size as u64;
+
+        let mut redist_size: usize = 0;
+        let ret = unsafe { (bindings.hv_gic_get_redistributor_size)(&mut redist_size) };
+        if ret != HV_SUCCESS {
+            return Err(Error::VmCreate);
+        }
+
+        let redists_size = redist_size as u64 * vcpu_count;
+        let dist_addr = arch::MMIO_MEM_START - dist_size - redists_size;
+        let redists_addr = arch::MMIO_MEM_START - redists_size;
+
+        let gic_config = unsafe { (bindings.hv_gic_config_create)() };
+        let ret = unsafe { (bindings.hv_gic_config_set_distributor_base)(gic_config, dist_addr) };
+        if ret != HV_SUCCESS {
+            return Err(Error::VmCreate);
+        }
+
+        let ret = unsafe {
+            (bindings.hv_gic_config_set_redistributor_base)(
+                gic_config,
+                arch::MMIO_MEM_START - redists_size,
+            )
+        };
+        if ret != HV_SUCCESS {
+            return Err(Error::VmCreate);
+        }
+
+        let ret = unsafe { (bindings.hv_gic_create)(gic_config) };
+        if ret != HV_SUCCESS {
+            return Err(Error::VmCreate);
+        }
+
+        Ok(Self {
+            bindings,
+            properties: [dist_addr, dist_size, redists_addr, redists_size],
+            vcpu_count,
+        })
+    }
+}
+
+impl IrqChipT for HvfGicV3 {
+    fn get_mmio_addr(&self) -> u64 {
+        0
+    }
+
+    fn get_mmio_size(&self) -> u64 {
+        0
+    }
+
+    fn set_irq(
+        &self,
+        irq_line: Option<u32>,
+        _interrupt_evt: Option<&EventFd>,
+    ) -> Result<(), DeviceError> {
+        if let Some(irq_line) = irq_line {
+            let ret = unsafe { (self.bindings.hv_gic_set_spi)(irq_line, true) };
+            if ret != HV_SUCCESS {
+                Err(DeviceError::FailedSignalingUsedQueue(io::Error::new(
+                    io::ErrorKind::Other,
+                    "HVF returned error when setting SPI",
+                )))
+            } else {
+                Ok(())
+            }
+        } else {
+            Err(DeviceError::FailedSignalingUsedQueue(io::Error::new(
+                io::ErrorKind::InvalidData,
+                "IRQ not line configured",
+            )))
+        }
+    }
+}
+
+impl BusDevice for HvfGicV3 {
+    fn read(&mut self, _vcpuid: u64, _offset: u64, _data: &mut [u8]) {
+        unreachable!("MMIO operations are managed in-kernel");
+    }
+
+    fn write(&mut self, _vcpuid: u64, _offset: u64, _data: &[u8]) {
+        unreachable!("MMIO operations are managed in-kernel");
+    }
+}
+
+impl GICDevice for HvfGicV3 {
+    fn device_properties(&self) -> Vec<u64> {
+        self.properties.to_vec()
+    }
+
+    fn vcpu_count(&self) -> u64 {
+        self.vcpu_count
+    }
+
+    fn fdt_compatibility(&self) -> String {
+        "arm,gic-v3".to_string()
+    }
+
+    fn fdt_maint_irq(&self) -> u32 {
+        ARCH_GIC_V3_MAINT_IRQ
+    }
+
+    fn version(&self) -> u32 {
+        7
+    }
+}

src/devices/src/legacy/hvfgicv3.rs

@@ -8,6 +8,8 @@
 pub mod gic;
 #[cfg(target_os = "macos")]
 mod gicv3;
+#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
+mod hvfgicv3;
 mod i8042;
 mod irqchip;
 #[cfg(all(target_os = "linux", target_arch = "aarch64"))]
@@ -33,6 +35,8 @@ use aarch64::serial;
 pub use self::gicv3::GicV3;
 #[cfg(target_arch = "aarch64")]
 pub use self::gpio::Gpio;
+#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
+pub use self::hvfgicv3::HvfGicV3;
 pub use self::i8042::Error as I8042DeviceError;
 pub use self::i8042::I8042Device;
 pub use self::irqchip::{IrqChip, IrqChipDevice, IrqChipT};

src/devices/src/legacy/mod.rs

@@ -6,6 +6,7 @@ edition = "2021"
 
 [dependencies]
 crossbeam-channel = "0.5"
+libloading = "0.8"
 log = "0.4.0"
 env_logger = "0.9.0"
 

src/hvf/Cargo.toml

@@ -7,7 +7,7 @@
 #[allow(non_snake_case)]
 #[allow(non_upper_case_globals)]
 #[allow(deref_nullptr)]
-mod bindings;
+pub mod bindings;
 
 use bindings::*;
 
@@ -51,8 +51,9 @@ const EC_SYSTEMREGISTERTRAP: u64 = 0x18;
 const EC_DATAABORT: u64 = 0x24;
 const EC_AA64_BKPT: u64 = 0x3c;
 
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub enum Error {
+    FindSymbol(libloading::Error),
     MemoryMap,
     MemoryUnmap,
     VcpuCreate,
@@ -73,6 +74,7 @@ impl Display for Error {
         use self::Error::*;
 
         match self {
+            FindSymbol(ref err) => write!(f, "Couldn't find symbol in HVF library: {}", err),
             MemoryMap => write!(f, "Error registering memory region in HVF"),
             MemoryUnmap => write!(f, "Error unregistering memory region in HVF"),
             VcpuCreate => write!(f, "Error creating HVF vCPU instance"),

src/hvf/src/lib.rs

@@ -6,6 +6,7 @@
 #[cfg(target_os = "macos")]
 use crossbeam_channel::{unbounded, Sender};
 use kernel::cmdline::Cmdline;
+#[cfg(target_os = "macos")]
 use std::collections::HashMap;
 use std::fmt::{Display, Formatter};
 use std::fs::File;
@@ -22,15 +23,15 @@ use crate::device_manager::legacy::PortIODeviceManager;
 use crate::device_manager::mmio::MMIODeviceManager;
 use crate::resources::VmResources;
 use crate::vmm_config::external_kernel::{ExternalKernel, KernelFormat};
-#[cfg(target_os = "macos")]
-use devices::legacy::GicV3;
 #[cfg(all(target_os = "linux", target_arch = "aarch64"))]
 use devices::legacy::KvmGicV3;
 #[cfg(target_arch = "x86_64")]
 use devices::legacy::KvmIoapic;
 use devices::legacy::Serial;
 #[cfg(target_os = "macos")]
 use devices::legacy::VcpuList;
+#[cfg(target_os = "macos")]
+use devices::legacy::{GicV3, HvfGicV3};
 use devices::legacy::{IrqChip, IrqChipDevice};
 #[cfg(feature = "net")]
 use devices::virtio::Net;
@@ -89,6 +90,9 @@ static EDK2_BINARY: &[u8] = include_bytes!("../../../edk2/KRUN_EFI.silent.fd");
 pub enum StartMicrovmError {
     /// Unable to attach block device to Vmm.
     AttachBlockDevice(io::Error),
+    #[cfg(target_os = "macos")]
+    /// Failed to create HVF in-kernel IrqChip.
+    CreateHvfIrqChip(hvf::Error),
     #[cfg(target_os = "linux")]
     /// Failed to create KVM in-kernel IrqChip.
     CreateKvmIrqChip(kvm_ioctls::Error),
@@ -208,6 +212,10 @@ impl Display for StartMicrovmError {
             AttachBlockDevice(ref err) => {
                 write!(f, "Unable to attach block device to Vmm. Error: {err}")
             }
+            #[cfg(target_os = "macos")]
+            CreateHvfIrqChip(ref err) => {
+                write!(f, "Cannot create HVF in-kernel IrqChip: {err}")
+            }
             #[cfg(target_os = "linux")]
             CreateKvmIrqChip(ref err) => {
                 write!(f, "Cannot create KVM in-kernel IrqChip: {err}")
@@ -698,9 +706,15 @@ pub fn build_microvm(
 
     #[cfg(all(target_arch = "aarch64", target_os = "macos"))]
     {
-        intc = Arc::new(Mutex::new(IrqChipDevice::new(Box::new(GicV3::new(
-            vcpu_list.clone(),
-        )))));
+        intc = {
+            // If the system supports the in-kernel GIC, use it. Otherwise, fall back to the
+            // userspace implementation.
+            let gic = match HvfGicV3::new(vm_resources.vm_config().vcpu_count.unwrap() as u64) {
+                Ok(hvfgic) => IrqChipDevice::new(Box::new(hvfgic)),
+                Err(_) => IrqChipDevice::new(Box::new(GicV3::new(vcpu_list.clone()))),
+            };
+            Arc::new(Mutex::new(gic))
+        };
 
         vcpus = create_vcpus_aarch64(
             &vm,

src/vmm/src/builder.rs

@@ -265,9 +265,13 @@ impl MMIODeviceManager {
             (intc.get_mmio_addr(), intc.get_mmio_size())
         };
 
-        self.bus
-            .insert(intc, mmio_addr, mmio_size)
-            .map_err(Error::BusError)?;
+        // The in-kernel GIC reports a size of 0 to tell us we don't need to map
+        // anything in the guest.
+        if mmio_size != 0 {
+            self.bus
+                .insert(intc, mmio_addr, mmio_size)
+                .map_err(Error::BusError)?;
+        }
 
         Ok(())
     }

src/vmm/src/device_manager/hvf/mmio.rs

@@ -104,6 +104,10 @@ impl Vm {
         Ok(Vm { hvf_vm })
     }
 
+    pub fn hvf_vm(&self) -> &HvfVm {
+        &self.hvf_vm
+    }
+
     /// Initializes the guest memory.
     pub fn memory_init(&mut self, guest_mem: &GuestMemoryMmap) -> Result<()> {
         for region in guest_mem.iter() {