vmm: Deassign ioeventfds on PCI device hot-unplug

Closing the eventfds registered with KVM_IOEVENTFD is not sufficient,
KVM holds its own references. We need to explicitly assign them by
calling KVM_IOEVENTFD again with the KVM_IOEVENTFD_FLAG_DEASSIGN flag.
Do that in hot_unplug_device().

Add VirtioPciDevice::unregister_notification_ioevents() as the
counterpart to register_notification_ioevent() and move the common logic
into a private helper. Since we are here rename
register_notification_ioevent() to register_notification_ioevents()
since it's used to register multiple events.

Signed-off-by: Ilias Stamatis <ilstam@amazon.com>

Author: ilstam

src/vmm/src/device_manager/mod.rs

@@ -599,6 +599,10 @@ impl DeviceManager {
         let pci_device_arc = self.pci_devices.virtio_devices.remove(&device_id).unwrap();
         let pci_device = pci_device_arc.lock().expect("Poisoned lock");
 
+        pci_device
+            .unregister_notification_ioevents(&vm)
+            .map_err(PciManagerError::Kvm)?;
+
         vm.common
             .mmio_bus
             .remove(pci_device.config_bar_addr(), CAPABILITY_BAR_SIZE)

src/vmm/src/device_manager/pci_mngr.rs

@@ -130,7 +130,7 @@ impl PciDevices {
         Self::register_bars_with_bus(vm, &virtio_device)?;
 
         let mut device = virtio_device.lock().expect("Poisoned lock");
-        device.register_notification_ioevent(vm)?;
+        device.register_notification_ioevents(vm)?;
 
         let sub_id = event_manager.add_subscriber(device.virtio_device());
         device.sub_id = Some(sub_id);

src/vmm/src/devices/virtio/transport/pci/device.rs

@@ -620,8 +620,7 @@ impl VirtioPciDevice {
         !self.device_activated.load(Ordering::SeqCst) && self.is_driver_ready()
     }
 
-    /// Register the IoEvent notification for a VirtIO device
-    pub fn register_notification_ioevent(&self, vm: &KvmVm) -> Result<(), errno::Error> {
+    fn set_notification_ioevents(&self, vm: &KvmVm, assign: bool) -> Result<(), errno::Error> {
         let bar_addr = self.config_bar_addr();
         for (i, queue_evt) in self
             .device
@@ -634,11 +633,26 @@ impl VirtioPciDevice {
             let notify_base = bar_addr + u64::from(NOTIFICATION_BAR_OFFSET);
             let io_addr =
                 IoEventAddress::Mmio(notify_base + i as u64 * u64::from(NOTIFY_OFF_MULTIPLIER));
-            vm.fd().register_ioevent(queue_evt, &io_addr, NoDatamatch)?;
+            if assign {
+                vm.fd().register_ioevent(queue_evt, &io_addr, NoDatamatch)?;
+            } else {
+                vm.fd()
+                    .unregister_ioevent(queue_evt, &io_addr, NoDatamatch)?;
+            }
         }
         Ok(())
     }
 
+    /// Register the IoEvent notifications for a VirtIO device.
+    pub fn register_notification_ioevents(&self, vm: &KvmVm) -> Result<(), errno::Error> {
+        self.set_notification_ioevents(vm, true)
+    }
+
+    /// Unregister the IoEvent notifications for a VirtIO device.
+    pub fn unregister_notification_ioevents(&self, vm: &KvmVm) -> Result<(), errno::Error> {
+        self.set_notification_ioevents(vm, false)
+    }
+
     pub fn state(&self) -> VirtioPciDeviceState {
         VirtioPciDeviceState {
             sbdf: self.sbdf,