diff --git a/alioth-cli/src/vu.rs b/alioth-cli/src/vu.rs index e4ac317c..6c1aee0f 100644 --- a/alioth-cli/src/vu.rs +++ b/alioth-cli/src/vu.rs @@ -24,7 +24,7 @@ use alioth::virtio::dev::fs::shared_dir::SharedDirSpec; use alioth::virtio::dev::net::tap::TapNetSpec; use alioth::virtio::dev::vsock::UdsVsockSpec; use alioth::virtio::dev::{DevSpec, Virtio, VirtioDevice}; -use alioth::virtio::vu::backend::{VuBackend, VuEventfd, VuIrqSender}; +use alioth::virtio::vu::backend::{VuBackend, VuIrqSender}; use clap::{Args, Subcommand}; use serde::Deserialize; use serde_aco::{Help, help_text}; @@ -108,7 +108,7 @@ fn create_dev( name: String, args: &DevArgs

, memory: Arc, -) -> Result, Error> +) -> Result, Error> where D: Virtio, P: DevSpec + Help + for<'a> Deserialize<'a> + Send + Sync + 'static, diff --git a/alioth/src/hv/hv.rs b/alioth/src/hv/hv.rs index dc7e6ecf..037d3d6a 100644 --- a/alioth/src/hv/hv.rs +++ b/alioth/src/hv/hv.rs @@ -44,6 +44,7 @@ use crate::arch::sev::{SevPolicy, SevStatus, SnpPageType, SnpPolicy}; #[cfg(target_arch = "x86_64")] use crate::arch::tdx::TdAttr; use crate::errors::{DebugTrace, trace_error}; +use crate::sync::notifier::Notifier; #[cfg(target_os = "macos")] pub use self::hvf::Hvf; @@ -90,8 +91,8 @@ pub enum Error { MemEncrypt { error: std::io::Error }, #[snafu(display("Failed to configure an IrqFd"))] IrqFd { error: std::io::Error }, - #[snafu(display("Failed to configure an IoeventFd"))] - IoeventFd { error: std::io::Error }, + #[snafu(display("Failed to configure a Notifier"))] + Notifier { error: std::io::Error }, #[snafu(display("Failed to create an IrqSender for pin {pin}"))] CreateIrq { pin: u8, error: std::io::Error }, #[snafu(display("Failed to send an interrupt"))] @@ -243,13 +244,9 @@ pub trait MsiSender: Debug + Send + Sync + 'static { fn create_irqfd(&self) -> Result; } -pub trait IoeventFd: Debug + Send + Sync + AsFd + 'static {} - -pub trait IoeventFdRegistry: Debug + Send + Sync + 'static { - type IoeventFd: IoeventFd; - fn create(&self) -> Result; - fn register(&self, fd: &Self::IoeventFd, gpa: u64, len: u8, data: Option) -> Result<()>; - fn deregister(&self, fd: &Self::IoeventFd) -> Result<()>; +pub trait NotifierRegistry: Debug + Send + Sync + 'static { + fn register(&self, notifier: &Notifier, gpa: u64, len: u8, data: Option) -> Result<()>; + fn deregister(&self, notifier: &Notifier) -> Result<()>; } pub trait IrqFd: Debug + Send + Sync + AsFd + 'static { @@ -326,14 +323,14 @@ pub trait Vm: Debug + Send + Sync + 'static { type Vcpu: Vcpu; type IrqSender: IrqSender + Send + Sync; type MsiSender: MsiSender; - type IoeventFdRegistry: IoeventFdRegistry; + type NotifierRegistry: NotifierRegistry; fn create_vcpu(&self, index: u16, identity: u64) -> Result; fn create_irq_sender(&self, pin: u8) -> Result; fn create_msi_sender( &self, #[cfg(target_arch = "aarch64")] devid: u32, ) -> Result; - fn create_ioeventfd_registry(&self) -> Result; + fn create_notifier_registry(&self) -> Result; fn stop_vcpu(&self, identity: u64, handle: &JoinHandle) -> Result<(), Error>; fn map(&self, gpa: u64, size: u64, hva: usize, option: MemMapOption) -> Result<(), Error>; diff --git a/alioth/src/hv/hv_test.rs b/alioth/src/hv/hv_test.rs index 9f16e825..bdb68835 100644 --- a/alioth/src/hv/hv_test.rs +++ b/alioth/src/hv/hv_test.rs @@ -19,7 +19,8 @@ use std::sync::Arc; use parking_lot::{Condvar, Mutex, RwLock}; use snafu::ResultExt; -use crate::hv::{IoeventFd, IrqFd, IrqSender, MsiSender, Result, error}; +use crate::hv::{IrqFd, IrqSender, MsiSender, Result, error}; +use crate::sync::notifier::Notifier; #[derive(Debug)] struct TestIrqFdInner { @@ -141,17 +142,6 @@ impl MsiSender for TestMsiSender { } } -#[derive(Debug, Default)] -pub struct TestIoeventFd; - -impl AsFd for TestIoeventFd { - fn as_fd(&self) -> BorrowedFd<'_> { - unsafe { BorrowedFd::borrow_raw(0) } - } -} - -impl IoeventFd for TestIoeventFd {} - #[derive(Debug, Default, PartialEq, Eq)] pub struct RegisteredAddr { pub gpa: u64, @@ -160,30 +150,20 @@ pub struct RegisteredAddr { } #[derive(Debug, Default)] -pub struct TestIoeventFdRegistry { +pub struct TestNotifierRegistry { pub registered: Arc>>, pub deregistered: Arc>, - pub fail_mode: Option, } -impl super::IoeventFdRegistry for TestIoeventFdRegistry { - type IoeventFd = TestIoeventFd; - - fn create(&self) -> Result { - if let Some(kind) = self.fail_mode { - return Err(io::Error::from(kind)).context(error::IoeventFd); - } - Ok(TestIoeventFd) - } - - fn register(&self, _fd: &Self::IoeventFd, gpa: u64, len: u8, data: Option) -> Result<()> { +impl super::NotifierRegistry for TestNotifierRegistry { + fn register(&self, _notifier: &Notifier, gpa: u64, len: u8, data: Option) -> Result<()> { self.registered .lock() .push(RegisteredAddr { gpa, len, data }); Ok(()) } - fn deregister(&self, _fd: &Self::IoeventFd) -> Result<()> { + fn deregister(&self, _notifier: &Notifier) -> Result<()> { *self.deregistered.lock() += 1; Ok(()) } diff --git a/alioth/src/hv/hvf/vm.rs b/alioth/src/hv/hvf/vm.rs index 93d9e1af..2f233671 100644 --- a/alioth/src/hv/hvf/vm.rs +++ b/alioth/src/hv/hvf/vm.rs @@ -27,9 +27,10 @@ use crate::arch::reg::MpidrEl1; use crate::hv::hvf::vcpu::{HvfVcpu, VcpuHandle}; use crate::hv::hvf::{OsObject, check_ret}; use crate::hv::{ - GicV2, GicV2m, GicV3, IoeventFd, IoeventFdRegistry, IrqFd, IrqSender, Its, MemMapOption, - MsiSender, Result, Vm, error, + GicV2, GicV2m, GicV3, IrqFd, IrqSender, Its, MemMapOption, MsiSender, NotifierRegistry, Result, + Vm, error, }; +use crate::sync::notifier::Notifier; use crate::sys::hvf::{ HvMemoryFlag, hv_gic_config_create, hv_gic_config_set_distributor_base, hv_gic_config_set_msi_interrupt_range, hv_gic_config_set_msi_region_base, @@ -107,39 +108,25 @@ impl MsiSender for HvfMsiSender { } } +/// Hypervisor.framework has no `KVM_IOEVENTFD` equivalent, so this registry is +/// deliberately uninhabited: [`HvfVm::create_notifier_registry()`] always fails +/// and no value of this type can ever be constructed. #[derive(Debug)] -pub struct HvfIoeventFd {} +pub enum HvfNotifierRegistry {} -impl IoeventFd for HvfIoeventFd {} - -impl AsFd for HvfIoeventFd { - fn as_fd(&self) -> BorrowedFd<'_> { - unreachable!() - } -} - -#[derive(Debug)] -pub struct HvfIoeventFdRegistry; - -impl IoeventFdRegistry for HvfIoeventFdRegistry { - type IoeventFd = HvfIoeventFd; - - fn create(&self) -> Result { - Err(ErrorKind::Unsupported.into()).context(error::IoeventFd) - } - - fn deregister(&self, _fd: &Self::IoeventFd) -> Result<()> { - unreachable!() +impl NotifierRegistry for HvfNotifierRegistry { + fn deregister(&self, _notifier: &Notifier) -> Result<()> { + match *self {} } fn register( &self, - _fd: &Self::IoeventFd, + _notifier: &Notifier, _gpa: u64, _len: u8, _data: Option, ) -> Result<()> { - unreachable!() + match *self {} } } @@ -239,14 +226,15 @@ impl Vm for HvfVm { type GicV2 = HvfGicV2; type GicV2m = HvfGicV2m; type GicV3 = HvfGicV3; - type IoeventFdRegistry = HvfIoeventFdRegistry; type IrqSender = HvfIrqSender; type Its = HvfIts; type MsiSender = HvfMsiSender; + type NotifierRegistry = HvfNotifierRegistry; type Vcpu = HvfVcpu; - fn create_ioeventfd_registry(&self) -> Result { - Ok(HvfIoeventFdRegistry) + fn create_notifier_registry(&self) -> Result { + // Hypervisor.framework has no KVM_IOEVENTFD equivalent. + Err(ErrorKind::Unsupported.into()).context(error::Notifier) } fn create_msi_sender(&self, _devid: u32) -> Result { diff --git a/alioth/src/hv/kvm/vm/vm.rs b/alioth/src/hv/kvm/vm/vm.rs index 89c60c1c..f7e3099f 100644 --- a/alioth/src/hv/kvm/vm/vm.rs +++ b/alioth/src/hv/kvm/vm/vm.rs @@ -45,9 +45,10 @@ use crate::ffi; use crate::hv::kvm::vcpu::KvmVcpu; use crate::hv::kvm::{KvmError, check_extension, kvm_error}; use crate::hv::{ - Error, IoeventFd, IoeventFdRegistry, IrqFd, IrqSender, Kvm, MemMapOption, MsiSender, Result, - Vm, VmSpec, error, + Error, IrqFd, IrqSender, Kvm, MemMapOption, MsiSender, NotifierRegistry, Result, Vm, VmSpec, + error, }; +use crate::sync::notifier::Notifier; #[cfg(target_arch = "x86_64")] use crate::sys::kvm::KVM_IRQCHIP_IOAPIC; #[cfg(target_arch = "aarch64")] @@ -405,57 +406,34 @@ impl MsiSender for KvmMsiSender { } } -#[derive(Debug)] -pub struct KvmIoeventFd { - fd: OwnedFd, -} - -impl AsFd for KvmIoeventFd { - fn as_fd(&self) -> BorrowedFd<'_> { - self.fd.as_fd() - } -} - -impl IoeventFd for KvmIoeventFd {} - #[derive(Debug)] pub struct KvmIoeventFdRegistry { vm: Arc, } -impl IoeventFdRegistry for KvmIoeventFdRegistry { - type IoeventFd = KvmIoeventFd; - - fn create(&self) -> Result { - let fd = - ffi!(unsafe { eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK) }).context(error::IoeventFd)?; - Ok(KvmIoeventFd { - fd: unsafe { OwnedFd::from_raw_fd(fd) }, - }) - } - - fn register(&self, fd: &Self::IoeventFd, gpa: u64, len: u8, data: Option) -> Result<()> { +impl NotifierRegistry for KvmIoeventFdRegistry { + fn register(&self, notifier: &Notifier, gpa: u64, len: u8, data: Option) -> Result<()> { let mut request = KvmIoEventFd { addr: gpa, len: len as u32, - fd: fd.as_fd().as_raw_fd(), + fd: notifier.as_fd().as_raw_fd(), ..Default::default() }; if let Some(data) = data { request.datamatch = data; request.flags |= KvmIoEventFdFlag::DATA_MATCH; } - unsafe { kvm_ioeventfd(&self.vm.fd, &request) }.context(error::IoeventFd)?; + unsafe { kvm_ioeventfd(&self.vm.fd, &request) }.context(error::Notifier)?; let mut fds = self.vm.ioeventfds.lock(); fds.insert(request.fd, request); Ok(()) } - fn deregister(&self, fd: &Self::IoeventFd) -> Result<()> { + fn deregister(&self, notifier: &Notifier) -> Result<()> { let mut fds = self.vm.ioeventfds.lock(); - if let Some(mut request) = fds.remove(&fd.as_fd().as_raw_fd()) { + if let Some(mut request) = fds.remove(¬ifier.as_fd().as_raw_fd()) { request.flags |= KvmIoEventFdFlag::DEASSIGN; - unsafe { kvm_ioeventfd(&self.vm.fd, &request) }.context(error::IoeventFd)?; + unsafe { kvm_ioeventfd(&self.vm.fd, &request) }.context(error::Notifier)?; } Ok(()) } @@ -507,11 +485,11 @@ impl Vm for KvmVm { type GicV2m = aarch64::KvmGicV2m; #[cfg(target_arch = "aarch64")] type GicV3 = aarch64::KvmGicV3; - type IoeventFdRegistry = KvmIoeventFdRegistry; type IrqSender = KvmIrqSender; #[cfg(target_arch = "aarch64")] type Its = aarch64::KvmIts; type MsiSender = KvmMsiSender; + type NotifierRegistry = KvmIoeventFdRegistry; type Vcpu = KvmVcpu; fn create_vcpu(&self, index: u16, identity: u64) -> Result { @@ -558,7 +536,7 @@ impl Vm for KvmVm { }) } - fn create_ioeventfd_registry(&self) -> Result { + fn create_notifier_registry(&self) -> Result { Ok(KvmIoeventFdRegistry { vm: self.vm.clone(), }) diff --git a/alioth/src/sync/notifier/notifier_linux.rs b/alioth/src/sync/notifier/notifier_linux.rs index 95615beb..d00c17c2 100644 --- a/alioth/src/sync/notifier/notifier_linux.rs +++ b/alioth/src/sync/notifier/notifier_linux.rs @@ -14,7 +14,7 @@ use std::fs::File; use std::io::{ErrorKind, Read, Result, Write}; -use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd}; +use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, OwnedFd}; use libc::{EFD_CLOEXEC, EFD_NONBLOCK, eventfd}; use mio::event::Source; @@ -51,6 +51,22 @@ impl Notifier { } } +/// Wraps an eventfd created elsewhere, e.g. a vhost-user kick fd sent by a +/// frontend. +/// +/// The fd is forced into non-blocking mode, since [`Notifier::notify()`] +/// relies on `EAGAIN` to detect a saturated eventfd counter. An external +/// sender is under no obligation to have set `EFD_NONBLOCK` itself. +impl TryFrom for Notifier { + type Error = std::io::Error; + + fn try_from(fd: OwnedFd) -> Result { + let flags = ffi!(unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) })?; + ffi!(unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_SETFL, flags | libc::O_NONBLOCK) })?; + Ok(Notifier { fd: File::from(fd) }) + } +} + impl Source for Notifier { fn register(&mut self, registry: &Registry, token: Token, interests: Interest) -> Result<()> { registry.register(&mut SourceFd(&self.fd.as_raw_fd()), token, interests) diff --git a/alioth/src/virtio/dev/balloon.rs b/alioth/src/virtio/dev/balloon.rs index e68439c6..c5147520 100644 --- a/alioth/src/virtio/dev/balloon.rs +++ b/alioth/src/virtio/dev/balloon.rs @@ -27,7 +27,6 @@ use serde::Deserialize; use serde_aco::Help; use zerocopy::{FromBytes, Immutable, IntoBytes}; -use crate::hv::IoeventFd; use crate::mem::emulated::{Action, Mmio}; use crate::mem::mapped::{Ram, RamBus}; use crate::sync::notifier::Notifier; @@ -197,15 +196,14 @@ impl Virtio for Balloon { &self.name } - fn spawn_worker( + fn spawn_worker( self, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where S: IrqSender, - E: IoeventFd, { Mio::spawn_worker(self, event_rx, memory, queue_regs) } @@ -224,15 +222,14 @@ impl Virtio for Balloon { } impl VirtioMio for Balloon { - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, feature: u128, - _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let feature = BalloonFeature::from_bits_retain(feature); self.queues[0] = BalloonQueue::Inflate; @@ -252,15 +249,14 @@ impl VirtioMio for Balloon { Ok(()) } - fn handle_queue<'m, Q, S, E>( + fn handle_queue<'m, Q, S>( &mut self, index: u16, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let Some(Some(queue)) = active_mio.queues.get_mut(index as usize) else { log::error!("{}: invalid queue index {index}", self.name); @@ -295,15 +291,14 @@ impl VirtioMio for Balloon { }) } - fn handle_event<'a, 'm, Q, S, E>( + fn handle_event<'a, 'm, Q, S>( &mut self, _event: &Event, - _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { Ok(()) } diff --git a/alioth/src/virtio/dev/blk.rs b/alioth/src/virtio/dev/blk.rs index 42971b95..34076268 100644 --- a/alioth/src/virtio/dev/blk.rs +++ b/alioth/src/virtio/dev/blk.rs @@ -35,7 +35,6 @@ use serde_aco::Help; use snafu::ResultExt; use zerocopy::{FromBytes, FromZeros, Immutable, IntoBytes}; -use crate::hv::IoeventFd; use crate::mem::mapped::RamBus; use crate::sync::notifier::Notifier; use crate::virtio::dev::{DevSpec, Virtio, WakeEvent}; @@ -302,15 +301,14 @@ impl Virtio for Block { self.feature.bits() | FEATURE_BUILT_IN } - fn spawn_worker( + fn spawn_worker( self, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where S: IrqSender, - E: IoeventFd, { match self.api { #[cfg(target_os = "linux")] @@ -323,41 +321,38 @@ impl Virtio for Block { impl VirtioMio for Block { fn reset(&mut self, _registry: &Registry) {} - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, _feature: u128, - _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { Ok(()) } - fn handle_event<'a, 'm, Q, S, E>( + fn handle_event<'a, 'm, Q, S>( &mut self, _event: &Event, - _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { Ok(()) } - fn handle_queue<'m, Q, S, E>( + fn handle_queue<'m, Q, S>( &mut self, index: u16, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let Some(Some(queue)) = active_mio.queues.get_mut(index as usize) else { log::error!("{}: invalid queue index {index}", self.name); @@ -418,15 +413,14 @@ impl VirtioMio for Block { #[cfg(target_os = "linux")] impl VirtioIoUring for Block { - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, _feature: u128, - _ring: &mut ActiveIoUring<'_, '_, 'm, Q, S, E>, + _ring: &mut ActiveIoUring<'_, '_, 'm, Q, S>, ) -> Result<()> where S: IrqSender, Q: VirtQueue<'m>, - E: IoeventFd, { Ok(()) } diff --git a/alioth/src/virtio/dev/dev.rs b/alioth/src/virtio/dev/dev.rs index 44d182b0..07018d00 100644 --- a/alioth/src/virtio/dev/dev.rs +++ b/alioth/src/virtio/dev/dev.rs @@ -31,7 +31,6 @@ use bitflags::Flags; use flume::{Receiver, Sender}; use snafu::ResultExt; -use crate::hv::IoeventFd; use crate::mem::emulated::Mmio; use crate::mem::mapped::{Ram, RamBus}; use crate::mem::{LayoutChanged, LayoutUpdated, MemRegion}; @@ -52,16 +51,16 @@ pub trait Virtio: Debug + Send + Sync + 'static { fn num_queues(&self) -> u16; fn config(&self) -> Arc; fn feature(&self) -> u128; - fn spawn_worker( + fn spawn_worker( self, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)>; fn shared_mem_regions(&self) -> Option> { None } - fn ioeventfd_offloaded(&self, _q_index: u16) -> Result { + fn notifier_offloaded(&self, _q_index: u16) -> Result { Ok(false) } fn mem_update_callback(&self) -> Option> { @@ -97,21 +96,19 @@ impl Register { const TOKEN_WARKER: u64 = 1 << 63; #[derive(Debug, Clone)] -pub struct StartParam +pub struct StartParam where S: IrqSender, - E: IoeventFd, { pub(crate) feature: u128, pub(crate) irq_sender: Arc, - pub(crate) ioeventfds: Option>, + pub(crate) notifiers: Option>, } #[derive(Debug, Clone)] -pub enum WakeEvent +pub enum WakeEvent where S: IrqSender, - E: IoeventFd, { Notify { q_index: u16, @@ -122,7 +119,7 @@ where channel: Arc, }, Start { - param: StartParam, + param: StartParam, }, Reset, } @@ -135,20 +132,18 @@ pub enum WorkerState { } #[derive(Debug)] -pub struct Worker +pub struct Worker where S: IrqSender, - E: IoeventFd, { - context: Context, + context: Context, backend: B, } #[derive(Debug)] -pub struct VirtioDevice +pub struct VirtioDevice where S: IrqSender, - E: IoeventFd, { pub name: Arc, pub id: DeviceId, @@ -157,14 +152,13 @@ where pub queue_regs: Arc<[QueueReg]>, pub shared_mem_regions: Option>, pub notifier: Arc, - pub event_tx: Sender>, + pub event_tx: Sender>, pub(crate) worker_handle: Option>, } -impl VirtioDevice +impl VirtioDevice where S: IrqSender, - E: IoeventFd, { fn shutdown(&mut self) -> Result<(), Box> { let Some(handle) = self.worker_handle.take() else { @@ -226,10 +220,9 @@ where } } -impl Drop for VirtioDevice +impl Drop for VirtioDevice where S: IrqSender, - E: IoeventFd, { fn drop(&mut self) { if let Err(e) = self.shutdown() { @@ -241,17 +234,16 @@ where pub trait Backend: Send + 'static { fn register_notifier(&mut self, token: u64) -> Result>; fn reset(&self, dev: &mut D) -> Result<()>; - fn event_loop<'m, S, Q, E>( + fn event_loop<'m, S, Q>( &mut self, memory: &'m Ram, - context: &mut Context, + context: &mut Context, queues: &mut [Option>], - param: &StartParam, + param: &StartParam, ) -> Result<()> where S: IrqSender, - Q: VirtQueue<'m>, - E: IoeventFd; + Q: VirtQueue<'m>; } pub trait BackendEvent { @@ -265,23 +257,21 @@ pub trait ActiveBackend { } #[derive(Debug)] -pub struct Context +pub struct Context where S: IrqSender, - E: IoeventFd, { pub dev: D, memory: Arc, - event_rx: Receiver>, + event_rx: Receiver>, queue_regs: Arc<[QueueReg]>, pub state: WorkerState, } -impl Context +impl Context where D: Virtio, S: IrqSender, - E: IoeventFd, { fn handle_wake_events(&mut self, backend: &mut B) -> Result<()> where @@ -309,7 +299,7 @@ where Ok(()) } - fn wait_start(&mut self) -> Option> { + fn wait_start(&mut self) -> Option> { for wake_event in self.event_rx.iter() { match wake_event { WakeEvent::Reset => {} @@ -344,17 +334,16 @@ where } } -impl Worker +impl Worker where D: Virtio, S: IrqSender, B: Backend, - E: IoeventFd, { pub fn spawn( dev: D, mut backend: B, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> { @@ -381,11 +370,10 @@ where &mut self, queues: &mut [Option>], ram: &'m Ram, - param: &StartParam, + param: &StartParam, ) -> Result<()> where Q: VirtQueue<'m>, - E: IoeventFd, { log::debug!( "{}: activated with {:x?}, {:x?}", diff --git a/alioth/src/virtio/dev/entropy.rs b/alioth/src/virtio/dev/entropy.rs index 0ec73621..a1522b25 100644 --- a/alioth/src/virtio/dev/entropy.rs +++ b/alioth/src/virtio/dev/entropy.rs @@ -27,7 +27,6 @@ use serde::Deserialize; use serde_aco::Help; use snafu::ResultExt; -use crate::hv::IoeventFd; use crate::mem::emulated::{Action, Mmio}; use crate::mem::mapped::RamBus; use crate::sync::notifier::Notifier; @@ -92,15 +91,14 @@ impl Virtio for Entropy { &self.name } - fn spawn_worker( + fn spawn_worker( self, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where S: IrqSender, - E: IoeventFd, { Mio::spawn_worker(self, event_rx, memory, queue_regs) } @@ -119,28 +117,26 @@ impl Virtio for Entropy { } impl VirtioMio for Entropy { - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, _feature: u128, - _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { Ok(()) } - fn handle_queue<'m, Q, S, E>( + fn handle_queue<'m, Q, S>( &mut self, index: u16, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let Some(Some(queue)) = active_mio.queues.get_mut(index as usize) else { log::error!("{}: invalid queue index {index}", self.name); @@ -149,15 +145,14 @@ impl VirtioMio for Entropy { queue.handle_desc(index, active_mio.irq_sender, copy_from_reader(&self.source)) } - fn handle_event<'a, 'm, Q, S, E>( + fn handle_event<'a, 'm, Q, S>( &mut self, _event: &Event, - _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { Ok(()) } diff --git a/alioth/src/virtio/dev/entropy_test.rs b/alioth/src/virtio/dev/entropy_test.rs index f95d0b54..66b63164 100644 --- a/alioth/src/virtio/dev/entropy_test.rs +++ b/alioth/src/virtio/dev/entropy_test.rs @@ -25,14 +25,13 @@ use tempfile::TempDir; use crate::ffi; use crate::mem::emulated::{Action, Mmio}; +use crate::sync::notifier::Notifier; use crate::virtio::dev::entropy::{EntropyConfig, EntropySpec}; use crate::virtio::dev::{DevSpec, StartParam, Virtio, WakeEvent}; use crate::virtio::queue::QueueReg; use crate::virtio::queue::split::SplitQueue; use crate::virtio::queue::tests::GuestQueue; -use crate::virtio::tests::{ - DATA_ADDR, FakeIoeventFd, FakeIrqSender, fixture_queues, fixture_ram_bus, -}; +use crate::virtio::tests::{DATA_ADDR, FakeIrqSender, fixture_queues, fixture_ram_bus}; use crate::virtio::{DeviceId, FEATURE_BUILT_IN, VirtioFeature}; #[test] @@ -83,7 +82,7 @@ fn entropy_test() { let start_param = StartParam { feature: VirtioFeature::VERSION_1.bits(), irq_sender, - ioeventfds: Option::>::None, + notifiers: Option::>::None, }; tx.send(WakeEvent::Start { param: start_param }).unwrap(); notifier.notify().unwrap(); diff --git a/alioth/src/virtio/dev/fs/fs.rs b/alioth/src/virtio/dev/fs/fs.rs index 649310af..37e66a3a 100644 --- a/alioth/src/virtio/dev/fs/fs.rs +++ b/alioth/src/virtio/dev/fs/fs.rs @@ -30,7 +30,6 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::fuse::bindings::{FuseInHeader, FuseOpcode, FuseOutHeader, FuseSetupmappingFlag}; use crate::fuse::{self, DaxRegion, Fuse}; -use crate::hv::IoeventFd; use crate::mem::mapped::{ArcMemPages, RamBus}; use crate::mem::{MemRegion, MemRegionType}; use crate::sync::notifier::Notifier; @@ -338,42 +337,39 @@ impl VirtioMio for Fs where F: Fuse + Debug + Send + Sync + 'static, { - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, feature: u128, - _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { self.driver_feature = FsFeature::from_bits_retain(feature); Ok(()) } - fn handle_event<'a, 'm, Q, S, E>( + fn handle_event<'a, 'm, Q, S>( &mut self, _event: &Event, - _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { unreachable!() } - fn handle_queue<'m, Q, S, E>( + fn handle_queue<'m, Q, S>( &mut self, index: u16, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let Some(Some(queue)) = active_mio.queues.get_mut(index as usize) else { log::error!("{}: invalid queue index {index}", self.name); @@ -424,15 +420,14 @@ where self.config.clone() } - fn spawn_worker( + fn spawn_worker( self, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where S: IrqSender, - E: IoeventFd, { Mio::spawn_worker(self, event_rx, memory, queue_regs) } diff --git a/alioth/src/virtio/dev/fs/vu.rs b/alioth/src/virtio/dev/fs/vu.rs index dfff162d..c83cb6a3 100644 --- a/alioth/src/virtio/dev/fs/vu.rs +++ b/alioth/src/virtio/dev/fs/vu.rs @@ -32,7 +32,6 @@ use zerocopy::{FromZeros, IntoBytes}; use crate::errors::BoxTrace; use crate::fuse::bindings::FuseSetupmappingFlag; use crate::fuse::{self, DaxRegion}; -use crate::hv::IoeventFd; use crate::mem::mapped::{ArcMemPages, RamBus}; use crate::mem::{LayoutChanged, MemRegion, MemRegionType}; use crate::sync::notifier::Notifier; @@ -151,21 +150,20 @@ impl Virtio for VuFs { self.frontend.num_queues() } - fn spawn_worker( + fn spawn_worker( self, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where S: IrqSender, - E: IoeventFd, { Mio::spawn_worker(self, event_rx, memory, queue_regs) } - fn ioeventfd_offloaded(&self, q_index: u16) -> Result { - self.frontend.ioeventfd_offloaded(q_index) + fn notifier_offloaded(&self, q_index: u16) -> Result { + self.frontend.notifier_offloaded(q_index) } fn shared_mem_regions(&self) -> Option> { @@ -182,15 +180,14 @@ impl Virtio for VuFs { } impl VirtioMio for VuFs { - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, feature: u128, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { self.frontend.activate(feature, active_mio)?; if let Some(channel) = self.frontend.channel() { @@ -204,15 +201,14 @@ impl VirtioMio for VuFs { Ok(()) } - fn handle_event<'a, 'm, Q, S, E>( + fn handle_event<'a, 'm, Q, S>( &mut self, event: &Event, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let q_index = event.token().0; if q_index < active_mio.queues.len() { @@ -304,15 +300,14 @@ impl VirtioMio for VuFs { Ok(()) } - fn handle_queue<'m, Q, S, E>( + fn handle_queue<'m, Q, S>( &mut self, index: u16, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { self.frontend.handle_queue(index, active_mio) } diff --git a/alioth/src/virtio/dev/net/tap.rs b/alioth/src/virtio/dev/net/tap.rs index deef4eb3..c555299f 100644 --- a/alioth/src/virtio/dev/net/tap.rs +++ b/alioth/src/virtio/dev/net/tap.rs @@ -36,7 +36,6 @@ use serde_aco::Help; use zerocopy::{FromBytes, IntoBytes}; use crate::device::net::MacAddr; -use crate::hv::IoeventFd; use crate::mem::mapped::RamBus; use crate::sync::notifier::Notifier; use crate::sys::if_tun::{TunFeature, tun_set_iff, tun_set_offload, tun_set_vnet_hdr_sz}; @@ -224,15 +223,14 @@ impl Virtio for Net { self.feature.bits() | FEATURE_BUILT_IN } - fn spawn_worker( + fn spawn_worker( self, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where S: IrqSender, - E: IoeventFd, { match self.api { WorkerApi::Mio => Mio::spawn_worker(self, event_rx, memory, queue_regs), @@ -247,15 +245,14 @@ impl VirtioMio for Net { let _ = registry.deregister(&mut SourceFd(&self.tap_sockets[0].as_raw_fd())); } - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, feature: u128, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { self.driver_feature = NetFeature::from_bits_retain(feature); let socket = &mut self.tap_sockets[0]; @@ -268,15 +265,14 @@ impl VirtioMio for Net { Ok(()) } - fn handle_event<'a, 'm, Q, S, E>( + fn handle_event<'a, 'm, Q, S>( &mut self, event: &Event, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let token = event.token().0; let irq_sender = active_mio.irq_sender; @@ -307,15 +303,14 @@ impl VirtioMio for Net { Ok(()) } - fn handle_queue<'m, Q, S, E>( + fn handle_queue<'m, Q, S>( &mut self, index: u16, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let Some(Some(queue)) = active_mio.queues.get_mut(index as usize) else { log::error!("{}: invalid queue index {index}", self.name); @@ -342,15 +337,14 @@ impl VirtioMio for Net { } impl VirtioIoUring for Net { - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, feature: u128, - _ring: &mut ActiveIoUring<'_, '_, 'm, Q, S, E>, + _ring: &mut ActiveIoUring<'_, '_, 'm, Q, S>, ) -> Result<()> where S: IrqSender, Q: VirtQueue<'m>, - E: IoeventFd, { self.driver_feature = NetFeature::from_bits_retain(feature); let socket = &mut self.tap_sockets[0]; diff --git a/alioth/src/virtio/dev/net/vmnet.rs b/alioth/src/virtio/dev/net/vmnet.rs index 30ce8272..c2743d1b 100644 --- a/alioth/src/virtio/dev/net/vmnet.rs +++ b/alioth/src/virtio/dev/net/vmnet.rs @@ -30,7 +30,6 @@ use serde_aco::Help; use zerocopy::IntoBytes; use crate::device::net::MacAddr; -use crate::hv::IoeventFd; use crate::mem::mapped::RamBus; use crate::sync::notifier::Notifier; use crate::sys::block::{_NSConcreteStackBlock, BlockDescriptor, BlockFlag}; @@ -260,15 +259,14 @@ impl Virtio for Net { self.feature.bits() | FEATURE_BUILT_IN } - fn spawn_worker( + fn spawn_worker( self, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where S: IrqSender, - E: IoeventFd, { Mio::spawn_worker(self, event_rx, memory, queue_regs) } @@ -293,15 +291,14 @@ impl VirtioMio for Net { let _ = registry.deregister(&mut self.rx_notifier); } - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, _feature: u128, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let registry = active_mio.poll.registry(); registry.register(&mut self.rx_notifier, Token(0), Interest::READABLE)?; @@ -351,15 +348,14 @@ impl VirtioMio for Net { Ok(()) } - fn handle_event<'a, 'm, Q, S, E>( + fn handle_event<'a, 'm, Q, S>( &mut self, event: &Event, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let token = event.token().0; let irq_sender = active_mio.irq_sender; @@ -375,15 +371,14 @@ impl VirtioMio for Net { Ok(()) } - fn handle_queue<'m, Q, S, E>( + fn handle_queue<'m, Q, S>( &mut self, index: u16, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let Some(Some(queue)) = active_mio.queues.get_mut(index as usize) else { log::error!("{}: invalid queue index {index}", self.name); diff --git a/alioth/src/virtio/dev/vsock/uds_vsock.rs b/alioth/src/virtio/dev/vsock/uds_vsock.rs index 336e35d6..913a20eb 100644 --- a/alioth/src/virtio/dev/vsock/uds_vsock.rs +++ b/alioth/src/virtio/dev/vsock/uds_vsock.rs @@ -33,7 +33,6 @@ use serde_aco::Help; use zerocopy::{FromBytes, IntoBytes}; use crate::ffi; -use crate::hv::IoeventFd; use crate::mem::mapped::RamBus; use crate::sync::notifier::Notifier; use crate::virtio::dev::vsock::{ @@ -473,14 +472,10 @@ impl UdsVsock { } } - fn handle_tx<'m, Q, S, E>( - &mut self, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, - ) -> Result<()> + fn handle_tx<'m, Q, S>(&mut self, active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let [Some(rx_q), Some(tx_q), ..] = active_mio.queues else { let tx_index = VsockVirtq::TX.raw(); @@ -793,30 +788,28 @@ impl Virtio for UdsVsock { VsockFeature::STREAM.bits() | FEATURE_BUILT_IN } - fn spawn_worker( + fn spawn_worker( self, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where S: IrqSender, - E: IoeventFd, { Mio::spawn_worker(self, event_rx, memory, queue_regs) } } impl VirtioMio for UdsVsock { - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, _feature: u128, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { active_mio.poll.registry().register( &mut SourceFd(&self.listener.as_raw_fd()), @@ -826,15 +819,14 @@ impl VirtioMio for UdsVsock { Ok(()) } - fn handle_event<'m, Q, S, E>( + fn handle_event<'m, Q, S>( &mut self, event: &Event, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let token = event.token(); let registry = active_mio.poll.registry(); @@ -856,15 +848,14 @@ impl VirtioMio for UdsVsock { } } - fn handle_queue<'m, Q, S, E>( + fn handle_queue<'m, Q, S>( &mut self, index: u16, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let index = VsockVirtq::from(index); let name = &self.name; diff --git a/alioth/src/virtio/dev/vsock/uds_vsock_test.rs b/alioth/src/virtio/dev/vsock/uds_vsock_test.rs index 60d360a5..ee503253 100644 --- a/alioth/src/virtio/dev/vsock/uds_vsock_test.rs +++ b/alioth/src/virtio/dev/vsock/uds_vsock_test.rs @@ -35,9 +35,7 @@ use crate::virtio::dev::{DevSpec, StartParam, Virtio, WakeEvent}; use crate::virtio::queue::QueueReg; use crate::virtio::queue::split::SplitQueue; use crate::virtio::queue::tests::{GuestQueue, VirtQueueGuest}; -use crate::virtio::tests::{ - DATA_ADDR, FakeIoeventFd, FakeIrqSender, fixture_queues, fixture_ram_bus, -}; +use crate::virtio::tests::{DATA_ADDR, FakeIrqSender, fixture_queues, fixture_ram_bus}; use crate::virtio::{DeviceId, FEATURE_BUILT_IN, VirtioFeature}; #[test] @@ -58,7 +56,7 @@ fn send_to_tx<'m, Q>( ram: &'m Ram, buf_addr: u64, q: &mut GuestQueue<'m, Q>, - tx: &Sender>, + tx: &Sender>, notifier: &Notifier, irq_rx: &Receiver, expect_rx: bool, @@ -141,7 +139,7 @@ fn vsock_conn_test() { let start_param = StartParam { feature: VirtioFeature::VERSION_1.bits(), irq_sender, - ioeventfds: Option::>::None, + notifiers: Option::>::None, }; tx.send(WakeEvent::Start { param: start_param }).unwrap(); @@ -421,7 +419,7 @@ fn vsock_host_close_test() { let start_param = StartParam { feature: VirtioFeature::VERSION_1.bits(), irq_sender, - ioeventfds: Option::>::None, + notifiers: Option::>::None, }; tx.send(WakeEvent::Start { param: start_param }).unwrap(); @@ -539,7 +537,7 @@ fn vsock_host_close_no_desc_test() { let start_param = StartParam { feature: VirtioFeature::VERSION_1.bits(), irq_sender, - ioeventfds: Option::>::None, + notifiers: Option::>::None, }; tx.send(WakeEvent::Start { param: start_param }).unwrap(); diff --git a/alioth/src/virtio/dev/vsock/vhost_vsock.rs b/alioth/src/virtio/dev/vsock/vhost_vsock.rs index f9bf41d5..6a717460 100644 --- a/alioth/src/virtio/dev/vsock/vhost_vsock.rs +++ b/alioth/src/virtio/dev/vsock/vhost_vsock.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; +use std::os::fd::{AsFd, AsRawFd, FromRawFd, OwnedFd}; use std::path::Path; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -27,7 +27,6 @@ use serde::Deserialize; use serde_aco::Help; use crate::ffi; -use crate::hv::IoeventFd; use crate::mem::LayoutUpdated; use crate::mem::mapped::RamBus; use crate::sync::notifier::Notifier; @@ -123,7 +122,7 @@ impl Virtio for VhostVsock { self.features as u128 } - fn ioeventfd_offloaded(&self, q_index: u16) -> Result { + fn notifier_offloaded(&self, q_index: u16) -> Result { match q_index { 0 | 1 => Ok(true), _ => Ok(false), @@ -136,36 +135,34 @@ impl Virtio for VhostVsock { })) } - fn spawn_worker( + fn spawn_worker( self, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where S: IrqSender, - E: IoeventFd, { Mio::spawn_worker(self, event_rx, memory, queue_regs) } } impl VirtioMio for VhostVsock { - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, feature: u128, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { self.vhost_dev.set_features(&(feature as u64))?; - for (index, fd) in active_mio.ioeventfds.iter().take(2).enumerate() { + for (index, notifier) in active_mio.notifiers.iter().take(2).enumerate() { let kick = VirtqFile { index: index as u32, - fd: fd.as_fd().as_raw_fd(), + fd: notifier.as_fd().as_raw_fd(), }; self.vhost_dev.set_virtq_kick(&kick)?; } @@ -237,15 +234,14 @@ impl VirtioMio for VhostVsock { } } - fn handle_event<'a, 'm, Q, S, E>( + fn handle_event<'a, 'm, Q, S>( &mut self, event: &Event, - _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let q_index = event.token(); error::VhostQueueErr { @@ -256,15 +252,14 @@ impl VirtioMio for VhostVsock { Ok(()) } - fn handle_queue<'m, Q, S, E>( + fn handle_queue<'m, Q, S>( &mut self, index: u16, - _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { match index { 0 | 1 => unreachable!("{}: queue 0 and 1 are offloaded to kernel", self.name), diff --git a/alioth/src/virtio/pci.rs b/alioth/src/virtio/pci.rs index 46c17655..3cb6411f 100644 --- a/alioth/src/virtio/pci.rs +++ b/alioth/src/virtio/pci.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::cmp::min; -use std::io::ErrorKind; use std::marker::PhantomData; use std::mem::size_of; use std::os::fd::{AsFd, AsRawFd, BorrowedFd}; @@ -26,7 +25,7 @@ use parking_lot::{Mutex, RwLock}; use zerocopy::{FromZeros, Immutable, IntoBytes}; use crate::device::Pause; -use crate::hv::{self, IoeventFd, IoeventFdRegistry, IrqFd, MsiSender}; +use crate::hv::{IrqFd, MsiSender, NotifierRegistry}; use crate::mem::emulated::{Action, Mmio}; use crate::mem::{MemRange, MemRegion, MemRegionCallback, MemRegionEntry}; use crate::pci::cap::{ @@ -185,26 +184,24 @@ pub struct VirtioPciRegister { } #[derive(Debug)] -pub struct VirtioPciRegisterMmio +pub struct VirtioPciRegisterMmio where M: MsiSender, - E: IoeventFd, { name: Arc, reg: Register, queues: Arc<[QueueReg]>, irq_sender: Arc>, - ioeventfds: Option>, - event_tx: Sender, E>>, + notifiers: Option>, + event_tx: Sender>>, notifier: Arc, } -impl VirtioPciRegisterMmio +impl VirtioPciRegisterMmio where M: MsiSender, - E: IoeventFd, { - fn wake_up_dev(&self, event: WakeEvent, E>) { + fn wake_up_dev(&self, event: WakeEvent>) { let is_start = matches!(event, WakeEvent::Start { .. }); if let Err(e) = self.event_tx.send(event) { log::error!("{}: failed to send event: {e}", self.name); @@ -252,10 +249,9 @@ where } } -impl Mmio for VirtioPciRegisterMmio +impl Mmio for VirtioPciRegisterMmio where M: MsiSender, - E: IoeventFd, { fn size(&self) -> u64 { // Reserve an extra 4-byte slot at the end of the notify capability to safely @@ -450,7 +446,7 @@ where let param = StartParam { feature: reg.get_driver_feature(), irq_sender: self.irq_sender.clone(), - ioeventfds: self.ioeventfds.clone(), + notifiers: self.notifiers.clone(), }; self.wake_up_dev(WakeEvent::Start { param }); } @@ -576,7 +572,7 @@ where + size_of::() * self.queues.len() => { let q_index = (offset - VirtioPciRegister::OFFSET_QUEUE_NOTIFY) as u16 / 4; - if self.ioeventfds.is_some() { + if self.notifiers.is_some() { log::warn!("{}: notifying queue-{q_index} by vm exit!", self.name); } let event = WakeEvent::Notify { q_index }; @@ -595,32 +591,32 @@ where } #[derive(Debug)] -struct IoeventFdCallback +struct NotifierCallback where - R: IoeventFdRegistry, + R: NotifierRegistry, { registry: R, - ioeventfds: Arc<[R::IoeventFd]>, + notifiers: Arc<[Notifier]>, } -impl MemRegionCallback for IoeventFdCallback +impl MemRegionCallback for NotifierCallback where - R: IoeventFdRegistry, + R: NotifierRegistry, { fn mapped(&self, addr: u64) -> mem::Result<()> { - for (q_index, fd) in self.ioeventfds.iter().enumerate() { + for (q_index, notifier) in self.notifiers.iter().enumerate() { let base_addr = addr + (12 << 10) + VirtioPciRegister::OFFSET_QUEUE_NOTIFY as u64; let notify_addr = base_addr + (q_index * size_of::()) as u64; - self.registry.register(fd, notify_addr, 0, None)?; - log::info!("q-{q_index} ioeventfd registered at {notify_addr:x}",) + self.registry.register(notifier, notify_addr, 0, None)?; + log::info!("q-{q_index} notifier registered at {notify_addr:x}",) } Ok(()) } fn unmapped(&self) -> mem::Result<()> { - for fd in self.ioeventfds.iter() { - self.registry.deregister(fd)?; - log::info!("ioeventfd {fd:?} de-registered") + for notifier in self.notifiers.iter() { + self.registry.deregister(notifier)?; + log::info!("notifier {notifier:?} de-registered") } Ok(()) } @@ -719,28 +715,26 @@ impl PciCap for VirtioPciNotifyCap { } #[derive(Debug)] -pub struct VirtioPciDevice +pub struct VirtioPciDevice where M: MsiSender, - E: IoeventFd, { - pub dev: VirtioDevice, E>, + pub dev: VirtioDevice>, pub config: EmulatedConfig, - pub registers: Arc>, + pub registers: Arc>, } -impl VirtioPciDevice +impl VirtioPciDevice where M: MsiSender, - E: IoeventFd, { pub fn new( - dev: VirtioDevice, E>, + dev: VirtioDevice>, msi_sender: M, - ioeventfd_reg: R, + notifier_reg: Option, ) -> Result where - R: IoeventFdRegistry, + R: NotifierRegistry, { let (class, subclass) = get_class(dev.id); let mut header = DeviceHeader { @@ -891,19 +885,21 @@ where .collect(), }; - let maybe_ioeventfds = (0..num_queues) - .map(|_| ioeventfd_reg.create()) - .collect::, _>>(); - let ioeventfds = match maybe_ioeventfds { - Ok(fds) => Some(fds), - Err(hv::Error::IoeventFd { error, .. }) if error.kind() == ErrorKind::Unsupported => { - None - } - Err(e) => { - log::warn!("{}: failed to create ioeventfds: {e:?}", dev.name); - None - } + // `Some` iff the hypervisor supports notifiers. Keeping the registry and + // the notifiers in one value makes it impossible to end up with + // notifiers that no registry ever registers, or vice versa. + let notifier_reg = match notifier_reg { + Some(registry) => { + let notifiers = (0..num_queues) + .map(|_| Notifier::new()) + .collect::, _>>()?; + Some((registry, notifiers)) + } + None => None, }; + let notifiers = notifier_reg + .as_ref() + .map(|(_, notifiers)| notifiers.clone()); let mut device_feature = [0u32; 4]; for (i, v) in device_feature.iter_mut().enumerate() { @@ -923,16 +919,16 @@ where msix_table: msix_table.clone(), msi_sender, }), - ioeventfds: ioeventfds.clone(), + notifiers: notifiers.clone(), }); bar0.ranges.push(MemRange::Emulated(msix_table)); bar0.ranges .push(MemRange::Span((12 << 10) - msix_table_size as u64)); bar0.ranges.push(MemRange::Emulated(registers.clone())); - if let Some(ioeventfds) = ioeventfds { - bar0.callbacks.lock().push(Box::new(IoeventFdCallback { - registry: ioeventfd_reg, - ioeventfds, + if let Some((registry, notifiers)) = notifier_reg { + bar0.callbacks.lock().push(Box::new(NotifierCallback { + registry, + notifiers, })); } if device_config.size() > 0 { @@ -968,17 +964,11 @@ where } } -impl Pause for VirtioPciDevice -where - M: MsiSender, - E: IoeventFd, -{ -} +impl Pause for VirtioPciDevice where M: MsiSender {} -impl Pci for VirtioPciDevice +impl Pci for VirtioPciDevice where M: MsiSender, - E: IoeventFd, { fn name(&self) -> &str { &self.dev.name diff --git a/alioth/src/virtio/pci_test.rs b/alioth/src/virtio/pci_test.rs index f32b4396..af26d163 100644 --- a/alioth/src/virtio/pci_test.rs +++ b/alioth/src/virtio/pci_test.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::io::ErrorKind; use std::mem::size_of; use std::os::fd::AsRawFd; use std::sync::Arc; @@ -23,10 +22,7 @@ use flume::Receiver; use parking_lot::{Mutex, RwLock}; use rstest::rstest; -use crate::hv::IoeventFd; -use crate::hv::tests::{ - RegisteredAddr, TestIoeventFd, TestIoeventFdRegistry, TestIrqFd, TestMsiSender, -}; +use crate::hv::tests::{RegisteredAddr, TestIrqFd, TestMsiSender, TestNotifierRegistry}; use crate::mem::emulated::{Action, Mmio}; use crate::mem::{self, MemRange, MemRegion, MemRegionEntry, MemRegionType}; use crate::pci::cap::{ @@ -42,11 +38,10 @@ use crate::virtio::pci::{ VirtioPciRegisterMmio, }; use crate::virtio::queue::{QUEUE_SIZE_MAX, QueueReg}; -use crate::virtio::tests::FakeIoeventFd; use crate::virtio::{DevStatus, DeviceId, IrqSender, VirtioFeature}; -type TestMmio = VirtioPciRegisterMmio; -type TestWakeReceiver = flume::Receiver, FakeIoeventFd>>; +type TestMmio = VirtioPciRegisterMmio; +type TestWakeReceiver = flume::Receiver>>; fn create_test_mmio(queues: Arc<[QueueReg]>) -> (TestMmio, TestWakeReceiver) { let (event_tx, event_rx) = flume::unbounded(); @@ -74,7 +69,7 @@ fn create_test_mmio(queues: Arc<[QueueReg]>) -> (TestMmio, TestWakeReceiver) { }, queues, irq_sender, - ioeventfds: None, + notifiers: None, event_tx, notifier, }; @@ -950,7 +945,7 @@ fn test_queue_notify(#[case] offset: usize, #[case] expect_wake: bool) { } #[test] -fn test_notify_with_ioeventfds() { +fn test_notify_with_notifiers() { let queues = Arc::new([QueueReg::default()]); let (event_tx, event_rx) = flume::unbounded(); let notifier = Arc::new(Notifier::new().unwrap()); @@ -967,14 +962,14 @@ fn test_notify_with_ioeventfds() { msi_sender, }); let mmio = VirtioPciRegisterMmio { - name: "test-virtio-pci-ioeventfd".into(), + name: "test-virtio-pci-notifier".into(), reg: Register { device_feature: [u32::MAX; 4], ..Default::default() }, queues, irq_sender, - ioeventfds: Some(Arc::new([FakeIoeventFd])), + notifiers: Some(Arc::new([Notifier::new().unwrap()])), event_tx, notifier, }; @@ -1049,15 +1044,14 @@ impl Mmio for TestDevConfig { } } -fn create_test_virtio_device( +fn create_test_virtio_device( id: DeviceId, config_size: u64, shared_mem: Option>, num_queues: usize, -) -> (VirtioDevice, Receiver>) +) -> (VirtioDevice, Receiver>) where S: IrqSender, - E: IoeventFd, { let (event_tx, event_rx) = flume::unbounded(); let notifier = Arc::new(Notifier::new().unwrap()); @@ -1283,12 +1277,11 @@ fn test_virtio_pci_device_classes( #[case] expected_subclass: u8, #[case] expected_dev_id: u16, ) { - let (dev, _rx) = - create_test_virtio_device::, TestIoeventFd>(id, 0, None, 2); + let (dev, _rx) = create_test_virtio_device::>(id, 0, None, 2); let pci_dev = VirtioPciDevice::new( dev, TestMsiSender::default(), - TestIoeventFdRegistry::default(), + Some(TestNotifierRegistry::default()), ) .unwrap(); @@ -1337,7 +1330,7 @@ fn test_virtio_pci_device_with_config_and_shared_memory_prefetchable() { ], callbacks: Mutex::new(vec![]), }); - let (dev, _rx) = create_test_virtio_device::, TestIoeventFd>( + let (dev, _rx) = create_test_virtio_device::>( DeviceId::FILE_SYSTEM, 32, Some(shared_mem), @@ -1346,7 +1339,7 @@ fn test_virtio_pci_device_with_config_and_shared_memory_prefetchable() { let pci_dev = VirtioPciDevice::new( dev, TestMsiSender::default(), - TestIoeventFdRegistry::default(), + Some(TestNotifierRegistry::default()), ) .unwrap(); @@ -1389,7 +1382,7 @@ fn test_virtio_pci_device_shared_memory_non_prefetchable() { }], callbacks: Mutex::new(vec![]), }); - let (dev, _rx) = create_test_virtio_device::, TestIoeventFd>( + let (dev, _rx) = create_test_virtio_device::>( DeviceId::FILE_SYSTEM, 0, Some(shared_mem), @@ -1398,7 +1391,7 @@ fn test_virtio_pci_device_shared_memory_non_prefetchable() { let pci_dev = VirtioPciDevice::new( dev, TestMsiSender::default(), - TestIoeventFdRegistry::default(), + Some(TestNotifierRegistry::default()), ) .unwrap(); @@ -1407,17 +1400,13 @@ fn test_virtio_pci_device_shared_memory_non_prefetchable() { } #[test] -fn test_virtio_pci_device_ioeventfd_callback() { - let (dev, _rx) = create_test_virtio_device::, TestIoeventFd>( - DeviceId::NET, - 0, - None, - 2, - ); - let registry = TestIoeventFdRegistry::default(); +fn test_virtio_pci_device_notifier_callback() { + let (dev, _rx) = + create_test_virtio_device::>(DeviceId::NET, 0, None, 2); + let registry = TestNotifierRegistry::default(); let registered = registry.registered.clone(); let deregistered = registry.deregistered.clone(); - let pci_dev = VirtioPciDevice::new(dev, TestMsiSender::default(), registry).unwrap(); + let pci_dev = VirtioPciDevice::new(dev, TestMsiSender::default(), Some(registry)).unwrap(); let PciBar::Mem(bar0) = &pci_dev.config.header.bars[0] else { panic!("expected Mem BAR"); @@ -1455,37 +1444,34 @@ fn test_virtio_pci_device_ioeventfd_callback() { assert_eq!(*deregistered.lock(), 2); } -#[rstest] -#[case(Some(ErrorKind::Unsupported))] -#[case(Some(ErrorKind::PermissionDenied))] -fn test_virtio_pci_device_ioeventfd_fallback(#[case] fail_mode: Option) { - let (dev, _rx) = create_test_virtio_device::, TestIoeventFd>( - DeviceId::NET, - 0, - None, - 1, - ); - let registry = TestIoeventFdRegistry { - fail_mode, - ..Default::default() - }; - let pci_dev = VirtioPciDevice::new(dev, TestMsiSender::default(), registry).unwrap(); +#[test] +fn test_virtio_pci_device_notifier_fallback() { + let (dev, _rx) = + create_test_virtio_device::>(DeviceId::NET, 0, None, 1); + let pci_dev = VirtioPciDevice::new( + dev, + TestMsiSender::default(), + Option::::None, + ) + .unwrap(); - assert!(pci_dev.registers.ioeventfds.is_none()); + assert!(pci_dev.registers.notifiers.is_none()); + let PciBar::Mem(bar0) = &pci_dev.config.header.bars[0] else { + panic!("expected Mem BAR"); + }; + // `EmulatedConfig::new_device()` always pushes a `BarCallback`, so the + // only callback left means no `NotifierCallback` was installed. + assert_eq!(bar0.callbacks.lock().len(), 1); } #[test] fn test_virtio_pci_device_pci_reset() { - let (dev, event_rx) = create_test_virtio_device::, TestIoeventFd>( - DeviceId::NET, - 0, - None, - 1, - ); + let (dev, event_rx) = + create_test_virtio_device::>(DeviceId::NET, 0, None, 1); let pci_dev = VirtioPciDevice::new( dev, TestMsiSender::default(), - TestIoeventFdRegistry::default(), + Some(TestNotifierRegistry::default()), ) .unwrap(); diff --git a/alioth/src/virtio/virtio_test.rs b/alioth/src/virtio/virtio_test.rs index 2d39addd..61babf41 100644 --- a/alioth/src/virtio/virtio_test.rs +++ b/alioth/src/virtio/virtio_test.rs @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::os::fd::{AsFd, BorrowedFd}; +use std::os::fd::BorrowedFd; use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU64}; use flume::Sender; use rstest::rstest; -use crate::hv::IoeventFd; use crate::mem::mapped::{ArcMemPages, RamBus}; use crate::virtio::queue::{QUEUE_SIZE_MAX, QueueReg}; use crate::virtio::{DevStatus, IrqSender, Result}; @@ -79,17 +78,6 @@ impl IrqSender for FakeIrqSender { } } -#[derive(Debug, Default)] -pub struct FakeIoeventFd; - -impl AsFd for FakeIoeventFd { - fn as_fd(&self) -> BorrowedFd<'_> { - unreachable!() - } -} - -impl IoeventFd for FakeIoeventFd {} - #[rstest] // Valid states in standard initialization sequence #[case(DevStatus::empty(), true)] diff --git a/alioth/src/virtio/vu/backend.rs b/alioth/src/virtio/vu/backend.rs index ea753672..a672e89e 100644 --- a/alioth/src/virtio/vu/backend.rs +++ b/alioth/src/virtio/vu/backend.rs @@ -16,7 +16,7 @@ use std::cmp::min; use std::fs::File; use std::io::{ErrorKind, Write}; use std::iter::zip; -use std::os::fd::{AsFd, AsRawFd, BorrowedFd, OwnedFd}; +use std::os::fd::{AsRawFd, BorrowedFd, OwnedFd}; use std::os::unix::net::UnixStream; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -26,8 +26,8 @@ use snafu::Snafu; use zerocopy::IntoBytes; use crate::errors::DebugTrace; -use crate::hv::IoeventFd; use crate::mem::mapped::{ArcMemPages, RamBus}; +use crate::sync::notifier::Notifier; use crate::virtio::dev::{StartParam, VirtioDevice, WakeEvent}; use crate::virtio::vu::Error as VuError; use crate::virtio::vu::bindings::{ @@ -59,8 +59,8 @@ pub enum Error { MissingSize { index: u16 }, #[snafu(display("frontend did not set addresses for queue {index}"))] MissingAddr { index: u16 }, - #[snafu(display("frontend did not set ioeventfd for queue {index}"))] - MissingIoeventfd { index: u16 }, + #[snafu(display("frontend did not set kick fd for queue {index}"))] + MissingNotifier { index: u16 }, #[snafu(display("cannot convert frontend HVA {hva:#x} to GPA"))] Convert { hva: u64 }, #[snafu(display("invalid message {req:?} with payload size {size}"))] @@ -119,25 +119,12 @@ impl IrqSender for VuIrqSender { } } -#[derive(Debug)] -pub struct VuEventfd { - fd: File, -} - -impl AsFd for VuEventfd { - fn as_fd(&self) -> BorrowedFd<'_> { - self.fd.as_fd() - } -} - -impl IoeventFd for VuEventfd {} - #[derive(Debug, Default)] struct VuQueueInit { enable: bool, size: Option, addr: Option, - ioeventfd: Option, + notifier: Option, irqfd: Option, errfd: Option, } @@ -154,14 +141,14 @@ pub struct VuBackend { channel: Option>, status: DevStatus, memory: Arc, - dev: VirtioDevice, + dev: VirtioDevice, init: VuInit, } impl VuBackend { pub fn new( conn: UnixStream, - dev: VirtioDevice, + dev: VirtioDevice, memory: Arc, ) -> Result { conn.set_nonblocking(false)?; @@ -184,7 +171,7 @@ impl VuBackend { self.dev.name.as_ref() } - fn wake_up_dev(&self, event: WakeEvent) { + fn wake_up_dev(&self, event: WakeEvent) { let is_start = matches!(event, WakeEvent::Start { .. }); if let Err(e) = self.dev.event_tx.send(event) { log::error!("{}: failed to send event: {e}", self.dev.name); @@ -207,7 +194,7 @@ impl VuBackend { error::Convert { hva }.fail() } - fn parse_init(&mut self) -> Result> { + fn parse_init(&mut self) -> Result> { for (index, (param, queue)) in zip(&self.init.queues, &*self.dev.queue_regs).enumerate() { let index = index as u16; queue.enabled.store(param.enable, Ordering::Release); @@ -241,13 +228,13 @@ impl VuBackend { queues: queue_irqfds, }; - let mut ioeventfds = vec![]; + let mut notifiers = vec![]; for (index, q) in queues.iter_mut().enumerate() { - match q.ioeventfd.take() { - Some(fd) => ioeventfds.push(VuEventfd { fd }), + match q.notifier.take() { + Some(notifier) => notifiers.push(notifier), None => { let index = index as u16; - return error::MissingIoeventfd { index }.fail(); + return error::MissingNotifier { index }.fail(); } } } @@ -255,7 +242,7 @@ impl VuBackend { Ok(StartParam { feature: self.init.drv_feat as u128, irq_sender: Arc::new(irq_sender), - ioeventfds: Some(ioeventfds.into()), + notifiers: Some(notifiers.into()), }) } @@ -348,7 +335,7 @@ impl VuBackend { return error::InvalidQueue { index }.fail(); }; log::debug!("{name}: queue-{index}: set kick fd: {}", fd.as_raw_fd()); - q.ioeventfd = Some(File::from(fd)); + q.notifier = Some(Notifier::try_from(fd)?); } (VuFrontMsg::SET_VIRTQ_NUM, 8) => { let virtq_num: VirtqState = self.session.recv_payload()?; diff --git a/alioth/src/virtio/vu/frontend.rs b/alioth/src/virtio/vu/frontend.rs index faf610b3..1acb5766 100644 --- a/alioth/src/virtio/vu/frontend.rs +++ b/alioth/src/virtio/vu/frontend.rs @@ -25,7 +25,6 @@ use mio::{Interest, Registry, Token}; use zerocopy::IntoBytes; use crate::errors::BoxTrace; -use crate::hv::IoeventFd; use crate::mem::emulated::{Action, Mmio}; use crate::mem::mapped::{ArcMemPages, RamBus}; use crate::mem::{LayoutChanged, MemRegion}; @@ -270,20 +269,19 @@ impl Virtio for VuFrontend { self.device_feature as u128 } - fn spawn_worker( + fn spawn_worker( self, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where S: IrqSender, - E: IoeventFd, { Mio::spawn_worker(self, event_rx, memory, queue_regs) } - fn ioeventfd_offloaded(&self, q_index: u16) -> Result { + fn notifier_offloaded(&self, q_index: u16) -> Result { if q_index < self.num_queues { Ok(true) } else { @@ -304,24 +302,24 @@ impl Virtio for VuFrontend { } impl VirtioMio for VuFrontend { - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, feature: u128, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { let name = &*self.name; self.session .set_features(&((feature | VirtioFeature::VHOST_PROTOCOL.bits()) as u64))?; log::trace!("{name}: set driver feature: {feature:x?}"); - for (index, fd) in active_mio.ioeventfds.iter().enumerate() { - self.session.set_virtq_kick(&(index as u64), fd.as_fd())?; - let raw_fd = fd.as_fd().as_raw_fd(); + for (index, notifier) in active_mio.notifiers.iter().enumerate() { + self.session + .set_virtq_kick(&(index as u64), notifier.as_fd())?; + let raw_fd = notifier.as_fd().as_raw_fd(); log::trace!("{name}: queue-{index}: set kick fd: {raw_fd}"); } @@ -388,28 +386,26 @@ impl VirtioMio for VuFrontend { Ok(()) } - fn handle_event<'a, 'm, Q, S, E>( + fn handle_event<'a, 'm, Q, S>( &mut self, _: &Event, - _: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { unreachable!() } - fn handle_queue<'m, Q, S, E>( + fn handle_queue<'m, Q, S>( &mut self, index: u16, - _: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + _: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { unreachable!( "{}: queue {index} notification should go to vhost-user backend", diff --git a/alioth/src/virtio/worker/io_uring.rs b/alioth/src/virtio/worker/io_uring.rs index c87b659e..b34e4014 100644 --- a/alioth/src/virtio/worker/io_uring.rs +++ b/alioth/src/virtio/worker/io_uring.rs @@ -22,7 +22,6 @@ use io_uring::cqueue::Entry as Cqe; use io_uring::squeue::Entry as Sqe; use io_uring::{SubmissionQueue, opcode, types}; -use crate::hv::IoeventFd; use crate::mem::mapped::{Ram, RamBus}; use crate::sync::notifier::Notifier; use crate::virtio::dev::{ @@ -38,15 +37,14 @@ pub enum BufferAction { } pub trait VirtioIoUring: Virtio { - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, feature: u128, - ring: &mut ActiveIoUring<'_, '_, 'm, Q, S, E>, + ring: &mut ActiveIoUring<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, - S: IrqSender, - E: IoeventFd; + S: IrqSender; fn handle_desc(&mut self, q_index: u16, chain: &mut DescChain) -> Result; @@ -70,15 +68,14 @@ impl IoUring { Ok(()) } - pub fn spawn_worker( + pub fn spawn_worker( dev: D, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where D: VirtioIoUring, - E: IoeventFd, S: IrqSender, { let notifier = Notifier::new()?; @@ -112,24 +109,23 @@ where Ok(()) } - fn event_loop<'m, S, Q, E>( + fn event_loop<'m, S, Q>( &mut self, memory: &'m Ram, - context: &mut Context, + context: &mut Context, queues: &mut [Option>], - param: &StartParam, + param: &StartParam, ) -> Result<()> where S: IrqSender, Q: VirtQueue<'m>, - E: IoeventFd, { let submit_counts = iter::repeat_n(0, queues.len()).collect(); let mut active_ring = ActiveIoUring { ring: io_uring::IoUring::new(RING_SIZE as u32)?, shared_count: RING_SIZE - 1, irq_sender: &*param.irq_sender, - ioeventfds: param.ioeventfds.as_deref().unwrap_or(&[]), + notifiers: param.notifiers.as_deref().unwrap_or(&[]), mem: memory, queues, submit_counts, @@ -137,13 +133,13 @@ where self.submit_notifier(&mut active_ring.ring.submission())?; context.dev.activate(param.feature, &mut active_ring)?; - if let Some(fds) = ¶m.ioeventfds { + if let Some(notifiers) = ¶m.notifiers { let sq = &mut active_ring.ring.submission(); - for (index, fd) in fds.iter().enumerate() { - if context.dev.ioeventfd_offloaded(index as u16)? { + for (index, notifier) in notifiers.iter().enumerate() { + if context.dev.notifier_offloaded(index as u16)? { continue; } - submit_queue_ioeventfd(index as u16, fd, sq)?; + submit_queue_notifier(index as u16, notifier, sq)?; active_ring.shared_count -= QUEUE_RESERVE_SIZE + 1; } } @@ -164,37 +160,33 @@ where } } -pub struct ActiveIoUring<'a, 'r, 'm, Q, S, E> +pub struct ActiveIoUring<'a, 'r, 'm, Q, S> where Q: VirtQueue<'m>, { ring: io_uring::IoUring, pub queues: &'a mut [Option>], pub irq_sender: &'a S, - pub ioeventfds: &'a [E], + pub notifiers: &'a [Notifier], pub mem: &'m Ram, shared_count: u16, submit_counts: Box<[u16]>, } -fn submit_queue_ioeventfd(index: u16, fd: &E, sq: &mut SubmissionQueue) -> Result<()> -where - E: IoeventFd, -{ +fn submit_queue_notifier(index: u16, notifier: &Notifier, sq: &mut SubmissionQueue) -> Result<()> { let token = index as u64 | TOKEN_QUEUE; - let fd = types::Fd(fd.as_fd().as_raw_fd()); + let fd = types::Fd(notifier.as_fd().as_raw_fd()); let poll = opcode::PollAdd::new(fd, libc::EPOLLIN as _).multi(true); let entry = poll.build().user_data(token); unsafe { sq.push(&entry) }.unwrap(); Ok(()) } -impl<'m, Q, S, E> ActiveIoUring<'_, '_, 'm, Q, S, E> +impl<'m, Q, S> ActiveIoUring<'_, '_, 'm, Q, S> where Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { fn submit_buffers(&mut self, dev: &mut D, q_index: u16) -> Result<()> where @@ -231,12 +223,11 @@ where } } -impl<'m, D, Q, S, E> ActiveBackend for ActiveIoUring<'_, '_, 'm, Q, S, E> +impl<'m, D, Q, S> ActiveBackend for ActiveIoUring<'_, '_, 'm, Q, S> where D: VirtioIoUring, Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { type Event = Cqe; diff --git a/alioth/src/virtio/worker/mio.rs b/alioth/src/virtio/worker/mio.rs index 38f7860d..e0c72d12 100644 --- a/alioth/src/virtio/worker/mio.rs +++ b/alioth/src/virtio/worker/mio.rs @@ -12,17 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::os::fd::AsRawFd; +#[cfg(target_os = "linux")] +use std::os::fd::{AsFd, AsRawFd}; use std::sync::Arc; use std::thread::JoinHandle; use flume::Receiver; use mio::event::Event; +#[cfg(target_os = "linux")] use mio::unix::SourceFd; use mio::{Events, Interest, Poll, Registry, Token}; use snafu::ResultExt; -use crate::hv::IoeventFd; use crate::mem::mapped::{Ram, RamBus}; use crate::sync::notifier::Notifier; use crate::virtio::dev::{ @@ -33,35 +34,32 @@ use crate::virtio::queue::{Queue, QueueReg, VirtQueue}; use crate::virtio::{IrqSender, Result, error}; pub trait VirtioMio: Virtio { - fn activate<'m, Q, S, E>( + fn activate<'m, Q, S>( &mut self, feature: u128, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, - S: IrqSender, - E: IoeventFd; + S: IrqSender; - fn handle_queue<'m, Q, S, E>( + fn handle_queue<'m, Q, S>( &mut self, index: u16, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, - S: IrqSender, - E: IoeventFd; + S: IrqSender; - fn handle_event<'m, Q, S, E>( + fn handle_event<'m, Q, S>( &mut self, event: &Event, - active_mio: &mut ActiveMio<'_, '_, 'm, Q, S, E>, + active_mio: &mut ActiveMio<'_, '_, 'm, Q, S>, ) -> Result<()> where Q: VirtQueue<'m>, - S: IrqSender, - E: IoeventFd; + S: IrqSender; fn reset(&mut self, registry: &Registry); } @@ -79,16 +77,15 @@ pub struct Mio { } impl Mio { - pub fn spawn_worker( + pub fn spawn_worker( dev: D, - event_rx: Receiver>, + event_rx: Receiver>, memory: Arc, queue_regs: Arc<[QueueReg]>, ) -> Result<(JoinHandle<()>, Arc)> where D: VirtioMio, S: IrqSender, - E: IoeventFd, { let poll = Poll::new().context(error::CreatePoll)?; let m = Mio { poll }; @@ -112,36 +109,37 @@ where Ok(()) } - fn event_loop<'m, S, Q, E>( + fn event_loop<'m, S, Q>( &mut self, memory: &'m Ram, - context: &mut Context, + context: &mut Context, queues: &mut [Option>], - param: &StartParam, + param: &StartParam, ) -> Result<()> where S: IrqSender, Q: VirtQueue<'m>, - E: IoeventFd, { let mut events = Events::with_capacity(128); let mut active_mio = ActiveMio { queues, irq_sender: &*param.irq_sender, - ioeventfds: param.ioeventfds.as_deref().unwrap_or(&[]), + notifiers: param.notifiers.as_deref().unwrap_or(&[]), poll: &mut self.poll, mem: memory, }; context.dev.activate(param.feature, &mut active_mio)?; - let registry = active_mio.poll.registry(); - for (index, fd) in active_mio.ioeventfds.iter().enumerate() { - if context.dev.ioeventfd_offloaded(index as u16)? { + // Only KVM offers ioeventfds, see [`crate::hv::NotifierRegistry`]. + #[cfg(target_os = "linux")] + for (index, notifier) in active_mio.notifiers.iter().enumerate() { + if context.dev.notifier_offloaded(index as u16)? { continue; } let token = index as u64 | TOKEN_QUEUE; + let registry = active_mio.poll.registry(); registry .register( - &mut SourceFd(&fd.as_fd().as_raw_fd()), + &mut SourceFd(¬ifier.as_fd().as_raw_fd()), Token(token as usize), Interest::READABLE, ) @@ -159,36 +157,36 @@ where } } } - let registry = active_mio.poll.registry(); - for (index, fd) in active_mio.ioeventfds.iter().enumerate() { - if context.dev.ioeventfd_offloaded(index as u16)? { + #[cfg(target_os = "linux")] + for (index, notifier) in active_mio.notifiers.iter().enumerate() { + if context.dev.notifier_offloaded(index as u16)? { continue; } + let registry = active_mio.poll.registry(); registry - .deregister(&mut SourceFd(&fd.as_fd().as_raw_fd())) + .deregister(&mut SourceFd(¬ifier.as_fd().as_raw_fd())) .context(error::EventSource)?; } Ok(()) } } -pub struct ActiveMio<'a, 'r, 'm, Q, S, E> +pub struct ActiveMio<'a, 'r, 'm, Q, S> where Q: VirtQueue<'m>, { pub queues: &'a mut [Option>], pub irq_sender: &'a S, - pub ioeventfds: &'a [E], + pub notifiers: &'a [Notifier], pub poll: &'a mut Poll, pub mem: &'m Ram, } -impl<'m, D, Q, S, E> ActiveBackend for ActiveMio<'_, '_, 'm, Q, S, E> +impl<'m, D, Q, S> ActiveBackend for ActiveMio<'_, '_, 'm, Q, S> where D: VirtioMio, Q: VirtQueue<'m>, S: IrqSender, - E: IoeventFd, { type Event = Event; diff --git a/alioth/src/vm/vm.rs b/alioth/src/vm/vm.rs index 042ebd03..376ab718 100644 --- a/alioth/src/vm/vm.rs +++ b/alioth/src/vm/vm.rs @@ -14,6 +14,7 @@ #[cfg(target_os = "linux")] use std::collections::HashMap; +use std::io::ErrorKind; #[cfg(target_os = "linux")] use std::path::Path; use std::sync::Arc; @@ -47,7 +48,7 @@ use crate::device::pl031::Pl031; #[cfg(target_arch = "x86_64")] use crate::device::serial::Serial; use crate::errors::{DebugTrace, trace_error}; -use crate::hv::{Hypervisor, IoeventFdRegistry, Vm}; +use crate::hv::{self, Hypervisor, Vm}; use crate::loader::PayloadSpec; use crate::pci::pvpanic::PvPanic; use crate::pci::{Bdf, Pci}; @@ -133,10 +134,7 @@ where pub vfio_containers: Mutex, Arc>>, } -pub type VirtioPciDev = VirtioPciDevice< - <::Vm as Vm>::MsiSender, - <<::Vm as Vm>::IoeventFdRegistry as IoeventFdRegistry>::IoeventFd, ->; +pub type VirtioPciDev = VirtioPciDevice<<::Vm as Vm>::MsiSender>; impl Machine where @@ -295,7 +293,15 @@ where if let Some(callback) = dev.mem_change_callback() { self.ctx.board.memory.register_change_callback(callback)?; } - let registry = self.ctx.board.vm.create_ioeventfd_registry()?; + let registry = match self.ctx.board.vm.create_notifier_registry() { + Ok(registry) => Some(registry), + // The hypervisor does not support notifiers, fall back to VM exits. + Err(hv::Error::Notifier { error, .. }) if error.kind() == ErrorKind::Unsupported => { + log::debug!("{name}: notifiers are not supported"); + None + } + Err(e) => return Err(e.into()), + }; let virtio_dev = VirtioDevice::new( name.clone(), dev,