diff --git a/src/arch/aarch64/kernel/core_local.rs b/src/arch/aarch64/kernel/core_local.rs index a700972923..44d4dffede 100644 --- a/src/arch/aarch64/kernel/core_local.rs +++ b/src/arch/aarch64/kernel/core_local.rs @@ -1,7 +1,7 @@ use alloc::boxed::Box; use alloc::vec::Vec; use core::arch::asm; -use core::cell::{Cell, RefCell, RefMut}; +use core::cell::{RefCell, RefMut}; use core::ptr; use core::sync::atomic::Ordering; @@ -20,7 +20,7 @@ pub(crate) struct CoreLocal { /// ID of the current Core. core_id: CoreId, /// Scheduler of the current Core. - scheduler: Cell<*mut PerCoreScheduler>, + scheduler: RefCell>, /// Interface to the interrupt counters irq_statistics: &'static IrqStatistics, /// Queue of async tasks @@ -44,7 +44,7 @@ impl CoreLocal { let this = Self { this: ptr::null_mut(), core_id, - scheduler: Cell::new(ptr::null_mut()), + scheduler: RefCell::new(None), irq_statistics, async_tasks: RefCell::new(Vec::new()), #[cfg(feature = "smp")] @@ -91,17 +91,18 @@ pub(crate) fn core_id() -> CoreId { } } -#[inline] -pub(crate) fn core_scheduler() -> &'static mut PerCoreScheduler { - unsafe { &mut *CoreLocal::get().scheduler.get() } +pub(crate) fn core_scheduler() -> RefMut<'static, PerCoreScheduler> { + RefMut::map(CoreLocal::get().scheduler.borrow_mut(), |scheduler| { + scheduler.as_mut().unwrap() + }) } pub(crate) fn async_tasks() -> RefMut<'static, Vec> { CoreLocal::get().async_tasks.borrow_mut() } -pub(crate) fn set_core_scheduler(scheduler: *mut PerCoreScheduler) { - CoreLocal::get().scheduler.set(scheduler); +pub(crate) fn set_core_scheduler(scheduler: PerCoreScheduler) { + *CoreLocal::get().scheduler.borrow_mut() = Some(scheduler); } pub(crate) fn increment_irq_counter(irq_no: u8) { diff --git a/src/arch/x86_64/kernel/apic.rs b/src/arch/x86_64/kernel/apic.rs index fc8804b25a..1aab2ede7a 100644 --- a/src/arch/x86_64/kernel/apic.rs +++ b/src/arch/x86_64/kernel/apic.rs @@ -225,9 +225,11 @@ extern "x86-interrupt" fn spurious_interrupt_handler(stack_frame: interrupts::Ex #[cfg(feature = "smp")] extern "x86-interrupt" fn wakeup_handler(_stack_frame: interrupts::ExceptionStackFrame) { + use crate::scheduler::PerCoreSchedulerExt; + debug!("Received Wakeup Interrupt"); increment_irq_counter(WAKEUP_INTERRUPT_NUMBER); - let core_scheduler = core_scheduler(); + let mut core_scheduler = core_scheduler(); core_scheduler.check_input(); eoi(); if core_scheduler.is_scheduling() { diff --git a/src/arch/x86_64/kernel/core_local.rs b/src/arch/x86_64/kernel/core_local.rs index bf6210f5fa..50509a9e22 100644 --- a/src/arch/x86_64/kernel/core_local.rs +++ b/src/arch/x86_64/kernel/core_local.rs @@ -24,7 +24,7 @@ pub(crate) struct CoreLocal { /// Sequential ID of this CPU Core. core_id: CoreId, /// Scheduler for this CPU Core. - scheduler: Cell<*mut PerCoreScheduler>, + scheduler: RefCell>, /// Task State Segment (TSS) allocated for this CPU Core. pub tss: Cell<*mut TaskStateSegment>, /// start address of the kernel stack @@ -54,7 +54,7 @@ impl CoreLocal { let this = Self { this: ptr::null_mut(), core_id, - scheduler: Cell::new(ptr::null_mut()), + scheduler: RefCell::new(None), tss: Cell::new(ptr::null_mut()), kernel_stack: Cell::new(0), irq_statistics, @@ -101,16 +101,18 @@ pub(crate) fn core_id() -> CoreId { } } -pub(crate) fn core_scheduler() -> &'static mut PerCoreScheduler { - unsafe { &mut *CoreLocal::get().scheduler.get() } +pub(crate) fn core_scheduler() -> RefMut<'static, PerCoreScheduler> { + RefMut::map(CoreLocal::get().scheduler.borrow_mut(), |scheduler| { + scheduler.as_mut().unwrap() + }) } pub(crate) fn async_tasks() -> RefMut<'static, Vec> { CoreLocal::get().async_tasks.borrow_mut() } -pub(crate) fn set_core_scheduler(scheduler: *mut PerCoreScheduler) { - CoreLocal::get().scheduler.set(scheduler); +pub(crate) fn set_core_scheduler(scheduler: PerCoreScheduler) { + *CoreLocal::get().scheduler.borrow_mut() = Some(scheduler); } pub(crate) fn increment_irq_counter(irq_no: u8) { diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index 41d11e3954..10fb48708b 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -16,6 +16,7 @@ use crate::arch::x86_64::mm::{PhysAddr, VirtAddr}; use crate::config::*; use crate::kernel; use crate::scheduler::task::{Task, TaskFrame}; +use crate::scheduler::PerCoreSchedulerExt; #[repr(C, packed)] struct State { diff --git a/src/drivers/net/mod.rs b/src/drivers/net/mod.rs index 64fbe47305..1144912e99 100644 --- a/src/drivers/net/mod.rs +++ b/src/drivers/net/mod.rs @@ -70,6 +70,8 @@ pub(crate) fn network_irqhandler(_state: &State) -> bool { #[cfg(target_arch = "x86_64")] pub(crate) extern "x86-interrupt" fn network_irqhandler(_stack_frame: ExceptionStackFrame) { + use crate::scheduler::PerCoreSchedulerExt; + debug!("Receive network interrupt"); apic::eoi(); let _ = _irqhandler(); diff --git a/src/lib.rs b/src/lib.rs index 29cb0edf2b..0f87964021 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,6 +56,7 @@ use mm::allocator::LockedAllocator; pub(crate) use crate::arch::*; pub(crate) use crate::config::*; use crate::kernel::is_uhyve_with_pci; +use crate::scheduler::{PerCoreScheduler, PerCoreSchedulerExt}; pub use crate::syscalls::*; #[macro_use] @@ -342,9 +343,8 @@ fn boot_processor_main() -> ! { // Start the initd task. scheduler::PerCoreScheduler::spawn(initd, 0, scheduler::task::NORMAL_PRIO, 0, USER_STACK_SIZE); - let core_scheduler = core_scheduler(); // Run the scheduler loop. - core_scheduler.run(); + PerCoreScheduler::run(); } /// Entry Point of HermitCore for an Application Processor @@ -358,9 +358,8 @@ fn application_processor_main() -> ! { synch_all_cores(); crate::executor::init(); - let core_scheduler = core_scheduler(); // Run the scheduler loop. - core_scheduler.run(); + PerCoreScheduler::run(); } #[cfg(target_os = "none")] diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 03eae19910..20eb4d755e 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -1,9 +1,8 @@ -use alloc::boxed::Box; use alloc::collections::{BTreeMap, VecDeque}; use alloc::rc::Rc; #[cfg(feature = "smp")] use alloc::vec::Vec; -use core::cell::RefCell; +use core::cell::{RefCell, RefMut}; use core::sync::atomic::{AtomicU32, Ordering}; use crossbeam_utils::Backoff; @@ -76,6 +75,70 @@ pub struct PerCoreScheduler { blocked_tasks: BlockedTaskQueue, } +pub trait PerCoreSchedulerExt { + /// Triggers the scheduler to reschedule the tasks. + /// Interrupt flag will be cleared during the reschedule + fn reschedule(self); +} + +impl PerCoreSchedulerExt for RefMut<'_, PerCoreScheduler> { + #[cfg(target_arch = "x86_64")] + fn reschedule(mut self) { + without_interrupts(|| { + if let Some(last_stack_pointer) = self.scheduler() { + let (new_stack_pointer, is_idle) = { + let borrowed = self.current_task.borrow(); + ( + borrowed.last_stack_pointer, + borrowed.status == TaskStatus::Idle, + ) + }; + + if is_idle || Rc::ptr_eq(&self.current_task, &self.fpu_owner) { + drop(self); + unsafe { + switch_to_fpu_owner(last_stack_pointer, new_stack_pointer.as_usize()); + } + } else { + drop(self); + unsafe { + switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); + } + } + } + }) + } + + /// Trigger an interrupt to reschedule the system + #[cfg(target_arch = "aarch64")] + fn reschedule(self) { + use core::arch::asm; + + use arm_gic::gicv3::{GicV3, IntId, SgiTarget}; + + use crate::interrupts::SGI_RESCHED; + + unsafe { + asm!("dsb nsh", "isb", options(nostack, nomem, preserves_flags)); + } + + let reschedid = IntId::sgi(SGI_RESCHED.into()); + GicV3::send_sgi( + reschedid, + SgiTarget::List { + affinity3: 0, + affinity2: 0, + affinity1: 0, + target_list: 0b1, + }, + ); + + drop(self); + + interrupts::enable(); + } +} + struct NewTask { tid: TaskId, func: extern "C" fn(usize), @@ -170,10 +233,12 @@ impl PerCoreScheduler { } /// Terminate the current task on the current core. - pub fn exit(&mut self, exit_code: i32) -> ! { + pub fn exit(exit_code: i32) -> ! { + let mut core_scheduler = core_scheduler(); + without_interrupts(|| { // Get the current task. - let mut current_task_borrowed = self.current_task.borrow_mut(); + let mut current_task_borrowed = (*core_scheduler.current_task).borrow_mut(); assert_ne!( current_task_borrowed.status, TaskStatus::Idle, @@ -194,12 +259,12 @@ impl PerCoreScheduler { // wakeup tasks, which are waiting for task with the identifier id if let Some(mut queue) = WAITING_TASKS.lock().remove(¤t_id) { while let Some(task) = queue.pop_front() { - self.custom_wakeup(task); + core_scheduler.custom_wakeup(task); } } }); - self.reschedule(); + core_scheduler.reschedule(); unreachable!() } @@ -397,7 +462,7 @@ impl PerCoreScheduler { pub fn set_current_task_priority(&mut self, prio: Priority) { without_interrupts(|| { trace!("Change priority of the current task"); - self.current_task.borrow_mut().prio = prio; + (*self.current_task).borrow_mut().prio = prio; }); } @@ -414,7 +479,7 @@ impl PerCoreScheduler { if other_core { warn!("Have to change the priority on another core"); } else if self.current_task.borrow().id == task.get_id() { - self.current_task.borrow_mut().prio = prio; + (*self.current_task).borrow_mut().prio = prio; } else { self.ready_queue .set_priority(task, prio) @@ -436,7 +501,7 @@ impl PerCoreScheduler { self.current_task.borrow().id ); - self.fpu_owner.borrow_mut().last_fpu_state.save(); + (*self.fpu_owner).borrow_mut().last_fpu_state.save(); self.current_task.borrow().last_fpu_state.restore(); self.fpu_owner = self.current_task.clone(); } @@ -465,76 +530,24 @@ impl PerCoreScheduler { } } - /// Triggers the scheduler to reschedule the tasks. - /// Interrupt flag will be cleared during the reschedule - #[cfg(target_arch = "x86_64")] - pub fn reschedule(&mut self) { - without_interrupts(|| { - if let Some(last_stack_pointer) = self.scheduler() { - let (new_stack_pointer, is_idle) = { - let borrowed = self.current_task.borrow(); - ( - borrowed.last_stack_pointer, - borrowed.status == TaskStatus::Idle, - ) - }; - - if is_idle || Rc::ptr_eq(&self.current_task, &self.fpu_owner) { - unsafe { - switch_to_fpu_owner(last_stack_pointer, new_stack_pointer.as_usize()); - } - } else { - unsafe { - switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); - } - } - } - }) - } - - /// Trigger an interrupt to reschedule the system - #[cfg(target_arch = "aarch64")] - pub fn reschedule(&self) { - use core::arch::asm; - - use arm_gic::gicv3::{GicV3, IntId, SgiTarget}; - - use crate::interrupts::SGI_RESCHED; - - unsafe { - asm!("dsb nsh", "isb", options(nostack, nomem, preserves_flags)); - } - - let reschedid = IntId::sgi(SGI_RESCHED.into()); - GicV3::send_sgi( - reschedid, - SgiTarget::List { - affinity3: 0, - affinity2: 0, - affinity1: 0, - target_list: 0b1, - }, - ); - - interrupts::enable(); - } - /// Only the idle task should call this function. /// Set the idle task to halt state if not another /// available. - pub fn run(&mut self) -> ! { + pub fn run() -> ! { let backoff = Backoff::new(); loop { + let mut core_scheduler = core_scheduler(); interrupts::disable(); // run async tasks crate::executor::run(); // do housekeeping - self.cleanup_tasks(); + core_scheduler.cleanup_tasks(); - if self.ready_queue.is_empty() { + if core_scheduler.ready_queue.is_empty() { + drop(core_scheduler); if backoff.is_completed() { interrupts::enable_and_wait(); } else { @@ -543,7 +556,7 @@ impl PerCoreScheduler { } } else { interrupts::enable(); - self.reschedule(); + core_scheduler.reschedule(); backoff.reset(); } } @@ -567,7 +580,7 @@ impl PerCoreScheduler { // Get information about the current task. let (id, last_stack_pointer, prio, status) = { - let mut borrowed = self.current_task.borrow_mut(); + let mut borrowed = (*self.current_task).borrow_mut(); ( borrowed.id, &mut borrowed.last_stack_pointer as *mut _ as *mut usize, @@ -587,7 +600,7 @@ impl PerCoreScheduler { } else { if status == TaskStatus::Finished { // Mark the finished task as invalid and add it to the finished tasks for a later cleanup. - self.current_task.borrow_mut().status = TaskStatus::Invalid; + (*self.current_task).borrow_mut().status = TaskStatus::Invalid; self.finished_tasks.push_back(self.current_task.clone()); } @@ -610,13 +623,13 @@ impl PerCoreScheduler { // Handle the current task. if status == TaskStatus::Running { // Mark the running task as ready again and add it back to the queue. - self.current_task.borrow_mut().status = TaskStatus::Ready; + (*self.current_task).borrow_mut().status = TaskStatus::Ready; self.ready_queue.push(self.current_task.clone()); } // Handle the new task and get information about it. let (new_id, new_stack_pointer) = { - let mut borrowed = task.borrow_mut(); + let mut borrowed = (*task).borrow_mut(); if borrowed.status != TaskStatus::Idle { // Mark the new task as running. borrowed.status = TaskStatus::Running; @@ -659,7 +672,7 @@ fn get_tid() -> TaskId { #[inline] pub fn abort() -> ! { - core_scheduler().exit(-1) + PerCoreScheduler::exit(-1) } /// Add a per-core scheduler for the current core. @@ -685,7 +698,7 @@ pub fn add_current_core() { "Initializing scheduler for core {} with idle task {}", core_id, tid ); - let boxed_scheduler = Box::new(PerCoreScheduler { + set_core_scheduler(PerCoreScheduler { #[cfg(feature = "smp")] core_id, current_task: idle_task.clone(), @@ -696,9 +709,6 @@ pub fn add_current_core() { finished_tasks: VecDeque::new(), blocked_tasks: BlockedTaskQueue::new(), }); - - let scheduler = Box::into_raw(boxed_scheduler); - set_core_scheduler(scheduler); #[cfg(feature = "smp")] { SCHEDULER_INPUTS.lock().insert( @@ -715,11 +725,9 @@ fn get_scheduler_input(core_id: CoreId) -> &'static InterruptTicketMutex Result<(), ()> { - let core_scheduler = core_scheduler(); - debug!( "Task {} is waiting for task {}", - core_scheduler.get_current_task_id(), + core_scheduler().get_current_task_id(), id ); @@ -727,6 +735,7 @@ pub fn join(id: TaskId) -> Result<(), ()> { let mut waiting_tasks_guard = WAITING_TASKS.lock(); if let Some(queue) = waiting_tasks_guard.get_mut(&id) { + let mut core_scheduler = core_scheduler(); queue.push_back(core_scheduler.get_current_task_handle()); core_scheduler.block_current_task(None); diff --git a/src/synch/futex.rs b/src/synch/futex.rs index db1196d281..a22d5a4ebe 100644 --- a/src/synch/futex.rs +++ b/src/synch/futex.rs @@ -10,6 +10,7 @@ use crate::arch::kernel::core_local::core_scheduler; use crate::arch::kernel::processor::get_timer_ticks; use crate::errno::{EAGAIN, EINVAL, ETIMEDOUT}; use crate::scheduler::task::TaskHandlePriorityQueue; +use crate::scheduler::PerCoreSchedulerExt; // TODO: Replace with a concurrent hashmap. static PARKING_LOT: InterruptTicketMutex> = @@ -51,14 +52,15 @@ pub(crate) fn futex_wait( timeout }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); scheduler.block_current_task(wakeup_time); let handle = scheduler.get_current_task_handle(); parking_lot.entry(addr(address)).or_default().push(handle); drop(parking_lot); + drop(scheduler); loop { - scheduler.reschedule(); + core_scheduler().reschedule(); let mut parking_lot = PARKING_LOT.lock(); if matches!(wakeup_time, Some(t) if t <= get_timer_ticks()) { @@ -87,7 +89,7 @@ pub(crate) fn futex_wait( } else { // A spurious wakeup occurred, sleep again. // Tasks do not change core, so the handle in the parking lot is still current. - scheduler.block_current_task(wakeup_time); + core_scheduler().block_current_task(wakeup_time); } } drop(parking_lot); @@ -120,14 +122,15 @@ pub(crate) fn futex_wait_and_set( timeout }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); scheduler.block_current_task(wakeup_time); let handle = scheduler.get_current_task_handle(); parking_lot.entry(addr(address)).or_default().push(handle); drop(parking_lot); + drop(scheduler); loop { - scheduler.reschedule(); + core_scheduler().reschedule(); let mut parking_lot = PARKING_LOT.lock(); if matches!(wakeup_time, Some(t) if t <= get_timer_ticks()) { @@ -156,7 +159,7 @@ pub(crate) fn futex_wait_and_set( } else { // A spurious wakeup occurred, sleep again. // Tasks do not change core, so the handle in the parking lot is still current. - scheduler.block_current_task(wakeup_time); + core_scheduler().block_current_task(wakeup_time); } } drop(parking_lot); @@ -177,7 +180,7 @@ pub(crate) fn futex_wake(address: &AtomicU32, count: i32) -> i32 { Entry::Vacant(_) => return 0, }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); let mut woken = 0; while woken != count || count == i32::MAX { match queue.get_mut().pop() { @@ -212,7 +215,7 @@ pub(crate) fn futex_wake_or_set(address: &AtomicU32, count: i32, new_value: u32) } }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); let mut woken = 0; while woken != count || count == i32::MAX { match queue.get_mut().pop() { diff --git a/src/synch/recmutex.rs b/src/synch/recmutex.rs index 885fa9fdd1..03520a97f5 100644 --- a/src/synch/recmutex.rs +++ b/src/synch/recmutex.rs @@ -2,6 +2,7 @@ use hermit_sync::TicketMutex; use crate::arch::core_local::*; use crate::scheduler::task::{TaskHandlePriorityQueue, TaskId}; +use crate::scheduler::PerCoreSchedulerExt; struct RecursiveMutexState { current_tid: Option, @@ -26,10 +27,10 @@ impl RecursiveMutex { pub fn acquire(&self) { // Get information about the current task. - let core_scheduler = core_scheduler(); - let tid = core_scheduler.get_current_task_id(); + let tid = core_scheduler().get_current_task_id(); loop { + let mut core_scheduler = core_scheduler(); { let mut locked_state = self.state.lock(); diff --git a/src/synch/semaphore.rs b/src/synch/semaphore.rs index 2bf81ec22a..b0bb5558d9 100644 --- a/src/synch/semaphore.rs +++ b/src/synch/semaphore.rs @@ -4,6 +4,7 @@ use hermit_sync::InterruptTicketMutex; use crate::arch::core_local::*; use crate::scheduler::task::TaskHandlePriorityQueue; +use crate::scheduler::PerCoreSchedulerExt; struct SemaphoreState { /// Resource available count @@ -67,12 +68,12 @@ impl Semaphore { pub fn acquire(&self, time: Option) -> bool { #[cfg(feature = "smp")] let backoff = Backoff::new(); - let core_scheduler = core_scheduler(); let wakeup_time = time.map(|ms| crate::arch::processor::get_timer_ticks() + ms * 1000); // Loop until we have acquired the semaphore. loop { + let mut core_scheduler = core_scheduler(); let mut locked_state = self.state.lock(); if locked_state.count > 0 { diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index 8edbac4880..138f8772bd 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -14,6 +14,7 @@ use crate::errno::*; #[cfg(feature = "newlib")] use crate::mm::{task_heap_end, task_heap_start}; use crate::scheduler::task::{Priority, TaskHandle, TaskId}; +use crate::scheduler::{PerCoreScheduler, PerCoreSchedulerExt}; use crate::syscalls::timer::timespec; use crate::{arch, scheduler, syscalls}; @@ -65,7 +66,7 @@ pub extern "C" fn sys_exit(arg: i32) -> ! { extern "C" fn __sys_thread_exit(arg: i32) -> ! { debug!("Exit thread with error code {}!", arg); - core_scheduler().exit(arg) + PerCoreScheduler::exit(arg) } #[no_mangle] @@ -115,7 +116,7 @@ pub(crate) extern "C" fn __sys_usleep(usecs: u64) { // Enough time to set a wakeup timer and block the current task. debug!("sys_usleep blocking the task for {} microseconds", usecs); let wakeup_time = arch::processor::get_timer_ticks() + usecs; - let core_scheduler = core_scheduler(); + let mut core_scheduler = core_scheduler(); core_scheduler.block_current_task(Some(wakeup_time)); // Switch to the next task. @@ -299,7 +300,7 @@ static BLOCKED_TASKS: InterruptTicketMutex> = extern "C" fn __sys_block_current_task(timeout: &Option) { let wakeup_time = timeout.map(|t| arch::processor::get_timer_ticks() + t * 1000); - let core_scheduler = core_scheduler(); + let mut core_scheduler = core_scheduler(); let handle = core_scheduler.get_current_task_handle(); let tid = core_scheduler.get_current_task_id();