From 582f75b64993d50a398140575b89b0ef4b54ad1b Mon Sep 17 00:00:00 2001 From: wuzheng Date: Fri, 17 May 2024 14:26:38 +0800 Subject: [PATCH] implement pagefault and tlb_flush for x86_64 to support mmap. --- Cargo.lock | 2 + api/ruxos_posix_api/src/imp/mmap/mod.rs | 3 +- api/ruxos_posix_api/src/imp/mmap/trap.rs | 2 +- crates/allocator/Cargo.toml | 1 + crates/scheduler/Cargo.toml | 1 + modules/ruxhal/src/arch/aarch64/trap.rs | 12 ++- modules/ruxhal/src/arch/x86_64/mod.rs | 96 +++++++++++++++++++++- modules/ruxhal/src/arch/x86_64/trap.rs | 30 ++++++- modules/ruxhal/src/platform/x86_pc/apic.rs | 18 +++- scripts/make/build_c.mk | 1 + scripts/make/build_musl.mk | 1 + scripts/make/features.mk | 4 +- ulib/ruxlibc/Cargo.toml | 5 ++ 13 files changed, 157 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b6fae1c2..3da09f1ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,7 @@ dependencies = [ "bitmap-allocator", "buddy_system_allocator", "criterion", + "log", "rand", "rlsf", "slab_allocator", @@ -1773,6 +1774,7 @@ name = "scheduler" version = "0.1.0" dependencies = [ "linked_list", + "log", ] [[package]] diff --git a/api/ruxos_posix_api/src/imp/mmap/mod.rs b/api/ruxos_posix_api/src/imp/mmap/mod.rs index b18d2c8e4..0ab39c2ed 100644 --- a/api/ruxos_posix_api/src/imp/mmap/mod.rs +++ b/api/ruxos_posix_api/src/imp/mmap/mod.rs @@ -8,7 +8,8 @@ */ cfg_if::cfg_if! { - if #[cfg(all(feature = "paging", target_arch = "aarch64"))] { + // for X86_64 with SMP, it must flush TLB via IPI + if #[cfg( all(feature = "paging", any(target_arch = "aarch64", any( all(target_arch = "x86_64", feature = "irq", feature = "smp"), all(target_arch = "x86_64", not(feature = "smp")) ) ) ))] { #[macro_use] mod utils; mod api; diff --git a/api/ruxos_posix_api/src/imp/mmap/trap.rs b/api/ruxos_posix_api/src/imp/mmap/trap.rs index 5b0076be6..38cdee849 100644 --- a/api/ruxos_posix_api/src/imp/mmap/trap.rs +++ b/api/ruxos_posix_api/src/imp/mmap/trap.rs @@ -89,7 +89,7 @@ impl ruxhal::trap::TrapHandler for TrapHandlerImpl { preload_page_with_swap(&mut memory_map, &mut swaped_map, &mut off_pool); // Fill target data to assigned physical addresses, from file or zero according to mapping type - let dst = fake_vaddr.as_mut_ptr(); + let dst: *mut u8 = fake_vaddr.as_mut_ptr(); #[cfg(feature = "fs")] { if let Some(off) = swaped_map.remove(&vaddr) { diff --git a/crates/allocator/Cargo.toml b/crates/allocator/Cargo.toml index b8770f124..5c458461a 100644 --- a/crates/allocator/Cargo.toml +++ b/crates/allocator/Cargo.toml @@ -22,6 +22,7 @@ buddy = ["dep:buddy_system_allocator"] allocator_api = [] [dependencies] +log = "0.4" buddy_system_allocator = { version = "0.9", default-features = false, optional = true } slab_allocator = { path = "../slab_allocator", optional = true } rlsf = { version = "0.2", optional = true } diff --git a/crates/scheduler/Cargo.toml b/crates/scheduler/Cargo.toml index e6a26af2c..0b7dbaf4d 100644 --- a/crates/scheduler/Cargo.toml +++ b/crates/scheduler/Cargo.toml @@ -11,3 +11,4 @@ documentation = "https://rcore-os.github.io/arceos/scheduler/index.html" [dependencies] linked_list = { path = "../linked_list" } +log = "0.4" diff --git a/modules/ruxhal/src/arch/aarch64/trap.rs b/modules/ruxhal/src/arch/aarch64/trap.rs index 42597ffbc..e81cdff95 100644 --- a/modules/ruxhal/src/arch/aarch64/trap.rs +++ b/modules/ruxhal/src/arch/aarch64/trap.rs @@ -96,18 +96,16 @@ fn handle_sync_exception(tf: &mut TrapFrame) { // this cause is coded like linux. let cause: PageFaultCause = match esr.read_as_enum(ESR_EL1::EC) { - Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => { - if iss & 0x40 != 0 { - PageFaultCause::WRITE // = store - } else { - PageFaultCause::READ // = load - } + Some(ESR_EL1::EC::Value::DataAbortCurrentEL) if iss & 0x40 != 0 => { + PageFaultCause::WRITE // = store + } + Some(ESR_EL1::EC::Value::DataAbortCurrentEL) if iss & 0x40 == 0 => { + PageFaultCause::READ // = load } _ => { PageFaultCause::INSTRUCTION // = instruction fetch } }; - debug!("mapped vaddr in Page Fault: {:X} {:?}", vaddr, cause); if crate::trap::handle_page_fault(vaddr, cause) { return; } diff --git a/modules/ruxhal/src/arch/x86_64/mod.rs b/modules/ruxhal/src/arch/x86_64/mod.rs index 9dc142381..973ff5cf6 100644 --- a/modules/ruxhal/src/arch/x86_64/mod.rs +++ b/modules/ruxhal/src/arch/x86_64/mod.rs @@ -11,7 +11,6 @@ mod context; mod gdt; mod idt; -#[cfg(target_os = "none")] mod trap; use core::arch::asm; @@ -23,6 +22,20 @@ use x86_64::instructions::interrupts; #[cfg(feature = "musl")] use x86_64::registers::model_specific::EferFlags; +#[cfg(feature = "irq")] +extern crate alloc; + +#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))] +use { + crate::{ + cpu::this_cpu_id, + platform::irq::{end_of_interrupt, send_ipi_excluding_self}, + }, + alloc::vec::Vec, + ruxconfig::SMP, + spinlock::SpinRaw, +}; + pub use self::context::{ExtendedState, FxsaveArea, TaskContext, TrapFrame}; pub use self::gdt::GdtStruct; pub use self::idt::IdtStruct; @@ -88,6 +101,27 @@ pub unsafe fn write_page_table_root(root_paddr: PhysAddr) { } } +/// data structure for communicating between IPI for the TLB flushing. +/// +/// `Vaddr` means flushing the TLB entry that maps the given virtual address. +/// `All` means flushing the entire TLB. +#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))] +enum FlushTlbIpiData { + Vaddr(usize), + All, +} + +// const variable for every CPU's flushing addresses. +#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))] +#[allow(clippy::declare_interior_mutable_const)] +const FLUSH_QUEUES: SpinRaw> = SpinRaw::new(Vec::new()); +/// static variable for communicating between IPI for the TLB flushing. +#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))] +static FLUSHING_ADDRESSES: [SpinRaw>; SMP] = [FLUSH_QUEUES; SMP]; +/// const irq vector for TLB flushing IPI. +#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))] +pub const INVALID_TLB_VECTOR: u8 = 0xff; // SPURIOUS APIC INTERRUPT + /// Flushes the TLB. /// /// If `vaddr` is [`None`], flushes the entire TLB. Otherwise, flushes the TLB @@ -95,10 +129,66 @@ pub unsafe fn write_page_table_root(root_paddr: PhysAddr) { #[inline] pub fn flush_tlb(vaddr: Option) { if let Some(vaddr) = vaddr { - unsafe { tlb::flush(vaddr.into()) } + trace!("flush TLB entry: {:#x}", vaddr); + unsafe { + tlb::flush(vaddr.into()); + } + #[cfg(all(feature = "irq", feature = "paging", feature = "smp"))] + { + for (i, flushing_vec) in FLUSHING_ADDRESSES.iter().enumerate().take(SMP) { + if i != this_cpu_id() { + flushing_vec + .lock() + .push(FlushTlbIpiData::Vaddr(vaddr.into())); + } + } + unsafe { + send_ipi_excluding_self(INVALID_TLB_VECTOR); + } + } } else { - unsafe { tlb::flush_all() } + trace!("flush all TLB entry"); + unsafe { + tlb::flush_all(); + } + #[cfg(all(feature = "irq", feature = "paging", feature = "smp"))] + { + for (i, flushing_vec) in FLUSHING_ADDRESSES.iter().enumerate().take(SMP) { + if i != this_cpu_id() { + let mut flushing_addresses = flushing_vec.lock(); + // clear all flushing addresses to avoid flushing again in IPI handler. + flushing_addresses.clear(); + flushing_addresses.push(FlushTlbIpiData::All); + } + } + unsafe { + send_ipi_excluding_self(INVALID_TLB_VECTOR); + } + } + } +} + +/// Flushes the TLB in IPI handler. +/// +/// This function is called in IPI handler, and it flushes the TLB entry that maps the given virtual address. +#[inline] +#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))] +pub(crate) fn flush_tlb_ipi_handler() { + // error!("flush TLB entry in IPI handler"); + let guard = kernel_guard::NoPreempt::new(); + unsafe { + let mut flushing_addresses = FLUSHING_ADDRESSES[this_cpu_id()].lock(); + while let Some(flush_data) = flushing_addresses.pop() { + if let FlushTlbIpiData::Vaddr(vaddr) = flush_data { + tlb::flush(vaddr); + } else { + tlb::flush_all(); + flushing_addresses.clear(); + } + } + end_of_interrupt(); } + drop(guard); } /// Reads the thread pointer of the current CPU. diff --git a/modules/ruxhal/src/arch/x86_64/trap.rs b/modules/ruxhal/src/arch/x86_64/trap.rs index dfeabceff..0cbf12256 100644 --- a/modules/ruxhal/src/arch/x86_64/trap.rs +++ b/modules/ruxhal/src/arch/x86_64/trap.rs @@ -10,6 +10,13 @@ use x86::{controlregs::cr2, irq::*}; use super::context::TrapFrame; +#[cfg(all(feature = "paging", feature = "irq", feature = "smp"))] +use crate::arch::{flush_tlb_ipi_handler, INVALID_TLB_VECTOR}; +#[cfg(any( + all(feature = "paging", feature = "irq", feature = "smp"), + all(feature = "paging", not(feature = "smp")) +))] +use crate::trap::PageFaultCause; core::arch::global_asm!(include_str!("trap.S")); @@ -28,12 +35,25 @@ fn x86_trap_handler(tf: &TrapFrame) { tf.error_code, ); } else { + let vaddr = unsafe { cr2() }; + #[cfg(any( + all(feature = "paging", feature = "irq", feature = "smp"), + all(feature = "paging", not(feature = "smp")) + ))] + { + // this cause is coded like linux. + let cause: PageFaultCause = match tf.error_code { + x if x & 0x10 != 0 => PageFaultCause::INSTRUCTION, + x if x & 0x02 != 0 => PageFaultCause::WRITE, + _ => PageFaultCause::READ, + }; + if crate::trap::handle_page_fault(vaddr, cause) { + return; + } + } panic!( "Kernel #PF @ {:#x}, fault_vaddr={:#x}, error_code={:#x}:\n{:#x?}", - tf.rip, - unsafe { cr2() }, - tf.error_code, - tf, + tf.rip, vaddr, tf.error_code, tf, ); } } @@ -44,6 +64,8 @@ fn x86_trap_handler(tf: &TrapFrame) { tf.rip, tf.error_code, tf ); } + #[cfg(all(feature = "paging", feature = "irq", feature = "smp"))] + INVALID_TLB_VECTOR => flush_tlb_ipi_handler(), IRQ_VECTOR_START..=IRQ_VECTOR_END => crate::trap::handle_irq_extern(tf.vector as _), _ => { panic!( diff --git a/modules/ruxhal/src/platform/x86_pc/apic.rs b/modules/ruxhal/src/platform/x86_pc/apic.rs index d0b40fd86..d369e3d62 100644 --- a/modules/ruxhal/src/platform/x86_pc/apic.rs +++ b/modules/ruxhal/src/platform/x86_pc/apic.rs @@ -13,6 +13,8 @@ use lazy_init::LazyInit; use memory_addr::PhysAddr; use spinlock::SpinNoIrq; use x2apic::ioapic::IoApic; +#[cfg(feature = "irq")] +use x2apic::lapic::IpiAllShorthand; use x2apic::lapic::{xapic_base, LocalApic, LocalApicBuilder}; use x86_64::instructions::port::Port; @@ -71,8 +73,22 @@ pub fn dispatch_irq(vector: usize) { crate::irq::dispatch_irq_common(vector); unsafe { local_apic().end_of_interrupt() }; } +#[cfg(feature = "irq")] +pub(crate) unsafe fn end_of_interrupt() { + local_apic().end_of_interrupt() +} + +#[cfg(feature = "irq")] +pub(crate) unsafe fn send_ipi_excluding_self(vector: u8) { + local_apic().send_ipi_all(vector, IpiAllShorthand::AllExcludingSelf); +} + +#[cfg(feature = "irq")] +pub(crate) unsafe fn send_ipi_including_self(vector: u8) { + local_apic().send_ipi_all(vector, IpiAllShorthand::AllIncludingSelf); +} -pub(super) fn local_apic<'a>() -> &'a mut LocalApic { +pub fn local_apic<'a>() -> &'a mut LocalApic { // It's safe as LAPIC is per-cpu. unsafe { LOCAL_APIC.as_mut().unwrap() } } diff --git a/scripts/make/build_c.mk b/scripts/make/build_c.mk index e2b45fcec..d1522f98a 100644 --- a/scripts/make/build_c.mk +++ b/scripts/make/build_c.mk @@ -29,6 +29,7 @@ endif ifeq ($(ARCH), x86_64) LDFLAGS += --no-relax + CFLAGS += -mno-red-zone else ifeq ($(ARCH), riscv64) CFLAGS += -march=rv64gc -mabi=lp64d -mcmodel=medany endif diff --git a/scripts/make/build_musl.mk b/scripts/make/build_musl.mk index f0a474461..0cbc29c77 100644 --- a/scripts/make/build_musl.mk +++ b/scripts/make/build_musl.mk @@ -29,6 +29,7 @@ endif ifeq ($(ARCH), x86_64) LDFLAGS += --no-relax + CFLAGS += -mno-red-zone else ifeq ($(ARCH), riscv64) CFLAGS += -march=rv64gc -mabi=lp64d -mcmodel=medany endif diff --git a/scripts/make/features.mk b/scripts/make/features.mk index 6c605bb9f..05e944b56 100644 --- a/scripts/make/features.mk +++ b/scripts/make/features.mk @@ -18,7 +18,7 @@ ifeq ($(APP_TYPE),c) else lib_feat_prefix := ruxlibc/ endif - lib_features := fp_simd alloc paging multitask fs net fd pipe select poll epoll random-hw signal + lib_features := fp_simd alloc irq sched_rr paging multitask fs net fd pipe select poll epoll random-hw signal else # TODO: it's better to use `ruxfeat/` as `ax_feat_prefix`, but all apps need to have `ruxfeat` as a dependency ax_feat_prefix := axstd/ @@ -27,7 +27,7 @@ else endif ifeq ($(APP_TYPE),c) ifeq ($(MUSL), y) - lib_features += irq musl sched_rr + lib_features += musl endif endif diff --git a/ulib/ruxlibc/Cargo.toml b/ulib/ruxlibc/Cargo.toml index 196ed6cdf..8b00b811f 100644 --- a/ulib/ruxlibc/Cargo.toml +++ b/ulib/ruxlibc/Cargo.toml @@ -52,6 +52,11 @@ poll = ["ruxos_posix_api/poll"] epoll = ["ruxos_posix_api/epoll"] random-hw = ["ruxos_posix_api/random-hw"] +# Interrupts +irq = ["ruxos_posix_api/irq", "ruxfeat/irq"] + +sched_rr = ["irq", "ruxfeat/sched_rr"] + [dependencies] ruxfeat = { path = "../../api/ruxfeat" } ruxos_posix_api = { path = "../../api/ruxos_posix_api" }