Merge pull request #2596 from hermit-os/arch-start

refactor(arch): extract hermit-entry-specific start code

Author: mkroening

src/arch/aarch64/kernel/mod.rs

@@ -10,8 +10,6 @@ pub mod pci;
 pub mod processor;
 pub mod scheduler;
 pub mod serial;
-#[cfg(target_os = "none")]
-mod start;
 pub mod systemtime;
 
 use alloc::alloc::alloc;
@@ -112,7 +110,7 @@ pub fn boot_next_processor() {
 
 		use memory_addresses::VirtAddr;
 
-		use crate::arch::aarch64::kernel::start::{TTBR0, smp_start};
+		use crate::arch::aarch64::start::smp::{TTBR0, smp_start};
 		use crate::mm::virtual_to_physical;
 
 		if cpu_online == 0 {

src/arch/aarch64/mod.rs

@@ -1,2 +1,4 @@
 pub mod kernel;
 pub mod mm;
+#[cfg(target_os = "none")]
+pub mod start;

src/arch/aarch64/start/hermit_entry.rs

@@ -0,0 +1,111 @@
+use core::arch::{asm, naked_asm};
+
+use aarch64_cpu::asm::barrier::{SY, dsb};
+use hermit_entry::Entry;
+use hermit_entry::boot_info::RawBootInfo;
+
+use crate::config::KERNEL_STACK_SIZE;
+use crate::env;
+use crate::kernel::scheduler::TaskStacks;
+use crate::kernel::{CPU_ONLINE, CURRENT_STACK_ADDRESS};
+
+/// Entrypoint - Initialize Stack pointer and Exception Table
+#[unsafe(no_mangle)]
+#[unsafe(naked)]
+pub unsafe extern "C" fn _start(boot_info: Option<&'static RawBootInfo>, cpu_id: u32) -> ! {
+	// validate signatures
+	// `_Start` is compatible to `Entry`
+	{
+		unsafe extern "C" fn _entry(_boot_info: &'static RawBootInfo, _cpu_id: u32) -> ! {
+			unreachable!()
+		}
+		pub type _Start =
+			unsafe extern "C" fn(boot_info: Option<&'static RawBootInfo>, cpu_id: u32) -> !;
+		const _ENTRY: Entry = _entry;
+		const _START: _Start = _start;
+		const _PRE_INIT: _Start = pre_init;
+	}
+
+	naked_asm!(
+		// use core::sync::atomic::{AtomicU32, Ordering};
+		//
+		// pub static CPU_ONLINE: AtomicU32 = AtomicU32::new(0);
+		//
+		// while CPU_ONLINE.load(Ordering::Acquire) != this {
+		//     core::hint::spin_loop();
+		// }
+		"mrs x4, mpidr_el1",
+		"and x4, x4, #0xff",
+		"1:",
+		"adrp x8, {cpu_online}",
+		"ldr x5, [x8, #:lo12:{cpu_online}]",
+		"cmp x4, x5",
+		"b.eq 2f",
+		"b 1b",
+		"2:",
+
+		// we want to use sp_el1
+		"msr spsel, #1",
+
+		// Overwrite RSP if `CURRENT_STACK_ADDRESS != 0`
+		"adrp x8, {current_stack_address}",
+		"ldr x4, [x8, #:lo12:{current_stack_address}]",
+		"cmp x4, 0",
+		"b.eq 3f",
+		"mov sp, x4",
+		"b 4f",
+		"3:",
+		"mov x4, sp",
+		"4:",
+		"str x4, [x8, #:lo12:{current_stack_address}]",
+
+		// Add stack top offset
+		"mov x8, {stack_top_offset}",
+		"add sp, sp, x8",
+
+		// Jump to Rust code
+		"b {pre_init}",
+
+		cpu_online = sym CPU_ONLINE,
+		stack_top_offset = const KERNEL_STACK_SIZE - TaskStacks::MARKER_SIZE,
+		current_stack_address = sym CURRENT_STACK_ADDRESS,
+		pre_init = sym pre_init,
+	)
+}
+
+#[inline(never)]
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn pre_init(boot_info: Option<&'static RawBootInfo>, cpu_id: u32) -> ! {
+	// set exception table
+	unsafe {
+		asm!(
+			"adrp x4, vector_table",
+			"add x4, x4, #:lo12:vector_table",
+			"msr vbar_el1, x4",
+			out("x4") _,
+			options(nostack),
+		);
+	}
+
+	// Memory barrier
+	dsb(SY);
+
+	if cpu_id == 0 {
+		env::set_boot_info(*boot_info.unwrap());
+		crate::boot_processor_main()
+	} else {
+		#[cfg(not(feature = "smp"))]
+		{
+			let style = anstyle::Style::new().fg_color(Some(anstyle::AnsiColor::Red.into()));
+			let preamble = format_args!("[            ][{cpu_id}][{style}ERROR{style:#}]");
+			println!(
+				"{preamble} Secondary core booted, but Hermit was not built with SMP support!"
+			);
+			loop {
+				crate::arch::kernel::processor::halt();
+			}
+		}
+		#[cfg(feature = "smp")]
+		crate::application_processor_main()
+	}
+}

src/arch/aarch64/start/mod.rs

@@ -0,0 +1,3 @@
+pub mod hermit_entry;
+#[cfg(feature = "smp")]
+pub mod smp;

src/arch/aarch64/kernel/start.rs src/arch/aarch64/start/smp.rssrc/arch/aarch64/kernel/start.rs renamed to src/arch/aarch64/start/smp.rs

@@ -1,18 +1,7 @@
-#![allow(dead_code)]
-
-use core::arch::{asm, naked_asm};
-#[cfg(feature = "smp")]
 use core::ptr;
-#[cfg(feature = "smp")]
 use core::sync::atomic::AtomicPtr;
 
-use aarch64_cpu::asm::barrier::{SY, dsb};
-use hermit_entry::Entry;
-use hermit_entry::boot_info::RawBootInfo;
-
-use crate::arch::aarch64::kernel::scheduler::TaskStacks;
-use crate::config::KERNEL_STACK_SIZE;
-use crate::env;
+use crate::kernel::CURRENT_STACK_ADDRESS;
 
 /*
  * Memory types available.
@@ -31,100 +20,35 @@ const MT_NORMAL: u64 = 4;
 const TCR_IRGN_WBWA: u64 = ((1) << 8) | ((1) << 24);
 const TCR_ORGN_WBWA: u64 = ((1) << 10) | ((1) << 26);
 const TCR_SHARED: u64 = ((3) << 12) | ((3) << 28);
+#[expect(dead_code)]
 const TCR_TBI0: u64 = 1 << 37;
+#[expect(dead_code)]
 const TCR_TBI1: u64 = 1 << 38;
+#[expect(dead_code)]
 const TCR_ASID16: u64 = 1 << 36;
+#[expect(dead_code)]
 const TCR_TG1_16K: u64 = 1 << 30;
 const TCR_TG1_4K: u64 = 0 << 30;
 const TCR_FLAGS: u64 = TCR_IRGN_WBWA | TCR_ORGN_WBWA | TCR_SHARED;
 
 /// Number of virtual address bits for 4KB page
 const VA_BITS: u64 = 48;
 
-unsafe extern "C" {
-	static vector_table: u8;
-}
-
-/// Entrypoint - Initialize Stack pointer and Exception Table
-#[unsafe(no_mangle)]
-#[unsafe(naked)]
-pub unsafe extern "C" fn _start(boot_info: Option<&'static RawBootInfo>, cpu_id: u32) -> ! {
-	// validate signatures
-	// `_Start` is compatible to `Entry`
-	{
-		unsafe extern "C" fn _entry(_boot_info: &'static RawBootInfo, _cpu_id: u32) -> ! {
-			unreachable!()
-		}
-		pub type _Start =
-			unsafe extern "C" fn(boot_info: Option<&'static RawBootInfo>, cpu_id: u32) -> !;
-		const _ENTRY: Entry = _entry;
-		const _START: _Start = _start;
-		const _PRE_INIT: _Start = pre_init;
-	}
-
-	naked_asm!(
-		// use core::sync::atomic::{AtomicU32, Ordering};
-		//
-		// pub static CPU_ONLINE: AtomicU32 = AtomicU32::new(0);
-		//
-		// while CPU_ONLINE.load(Ordering::Acquire) != this {
-		//     core::hint::spin_loop();
-		// }
-		"mrs x4, mpidr_el1",
-		"and x4, x4, #0xff",
-		"1:",
-		"adrp x8, {cpu_online}",
-		"ldr x5, [x8, #:lo12:{cpu_online}]",
-		"cmp x4, x5",
-		"b.eq 2f",
-		"b 1b",
-		"2:",
-
-		// we want to use sp_el1
-		"msr spsel, #1",
-
-		// Overwrite RSP if `CURRENT_STACK_ADDRESS != 0`
-		"adrp x8, {current_stack_address}",
-		"ldr x4, [x8, #:lo12:{current_stack_address}]",
-		"cmp x4, 0",
-		"b.eq 3f",
-		"mov sp, x4",
-		"b 4f",
-		"3:",
-		"mov x4, sp",
-		"4:",
-		"str x4, [x8, #:lo12:{current_stack_address}]",
-
-		// Add stack top offset
-		"mov x8, {stack_top_offset}",
-		"add sp, sp, x8",
-
-		// Jump to Rust code
-		"b {pre_init}",
-
-		cpu_online = sym super::CPU_ONLINE,
-		stack_top_offset = const KERNEL_STACK_SIZE - TaskStacks::MARKER_SIZE,
-		current_stack_address = sym super::CURRENT_STACK_ADDRESS,
-		pre_init = sym pre_init,
-	)
-}
-
-#[cfg(feature = "smp")]
 const fn tcr_size(x: u64) -> u64 {
 	((64 - x) << 16) | (64 - x)
 }
 
-#[cfg(feature = "smp")]
 const fn mair(attr: u64, mt: u64) -> u64 {
 	attr << (mt * 8)
 }
 
-#[cfg(feature = "smp")]
 pub(crate) static TTBR0: AtomicPtr<u8> = AtomicPtr::new(ptr::null_mut());
 
-#[cfg(feature = "smp")]
 #[unsafe(naked)]
 pub(crate) unsafe extern "C" fn smp_start() -> ! {
+	use crate::config::KERNEL_STACK_SIZE;
+	use crate::kernel::scheduler::TaskStacks;
+
 	// Prepare system control register (SCTRL)
 	//
 	// UCI     [26] Enables EL0 access in AArch64 for DC CVAU, DC CIVAC,
@@ -154,7 +78,7 @@ pub(crate) unsafe extern "C" fn smp_start() -> ! {
 	#[cfg(target_endian = "big")]
 	const SCTLR_EL1: u64 = 0b111_0000_0101_1101_0000_0001_1101;
 
-	naked_asm!(
+	core::arch::naked_asm!(
 		// disable interrupts
 		"msr daifset, #0b111",
 
@@ -241,47 +165,9 @@ pub(crate) unsafe extern "C" fn smp_start() -> ! {
 		mair_el1 = const mair(0x00, MT_DEVICE_nGnRnE) | mair(0x04, MT_DEVICE_nGnRE) | mair(0x0c, MT_DEVICE_GRE) | mair(0x44, MT_NORMAL_NC) | mair(0xff, MT_NORMAL),
 		tcr_bits = const tcr_size(VA_BITS) | TCR_TG1_4K | TCR_FLAGS,
 		stack_top_offset = const KERNEL_STACK_SIZE - TaskStacks::MARKER_SIZE,
-		current_stack_address = sym super::CURRENT_STACK_ADDRESS,
+		current_stack_address = sym CURRENT_STACK_ADDRESS,
 		sctlr_el1 = const SCTLR_EL1,
 		ttbr0 = sym TTBR0,
-		pre_init = sym pre_init,
+		pre_init = sym crate::arch::start::hermit_entry::pre_init,
 	)
 }
-
-#[inline(never)]
-#[unsafe(no_mangle)]
-unsafe extern "C" fn pre_init(boot_info: Option<&'static RawBootInfo>, cpu_id: u32) -> ! {
-	// set exception table
-	unsafe {
-		asm!(
-			"adrp x4, {vector_table}",
-			"add x4, x4, #:lo12:{vector_table}",
-			"msr vbar_el1, x4",
-			vector_table = sym vector_table,
-			out("x4") _,
-			options(nostack),
-		);
-	}
-
-	// Memory barrier
-	dsb(SY);
-
-	if cpu_id == 0 {
-		env::set_boot_info(*boot_info.unwrap());
-		crate::boot_processor_main()
-	} else {
-		#[cfg(not(feature = "smp"))]
-		{
-			let style = anstyle::Style::new().fg_color(Some(anstyle::AnsiColor::Red.into()));
-			let preamble = format_args!("[            ][{cpu_id}][{style}ERROR{style:#}]");
-			println!(
-				"{preamble} Secondary core booted, but Hermit was not built with SMP support!"
-			);
-			loop {
-				crate::arch::kernel::processor::halt();
-			}
-		}
-		#[cfg(feature = "smp")]
-		crate::application_processor_main()
-	}
-}

src/arch/riscv64/kernel/mod.rs

@@ -8,7 +8,6 @@ pub mod pci;
 pub mod processor;
 pub mod scheduler;
 pub mod serial;
-mod start;
 pub mod switch;
 pub mod systemtime;
 use alloc::vec::Vec;
@@ -32,11 +31,11 @@ use crate::mm::{FrameAlloc, PageRangeAllocator};
 pub(crate) static HARTS_AVAILABLE: InitCell<Vec<usize>> = InitCell::new(Vec::new());
 
 /// Kernel header to announce machine features
-static CPU_ONLINE: AtomicU32 = AtomicU32::new(0);
-static CURRENT_BOOT_ID: AtomicU32 = AtomicU32::new(0);
-static CURRENT_STACK_ADDRESS: AtomicPtr<()> = AtomicPtr::new(ptr::null_mut());
-static HART_MASK: AtomicU64 = AtomicU64::new(0);
-static NUM_CPUS: AtomicU32 = AtomicU32::new(0);
+pub(crate) static CPU_ONLINE: AtomicU32 = AtomicU32::new(0);
+pub(crate) static CURRENT_BOOT_ID: AtomicU32 = AtomicU32::new(0);
+pub(crate) static CURRENT_STACK_ADDRESS: AtomicPtr<()> = AtomicPtr::new(ptr::null_mut());
+pub(crate) static HART_MASK: AtomicU64 = AtomicU64::new(0);
+pub(crate) static NUM_CPUS: AtomicU32 = AtomicU32::new(0);
 
 // FUNCTIONS
 
@@ -155,7 +154,7 @@ pub fn boot_next_processor() {
 	}
 
 	//When running bare-metal/QEMU we use the firmware to start the next hart
-	let start_addr = (start::_start as *const ()).expose_provenance();
+	let start_addr = (crate::arch::start::hermit_entry::_start as *const ()).expose_provenance();
 	sbi_rt::hart_start(next_hart_id as usize, start_addr, 0).unwrap();
 }
 

src/arch/riscv64/mod.rs

@@ -1,2 +1,3 @@
 pub mod kernel;
 pub mod mm;
+pub mod start;

src/arch/riscv64/kernel/start.rs src/arch/riscv64/start/hermit_entry.rssrc/arch/riscv64/kernel/start.rs renamed to src/arch/riscv64/start/hermit_entry.rs

@@ -4,12 +4,12 @@ use core::sync::atomic::Ordering;
 use hermit_entry::Entry;
 use hermit_entry::boot_info::RawBootInfo;
 
-use super::{CPU_ONLINE, CURRENT_BOOT_ID, HART_MASK, NUM_CPUS};
 use crate::arch::riscv64::kernel::CURRENT_STACK_ADDRESS;
 #[cfg(not(feature = "smp"))]
 use crate::arch::riscv64::kernel::processor;
 use crate::config::KERNEL_STACK_SIZE;
 use crate::env;
+use crate::kernel::{CPU_ONLINE, CURRENT_BOOT_ID, HART_MASK, NUM_CPUS};
 
 //static mut BOOT_STACK: [u8; KERNEL_STACK_SIZE] = [0; KERNEL_STACK_SIZE];
 

src/arch/riscv64/start/mod.rs

@@ -0,0 +1 @@
+pub mod hermit_entry;