diff --git a/.cargo/config.toml b/.cargo/config.toml index 6728891..8b887fc 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -8,12 +8,21 @@ target = "target-specs/x86_64-unknown-elpytios.json" [profile.bootloader] inherits = "release" -lto = "thin" +lto = "on" [profile.bootloader_debug] inherits = "bootloader" +strip = "none" debug-assertions = true debug = "full" +lto = "off" [target.x86_64-unknown-none] -rustflags = ["-C", "code-model=kernel", "-C", "relocation-model=static"] +rustflags = [ + "-C", + "no-redzone=yes", + "-C", + "relocation-model=pie", + "-C", + "linker-flavor=ld.lld", +] diff --git a/.gitignore b/.gitignore index 4d430b3..ae71a70 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ /*.exe /*.pdb +/*.log /target /esp *.qcow2 -/rustc-ice-* \ No newline at end of file +/Cargo.lock +/rustc-ice-* diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 47d3596..0000000 --- a/Cargo.lock +++ /dev/null @@ -1,640 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.25.1" -dependencies = [ - "alloc", - "core", - "gimli", - "object", - "rustc-demangle", -] - -[[package]] -name = "adler2" -version = "2.0.1" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "alloc" -version = "0.0.0" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "bit_field" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" - -[[package]] -name = "bitflags" -version = "2.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" - -[[package]] -name = "bytemuck" -version = "1.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" -dependencies = [ - "bytemuck_derive", -] - -[[package]] -name = "bytemuck_derive" -version = "1.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "cc" -version = "1.2.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" -dependencies = [ - "find-msvc-tools", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "compiler_builtins" -version = "0.1.160" -dependencies = [ - "cc", - "core", -] - -[[package]] -name = "const_panic" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" -dependencies = [ - "const_panic_proc_macros", - "typewit", -] - -[[package]] -name = "const_panic_proc_macros" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb0240417fe20ccf13397fa25e6f0a987dbbfaf27d9e13532419df7f593e65e8" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", -] - -[[package]] -name = "convert_case" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "core" -version = "0.0.0" - -[[package]] -name = "derive_more" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" -dependencies = [ - "convert_case", - "proc-macro2", - "quote", - "rustc_version", - "syn", - "unicode-xid", -] - -[[package]] -name = "dlmalloc" -version = "0.2.13" -dependencies = [ - "cfg-if 1.0.4", - "core", - "libc 0.2.185", - "windows-sys", -] - -[[package]] -name = "elpytios-abi" -version = "0.1.0" -dependencies = [ - "compiler_builtins", - "core", - "elpytios-abi-macros", -] - -[[package]] -name = "elpytios-abi-macros" -version = "0.1.0" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "elpytios-bootinfo" -version = "0.1.0" -dependencies = [ - "bitflags", - "bytemuck", - "derive_more", -] - -[[package]] -name = "elpytios-bootloader" -version = "0.1.0" -dependencies = [ - "bytemuck", - "const_panic", - "elpytios-bootinfo", - "elpytios-elf", - "log", - "uefi", -] - -[[package]] -name = "elpytios-elf" -version = "0.1.0" -dependencies = [ - "bitflags", - "bytemuck", - "const_panic", -] - -[[package]] -name = "elpytios-kernel" -version = "0.1.0" -dependencies = [ - "elpytios-abi", - "elpytios-bootinfo", -] - -[[package]] -name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - -[[package]] -name = "foldhash" -version = "0.2.0" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "fortanix-sgx-abi" -version = "0.6.1" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "gimli" -version = "0.32.3" -dependencies = [ - "alloc", - "core", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -dependencies = [ - "alloc", - "core", - "foldhash", - "libc 0.2.163", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.5.2" -dependencies = [ - "alloc", - "compiler_builtins", - "core", -] - -[[package]] -name = "libc" -version = "0.2.163" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fdaeca4cf44ed4ac623e86ef41f056e848dbeab7ec043ecb7326ba300b36fd0" - -[[package]] -name = "libc" -version = "0.2.185" -dependencies = [ - "rustc-std-workspace-core", -] - -[[package]] -name = "log" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" - -[[package]] -name = "memchr" -version = "2.7.6" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -dependencies = [ - "adler2", - "alloc", - "core", -] - -[[package]] -name = "moto-rt" -version = "0.16.0" -dependencies = [ - "alloc", - "core", -] - -[[package]] -name = "object" -version = "0.37.3" -dependencies = [ - "alloc", - "compiler_builtins", - "core", - "hashbrown", - "memchr", -] - -[[package]] -name = "panic_abort" -version = "0.0.0" -dependencies = [ - "alloc", - "compiler_builtins", - "core", - "libc 0.2.185", -] - -[[package]] -name = "panic_unwind" -version = "0.0.0" -dependencies = [ - "alloc", - "core", - "libc 0.2.185", - "unwind", -] - -[[package]] -name = "proc-macro2" -version = "1.0.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "ptr_meta" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "quote" -version = "1.0.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "r-efi-alloc" -version = "2.1.0" -dependencies = [ - "core", - "r-efi", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.27" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "rustc-std-workspace-core" -version = "1.99.0" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - -[[package]] -name = "semver" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" - -[[package]] -name = "shlex" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" - -[[package]] -name = "std" -version = "0.0.0" -dependencies = [ - "addr2line", - "alloc", - "cfg-if 1.0.4", - "core", - "dlmalloc", - "elpytios-abi", - "fortanix-sgx-abi", - "hashbrown", - "hermit-abi", - "libc 0.2.185", - "miniz_oxide", - "moto-rt", - "object", - "panic_abort", - "panic_unwind", - "r-efi", - "r-efi-alloc", - "rustc-demangle", - "std_detect", - "unwind", - "vex-sdk", - "wasip1", - "wasip2", - "wasip3", - "windows-link", -] - -[[package]] -name = "std_detect" -version = "0.1.5" -dependencies = [ - "alloc", - "core", - "libc 0.2.185", -] - -[[package]] -name = "syn" -version = "2.0.67" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "typewit" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" - -[[package]] -name = "ucs2" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79298e11f316400c57ec268f3c2c29ac3c4d4777687955cd3d4f3a35ce7eba" -dependencies = [ - "bit_field", -] - -[[package]] -name = "uefi" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66ab9569afdd1e33a31d8002343aa1df594f055347b1a66136bf9dd6cbc3ec37" -dependencies = [ - "bitflags", - "cfg-if 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "log", - "ptr_meta", - "ucs2", - "uefi-macros", - "uefi-raw", - "uguid", -] - -[[package]] -name = "uefi-macros" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4687412b5ac74d245d5bfb1733ede50c31be19bf8a4b6a967a29b451bab49e67" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "uefi-raw" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3775e5934877acaef4b00f254f252df1e2266903c31e51455c117f4f2824eda" -dependencies = [ - "bitflags", - "uguid", -] - -[[package]] -name = "uguid" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8352f8c05e47892e7eaf13b34abd76a7f4aeaf817b716e88789381927f199c" - -[[package]] -name = "unicode-ident" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" - -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - -[[package]] -name = "unwind" -version = "0.0.0" -dependencies = [ - "core", - "libc 0.2.185", - "unwinding", -] - -[[package]] -name = "unwinding" -version = "0.2.8" -dependencies = [ - "core", - "gimli", - "libc 0.2.185", -] - -[[package]] -name = "vex-sdk" -version = "0.27.1" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "wasip1" -version = "1.0.0" -dependencies = [ - "alloc", - "compiler_builtins", - "core", -] - -[[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" -dependencies = [ - "alloc", - "core", - "wit-bindgen", -] - -[[package]] -name = "wasip3" -version = "0.6.0+wasi-0.3.0-rc-2026-03-15" -dependencies = [ - "alloc", - "core", - "wit-bindgen", -] - -[[package]] -name = "windows-link" -version = "0.0.0" - -[[package]] -name = "windows-sys" -version = "0.61.100" -dependencies = [ - "compiler_builtins", - "core", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -dependencies = [ - "alloc", - "core", -] diff --git a/Cargo.toml b/Cargo.toml index 9bb137d..9386014 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,13 @@ [workspace] resolver = "3" -members = ["abi", "abi/macros", "elf", "std", "kernel", "bootloader", "bootinfo"] +members = [ + "abi", + "abi/macros", + "alloc", + "elf", + "std", + "kernel", + "bootloader", + "bootinfo", + "user/shell", +] diff --git a/abi/macros/src/lib.rs b/abi/macros/src/lib.rs index 1e98a66..b4c9b62 100644 --- a/abi/macros/src/lib.rs +++ b/abi/macros/src/lib.rs @@ -1,11 +1,14 @@ use heck::ToSnakeCase; use proc_macro2::{Span, TokenStream}; use quote::quote; -use syn::{Fields, Ident, ItemEnum, Token, punctuated::Punctuated}; +use syn::{ + Fields, Ident, ItemEnum, LitInt, Token, Type, parenthesized, + parse::{Parse, ParseStream}, +}; extern crate proc_macro; -#[proc_macro_derive(SyscallTable, attributes(args))] +#[proc_macro_derive(SyscallTable, attributes(args, max_entries))] pub fn derive_syscall_table(input: proc_macro::TokenStream) -> proc_macro::TokenStream { match execute(input.into()) { Ok(result) => result, @@ -14,55 +17,134 @@ pub fn derive_syscall_table(input: proc_macro::TokenStream) -> proc_macro::Token .into() } +struct Def { + names: Vec, + types: Vec, + ret: Type, +} + +impl Parse for Def { + fn parse(input: ParseStream) -> syn::Result { + let content; + parenthesized!(content in input); + + let mut names = vec![]; + let mut types = vec![]; + for (name, ty) in content.parse_terminated( + |input| { + let name: Ident = input.parse()?; + let _colon: Token![:] = input.parse()?; + let ty: Type = input.parse()?; + Ok((name, ty)) + }, + Token![,], + )? { + names.push(name); + types.push(ty); + } + + let _arrow: Token![=>] = input.parse()?; + let ret: Type = input.parse()?; + + Ok(Self { names, types, ret }) + } +} + fn execute(input: TokenStream) -> syn::Result { let data = syn::parse2::(input)?; - let mut functions = vec![]; + let mut entries = vec![]; + let mut userspace_functions = vec![]; + + let mut max_entries = None; + for attr in &data.attrs { + if attr.path().is_ident("max_entries") && max_entries.replace(attr.parse_args::()?.base10_parse::()?).is_some() { + return Err(syn::Error::new_spanned(attr, "Duplicate `max_entries(..)`!")) + } + } + let max_entries = max_entries.ok_or_else(|| syn::Error::new_spanned(&data, "Missing `max_entries(..)`!"))?; + let mut test_discriminants = vec![]; for variant in &data.variants { let Fields::Unit = variant.fields else { Err(syn::Error::new_spanned(&variant, "Only unit-variant is supported!"))? }; + if let Some((.., disc)) = &variant.discriminant { + test_discriminants.push(quote! { + assert!(#disc < #max_entries); + }); + } else { + return Err(syn::Error::new_spanned(variant, "Missing discriminant!")) + } + let mut args = None; for attr in &variant.attrs { - if attr.path().is_ident("args") - && args - .replace(attr.parse_args_with(Punctuated::::parse_terminated)?) - .is_some() - { + if attr.path().is_ident("args") && args.replace(attr.parse_args::()?).is_some() { return Err(syn::Error::new_spanned(&variant, "Duplicate `args(..)`!")) } } - let args = args.unwrap_or_default(); - let driver = Ident::new( - match args.len() { - 0 => "syscall0", - 1 => "syscall1", - 2 => "syscall2", - 3 => "syscall3", - 4 => "syscall4", - 5 => "syscall5", - n => return Err(syn::Error::new_spanned(args, format!("Too many arguments ({n}); maximum is 5!"))), - }, - Span::call_site(), - ); + let Def { names, types, ret } = match args { + Some(args) => args, + None => return Err(syn::Error::new_spanned(&variant, "Missing `args(..)`!")), + }; + + let [driver, fn_type] = match names.len() { + 0 => ["syscall0", "Syscall0Fn"], + 1 => ["syscall1", "Syscall1Fn"], + 2 => ["syscall2", "Syscall2Fn"], + 3 => ["syscall3", "Syscall3Fn"], + 4 => ["syscall4", "Syscall4Fn"], + 5 => ["syscall5", "Syscall5Fn"], + n => return Err(syn::Error::new_spanned(variant, format!("Too many arguments ({n}); maximum is 5!"))), + }; + let driver = Ident::new(driver, Span::call_site()); + let fn_type = Ident::new(fn_type, Span::call_site()); let variant_name = &variant.ident; let driver_name = Ident::new(&variant.ident.to_string().to_snake_case(), Span::call_site()); - let args = args.into_iter().collect::>(); - functions.push(quote! { + entries.push(quote! { + pub #driver_name: crate::kernel::#fn_type::<#(#types,)*> + }); + userspace_functions.push(quote! { #[inline(always)] - pub unsafe fn #driver_name(#(#args: usize),*) -> usize { - crate::userspace::#driver(Self::#variant_name as usize, #(#args),*) + pub unsafe fn #driver_name(#(#names: #types),*) -> #ret { + const { + #(__assert_is_arg::<#types>();)* + } + + unsafe { + let ret = crate::userspace::#driver(Self::#variant_name as usize, #(crate::SyscallArg::into_usize(#names)),*); + <#ret as crate::SyscallArg>::from_usize(ret) + } } }); } let data_name = data.ident.clone(); + let entry_name = Ident::new(&format!("{data_name}Entry"), Span::call_site()); + Ok(quote! { - #[cfg(not(target_os = "none"))] + const _: () = { + #(#test_discriminants)* + }; + + const fn __assert_is_arg() {} + impl #data_name { - #(#functions)* + pub const MAX_ENTRIES: usize = #max_entries; + pub const INVALID: usize = usize::MAX; + + #(#userspace_functions)* + } + + #[derive(Copy, Clone)] + pub union #entry_name { + missing: usize, + #(#entries),* + } + + impl #entry_name { + pub const MISSING: Self = Self { missing: 0 }; } }) } diff --git a/abi/src/kernel/x86_64.rs b/abi/src/kernel/x86_64.rs new file mode 100644 index 0000000..99e3b56 --- /dev/null +++ b/abi/src/kernel/x86_64.rs @@ -0,0 +1,7 @@ +pub type Syscall0Fn = unsafe extern "sysv64" fn() -> usize; +pub type Syscall1Fn = unsafe extern "sysv64" fn(A0) -> usize; +pub type Syscall2Fn = unsafe extern "sysv64" fn(A0, A1) -> usize; +pub type Syscall3Fn = unsafe extern "sysv64" fn(A0, A1, A2) -> usize; +pub type Syscall4Fn = unsafe extern "sysv64" fn(A0, A1, A2, A3) -> usize; +pub type Syscall5Fn = unsafe extern "sysv64" fn(A0, A1, A2, A3, A4) -> usize; +pub type Syscall6Fn = unsafe extern "sysv64" fn(A0, A1, A2, A3, A4, A5) -> usize; diff --git a/abi/src/lib.rs b/abi/src/lib.rs index e838427..c8032cb 100644 --- a/abi/src/lib.rs +++ b/abi/src/lib.rs @@ -1,31 +1,128 @@ +#![feature(ptr_alignment_type)] #![no_std] +use core::{alloc::Layout, mem::Alignment}; + use elpytios_abi_macros::SyscallTable; -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[repr(usize)] -#[derive(SyscallTable)] +pub const PAGE_SIZE: usize = 4096; +pub const PAGE_ALIGN: Alignment = unsafe { Alignment::new_unchecked(4096) }; +pub const PAGE_LAYOUT: Layout = unsafe { Layout::from_size_alignment_unchecked(PAGE_SIZE, PAGE_ALIGN) }; + +/// Physical allocators are aligned to 32 pages (128 KiB). +/// This means pointers of allocations up to 32 pages are guaranteed to be aligned. +pub const ALLOC_ALIGNMENT: Alignment = unsafe { Alignment::new_unchecked(1 << (32 * PAGE_SIZE).ilog2()) }; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, SyscallTable)] +#[max_entries(0x1000)] pub enum Syscall { - #[args(file, buffer, len)] - Write = 0, - #[args(file, buffer, len)] - Read = 1, -} - -#[cfg(not(target_os = "none"))] -#[expect( - unused, - reason = "Not all args are used yet, they will be in the future. Remove this `expect()` when that happens." -)] -pub(crate) mod userspace { + #[args((file: FileHandle, buffer: *const u8, len: usize) => usize)] + Write = 0x000, + #[args((file: FileHandle, buffer: *mut u8, len: usize) => usize)] + Read = 0x001, + + #[args((file: FileHandle, offset: usize, page_count: usize, flags: usize) => *mut u8)] + MemMap = 0x010, + + #[args(() => usize)] + YieldNow = 0x100, +} + +pub trait SyscallArg { + fn into_usize(self) -> usize; + + unsafe fn from_usize(value: usize) -> Self; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(transparent)] +pub struct FileHandle(usize); +impl FileHandle { + pub const STDOUT: Self = Self(0); + pub const STDIN: Self = Self(1); + + pub const NONE: Self = Self(usize::MAX); +} + +macro_rules! impl_syscall_arg { + ($(impl $target:ty { get!($get:ident => $($getter:tt)*), set!($set:ident => $($setter:tt)*) })*) => { + $(impl SyscallArg for $target { + #[inline(always)] + fn into_usize(self) -> usize { + let $get = self; + $($getter)* + } + + #[inline(always)] + unsafe fn from_usize($set: usize) -> Self { + $($setter)* + } + })* + }; +} + +impl_syscall_arg! { + impl FileHandle { get!(this => this.0), set!(id => Self(id)) } +} + +impl SyscallArg for usize { + #[inline(always)] + fn into_usize(self) -> usize { + self + } + + #[inline(always)] + unsafe fn from_usize(value: usize) -> Self { + value + } +} + +impl SyscallArg for *const T { + #[inline(always)] + fn into_usize(self) -> usize { + self as usize + } + + #[inline(always)] + unsafe fn from_usize(value: usize) -> Self { + value as Self + } +} + +impl SyscallArg for *mut T { + #[inline(always)] + fn into_usize(self) -> usize { + self as usize + } + + #[inline(always)] + unsafe fn from_usize(value: usize) -> Self { + value as Self + } +} + +#[expect(unused, reason = "Not all parameters are used yet")] +mod kernel { + cfg_select! { + target_arch = "x86_64" => { + mod x86_64; + pub use x86_64::*; + } + _ => { + compile_error!("Unsupported architecture"); + } + } +} + +#[expect(unused, reason = "Not all parameters are used yet")] +mod userspace { cfg_select! { target_arch = "x86_64" => { mod x86_64; pub use x86_64::*; } _ => { - mod unsupported; - pub use unsupported::*; + compile_error!("Unsupported architecture"); } } } diff --git a/abi/src/userspace/unsupported.rs b/abi/src/userspace/unsupported.rs deleted file mode 100644 index 3a3c1c0..0000000 --- a/abi/src/userspace/unsupported.rs +++ /dev/null @@ -1,34 +0,0 @@ -#[inline(always)] -pub unsafe fn syscall0(sys: usize) -> usize { - unimplemented!("{sys}") -} - -#[inline(always)] -pub unsafe fn syscall1(sys: usize, a0: usize) -> usize { - unimplemented!("{sys}, {a0}") -} - -#[inline(always)] -pub unsafe fn syscall2(sys: usize, a0: usize, a1: usize) -> usize { - unimplemented!("{sys}, {a0}, {a1}") -} - -#[inline(always)] -pub unsafe fn syscall3(sys: usize, a0: usize, a1: usize, a2: usize) -> usize { - unimplemented!("{sys}, {a0}, {a1}, {a2}") -} - -#[inline(always)] -pub unsafe fn syscall4(sys: usize, a0: usize, a1: usize, a2: usize, a3: usize) -> usize { - unimplemented!("{sys}, {a0}, {a1}, {a2}, {a3}") -} - -#[inline(always)] -pub unsafe fn syscall5(sys: usize, a0: usize, a1: usize, a2: usize, a3: usize, a4: usize) -> usize { - unimplemented!("{sys}, {a0}, {a1}, {a2}, {a3}, {a4}") -} - -#[inline(always)] -pub unsafe fn syscall6(sys: usize, a0: usize, a1: usize, a2: usize, a3: usize, a4: usize, a5: usize) -> usize { - unimplemented!("{sys}, {a0}, {a1}, {a2}, {a3}, {a4}, {a5}") -} diff --git a/abi/src/userspace/x86_64.rs b/abi/src/userspace/x86_64.rs index 7f62c44..715c1ee 100644 --- a/abi/src/userspace/x86_64.rs +++ b/abi/src/userspace/x86_64.rs @@ -8,7 +8,11 @@ pub unsafe fn syscall0(sys: usize) -> usize { "syscall", inlateout("rax") sys => ret, - options(nostack, preserves_flags), + out("rcx") _, + out("r11") _, + out("r12") _, + + options(att_syntax, nomem, nostack, preserves_flags), ); } ret @@ -24,7 +28,11 @@ pub unsafe fn syscall1(sys: usize, a0: usize) -> usize { in("rdi") a0, - options(nostack, preserves_flags), + out("rcx") _, + out("r11") _, + out("r12") _, + + options(att_syntax, nomem, nostack, preserves_flags), ); } ret @@ -41,7 +49,11 @@ pub unsafe fn syscall2(sys: usize, a0: usize, a1: usize) -> usize { in("rdi") a0, in("rsi") a1, - options(nostack, preserves_flags), + out("rcx") _, + out("r11") _, + out("r12") _, + + options(att_syntax, nomem, nostack, preserves_flags), ); } ret @@ -59,7 +71,11 @@ pub unsafe fn syscall3(sys: usize, a0: usize, a1: usize, a2: usize) -> usize { in("rsi") a1, in("rdx") a2, - options(nostack, preserves_flags), + out("rcx") _, + out("r11") _, + out("r12") _, + + options(att_syntax, nomem, nostack, preserves_flags), ); } ret @@ -78,7 +94,11 @@ pub unsafe fn syscall4(sys: usize, a0: usize, a1: usize, a2: usize, a3: usize) - in("rdx") a2, in("r10") a3, - options(nostack, preserves_flags), + out("rcx") _, + out("r11") _, + out("r12") _, + + options(att_syntax, nomem, nostack, preserves_flags), ); } ret @@ -98,7 +118,11 @@ pub unsafe fn syscall5(sys: usize, a0: usize, a1: usize, a2: usize, a3: usize, a in("r10") a3, in("r8") a4, - options(nostack, preserves_flags), + out("rcx") _, + out("r11") _, + out("r12") _, + + options(att_syntax, nomem, nostack, preserves_flags), ); } ret @@ -119,7 +143,11 @@ pub unsafe fn syscall6(sys: usize, a0: usize, a1: usize, a2: usize, a3: usize, a in("r8") a4, in("r9") a5, - options(nostack, preserves_flags), + out("rcx") _, + out("r11") _, + out("r12") _, + + options(att_syntax, nomem, nostack, preserves_flags), ); } ret diff --git a/alloc/Cargo.toml b/alloc/Cargo.toml new file mode 100644 index 0000000..530304c --- /dev/null +++ b/alloc/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "elpytios-alloc" +version = "0.1.0" +edition = "2024" +rust-version = "1.98" + +[dependencies] +elpytios-abi = { path = "../abi" } + +compiler_builtins = { path = "../std/rust-src/compiler-builtins/compiler-builtins", features = [ + "compiler-builtins", +], optional = true } +core = { path = "../std/rust-src/core", optional = true } + +[features] +sysroot-dep = ["dep:compiler_builtins", "dep:core"] diff --git a/alloc/src/lib.rs b/alloc/src/lib.rs new file mode 100644 index 0000000..220f412 --- /dev/null +++ b/alloc/src/lib.rs @@ -0,0 +1,450 @@ +#![feature( + allocator_api, + arbitrary_self_types_pointers, + atomic_ptr_null, + const_trait_impl, + pointer_is_aligned_to, + ptr_alignment_type, + slice_ptr_get +)] +#![no_std] + +use core::{ + alloc::{AllocError, Allocator, GlobalAlloc, Layout}, + cell::UnsafeCell, + hint::{cold_path, spin_loop, unreachable_unchecked}, + marker::PhantomData, + mem::Alignment, + ptr::{self, NonNull}, + sync::atomic::{ + AtomicPtr, AtomicUsize, + Ordering::{Acquire, Relaxed, Release}, + }, +}; + +use elpytios_abi::{ALLOC_ALIGNMENT, PAGE_SIZE}; + +/// # Safety +/// - Allocations of sizes up to [`ALLOC_ALIGNMENT`] must be aligned to the nearest power of two of +/// that size. +pub unsafe trait PageAllocator { + fn alloc(&self, order: u32) -> Option>; + + unsafe fn dealloc(&self, ptr: NonNull, order: u32); +} + +const MICRO_SIZES: [usize; 8] = [8, 16, 24, 32, 48, 64, 96, 128]; +const SMALL_SIZES: [usize; 8] = [192, 256, 384, 512, 768, 1024, 1536, 2048]; +const MEDIUM_SIZES: [usize; 4] = [3072, 4096, 6144, 8192]; +const LARGE_SIZES: [usize; 4] = [12288, 16384, 24576, 32768]; + +pub struct HeapAllocator { + page_alloc: T, + micro_bins: [AtomicPtr>; MICRO_SIZES.len()], + small_bins: [AtomicPtr>; SMALL_SIZES.len()], + medium_bins: [AtomicPtr>; MEDIUM_SIZES.len()], + large_bins: [AtomicPtr>; LARGE_SIZES.len()], +} + +#[repr(C, align(4096))] +struct Segment { + data: UnsafeCell<[u8; N]>, + meta: SegmentMeta, + _marker: PhantomData<*const T>, +} + +impl Segment { + const LOCK: usize = 1 << (usize::BITS - 1); + const MASK: usize = !Self::LOCK; + + #[inline] + fn new(page_alloc: &T, size_class: usize) -> *mut Self { + // Ensure that segment allocations are always aligned + _ = const { + assert!(size_of::().is_power_of_two()); + assert!(ALLOC_ALIGNMENT.as_usize().is_multiple_of(size_of::())); + }; + + let this = page_alloc + .alloc(size_of::().ilog2()) + .expect("Couldn't allocate pages for heap allocator") + .cast::() + .as_ptr(); + + debug_assert!( + this.is_aligned_to(size_of::()), + "Physical page allocator didn't allocate an aligned heap" + ); + + unsafe { + let offset = UnsafeCell::raw_get(&raw const (*this).data) as usize % size_class; + let available = ((N - offset) / size_class).min(1 << u16::BITS); + + let data = UnsafeCell::raw_get(&raw const (*this).data).cast::().byte_add(offset as usize); + for i in 0..available { + data.byte_add(i as usize * size_class).cast::().write(((i + 1) % available) as u16); + } + + (&raw mut (*this).meta).write(SegmentMeta { + head_and_lock: AtomicUsize::new(0), + next: UnsafeCell::new(ptr::null_mut()), + available: UnsafeCell::new(available), + offset, + }); + } + + this + } + + // `detach()` is called while this segment is still locked + unsafe fn alloc(self: *mut Self, size_class: usize, detach: impl FnOnce(*mut *mut Self)) -> *mut u8 { + unsafe { + let meta = &(*self).meta; + let mut curr_head = meta.head_and_lock.load(Relaxed) & Self::MASK; + + loop { + match meta + .head_and_lock + .compare_exchange_weak(curr_head, curr_head | Self::LOCK, Acquire, Relaxed) + { + Ok(..) => { + break if let Some(new_available) = meta.available.get().read().checked_sub(1) { + let data = UnsafeCell::raw_get(&raw const (*self).data) + .cast::() + .byte_add(meta.offset + curr_head * size_class); + + meta.available.get().write(new_available); + match new_available { + 0 => { + detach(meta.next.get().cast()); + meta.head_and_lock.store(0, Release); + data + } + _ => { + let next_head = data.cast::().read(); + meta.head_and_lock.store(next_head as usize, Release); + data + } + } + } else { + meta.head_and_lock.store(curr_head, Release); + ptr::null_mut() + } + } + Err(updated_head) => { + curr_head = updated_head & Self::MASK; + spin_loop(); + } + } + } + } + } + + // `resurrect()` is called while this segment is still locked + unsafe fn dealloc(self: *mut Self, size_class: usize, at: NonNull, resurrect: impl FnOnce()) { + unsafe { + let meta = &(*self).meta; + let mut curr_head = meta.head_and_lock.load(Relaxed) & Self::MASK; + + let base = UnsafeCell::raw_get(&raw const (*self).data).cast::().byte_add(meta.offset); + let at_index = at.as_ptr().byte_offset_from_unsigned(base) / size_class; + + loop { + match meta + .head_and_lock + .compare_exchange_weak(curr_head, curr_head | Self::LOCK, Acquire, Relaxed) + { + Ok(..) => { + break match meta.available.get().read() { + 0 => { + meta.available.get().write(1); + resurrect(); + meta.head_and_lock.store(at_index, Release); + } + available => { + at.cast::().write(curr_head as u16); + meta.available.get().write(available + 1); + meta.head_and_lock.store(at_index, Release); + } + } + } + Err(updated_head) => { + curr_head = updated_head & Self::MASK; + spin_loop(); + } + } + } + } + } +} + +// `align(64)` fits the meta to a cache line +#[repr(C, align(64))] +struct SegmentMeta { + head_and_lock: AtomicUsize, + /// Synchronizes-with `head_and_lock`. + next: UnsafeCell<*mut ()>, + /// Synchronizes with top-level `head_ptr`. + available: UnsafeCell, + offset: usize, +} + +type MicroSegment = Segment() }>; +type SmallSegment = Segment() }>; +type MediumSegment = Segment() }>; +type LargeSegment = Segment() }>; + +enum SizeClassify<'a, T: PageAllocator> { + Micro(&'a AtomicPtr>, usize), + Small(&'a AtomicPtr>, usize), + Medium(&'a AtomicPtr>, usize), + Large(&'a AtomicPtr>, usize), + Huge, +} + +impl<'a, T: PageAllocator> SizeClassify<'a, T> { + #[inline] + unsafe fn new(alloc: &'a HeapAllocator, size: usize) -> Self { + #[inline] + unsafe fn get<'a, const N: usize, T, R>( + ptrs: &'a [AtomicPtr], + size: usize, + segment_sizes: [usize; N], + f: impl FnOnce(&'a AtomicPtr, usize) -> R, + ) -> R { + unsafe { + let i = segment_sizes + .into_iter() + .enumerate() + .filter_map(|(i, segment_size)| Some((i, segment_size.checked_sub(size)?))) + .min_by_key(|&(.., segment_size)| segment_size) + .unwrap_unchecked() + .0; + + f(ptrs.get_unchecked(i), *segment_sizes.get_unchecked(i)) + } + } + + const MICRO_LARGEST: usize = MICRO_SIZES[MICRO_SIZES.len() - 1]; + const SMALL_LARGEST: usize = SMALL_SIZES[SMALL_SIZES.len() - 1]; + const MEDIUM_LARGEST: usize = MEDIUM_SIZES[MEDIUM_SIZES.len() - 1]; + const LARGE_LARGEST: usize = LARGE_SIZES[LARGE_SIZES.len() - 1]; + + unsafe { + match size { + 0 => unreachable_unchecked(), + ..=MICRO_LARGEST => get(&alloc.micro_bins, size, MICRO_SIZES, Self::Micro), + ..=SMALL_LARGEST => get(&alloc.small_bins, size, SMALL_SIZES, Self::Small), + ..=MEDIUM_LARGEST => get(&alloc.medium_bins, size, MEDIUM_SIZES, Self::Medium), + ..=LARGE_LARGEST => get(&alloc.large_bins, size, LARGE_SIZES, Self::Large), + _ => Self::Huge, + } + } + } +} + +impl HeapAllocator { + pub const fn new(page_alloc: T) -> Self { + Self { + page_alloc, + micro_bins: [const { AtomicPtr::null() }; _], + small_bins: [const { AtomicPtr::null() }; _], + medium_bins: [const { AtomicPtr::null() }; _], + large_bins: [const { AtomicPtr::null() }; _], + } + } + + fn alloc(page_alloc: &T, head: &AtomicPtr>, size_class: usize) -> NonNull { + let mut head_ptr = head.load(Relaxed); + loop { + // HEAD is locked (least-significant bit is set) + if !head_ptr.is_aligned() { + spin_loop(); + + head_ptr = head.load(Relaxed); + continue + } + + // HEAD is null, lock and allocate a new one + // This races with `dealloc()`'s resurrection logic when HEAD is null + if head_ptr.is_null() { + match head.compare_exchange(head_ptr, head_ptr.wrapping_byte_add(1), Acquire, Relaxed) { + Ok(..) => { + let new = Segment::new(page_alloc, size_class); + head.store(new, Release); + + head_ptr = new; + } + Err(curr_segment_ptr) => { + head_ptr = curr_segment_ptr; + spin_loop(); + + continue + } + } + } + + unsafe { + break match NonNull::new(head_ptr.alloc(size_class, |next| { + loop { + // Invariant: + // - HEAD is always `head_ptr`, either locked or unlocked + // - This is ensured because if `head_ptr` is not null, only `alloc()` ever changes HEAD directly + match head.compare_exchange_weak(head_ptr, head_ptr.wrapping_byte_add(1), Acquire, Relaxed) { + // Scenario A: Absolutely no detached segments is available: + // - `next` is null, and `alloc()` will lock and allocate a new segment + // Scenario B: A detached segment tries to resurrects, but `alloc()` wins the race: + // - `next` is null + // - If `alloc()`'s new segment wins the race, `dealloc()` will set `next` of new HEAD + // - If `dealloc()` wins the race, `next` will not be null + Ok(..) => { + head.store(ptr::replace(next, ptr::null_mut()), Release); + break + } + // Don't bother updating `head_ptr`, it won't change to a new segment + // It may be locked by `dealloc()`, however, so do a spin-loop + Err(..) => spin_loop(), + } + } + })) { + Some(at) => at, + None => { + spin_loop(); + head_ptr = head.load(Relaxed); + continue + } + } + } + } + } + + fn dealloc(head: &AtomicPtr>, size_class: usize, at: NonNull) { + let segment_ptr = (at.as_ptr() as usize & !(size_of::>() - 1)) as *mut Segment; + unsafe { + segment_ptr.dealloc(size_class, at, || { + let mut head_ptr = head.load(Relaxed); + loop { + if !head_ptr.is_aligned() { + spin_loop(); + + head_ptr = head.load(Relaxed); + continue + } + + // If HEAD is null and `dealloc()` wins the race, set HEAD directly + if head_ptr.is_null() { + match head.compare_exchange(head_ptr, head_ptr.wrapping_byte_add(1), Acquire, Relaxed) { + Ok(..) => { + head.store(segment_ptr, Release); + break + } + Err(curr_head_ptr) => { + head_ptr = curr_head_ptr; + spin_loop(); + continue + } + } + } + + // Otherwise, set HEAD's `next` instead + match head.compare_exchange_weak(head_ptr, head_ptr.wrapping_byte_add(1), Acquire, Relaxed) { + Ok(..) => { + let prev_next = ptr::replace((*head_ptr).meta.next.get(), segment_ptr.cast()); + (*segment_ptr).meta.next.get().write(prev_next); + + head.store(head_ptr, Release); + break + } + Err(curr_head_ptr) => { + head_ptr = curr_head_ptr; + spin_loop(); + } + } + } + }); + } + } + + #[cold] + fn alloc_huge(page_alloc: &T, layout: Layout) -> Option> { + if layout.alignment() <= ALLOC_ALIGNMENT { + page_alloc.alloc(usize::BITS - (layout.size() - 1).leading_zeros()) + } else { + cold_path(); + None + } + } + + #[cold] + unsafe fn dealloc_huge(page_alloc: &T, layout: Layout, at: NonNull) { + unsafe { page_alloc.dealloc(at, usize::BITS - (layout.size() - 1).leading_zeros()) } + } + + #[inline] + fn unionize(layout: Layout) -> Layout { + let new_size = layout.size().next_multiple_of(size_of::()); + let new_align = layout.alignment().max(Alignment::of::()); + unsafe { Layout::from_size_alignment_unchecked(new_size, new_align).pad_to_align() } + } +} + +unsafe impl Sync for HeapAllocator {} +unsafe impl Allocator for HeapAllocator { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + if layout.size() == 0 { + cold_path(); + return Err(AllocError) + } + + let page_alloc = &self.page_alloc; + let layout_padded = Self::unionize(layout); + + Ok(NonNull::slice_from_raw_parts( + match unsafe { SizeClassify::new(self, layout_padded.size()) } { + SizeClassify::Micro(ptr, size_class) => Self::alloc(page_alloc, ptr, size_class), + SizeClassify::Small(ptr, size_class) => Self::alloc(page_alloc, ptr, size_class), + SizeClassify::Medium(ptr, size_class) => { + cold_path(); + Self::alloc(page_alloc, ptr, size_class) + } + SizeClassify::Large(ptr, size_class) => { + cold_path(); + Self::alloc(page_alloc, ptr, size_class) + } + SizeClassify::Huge => Self::alloc_huge(page_alloc, layout).ok_or(AllocError)?, + }, + layout.size(), + )) + } + + unsafe fn deallocate(&self, at: NonNull, layout: Layout) { + let layout_padded = Self::unionize(layout); + unsafe { + match SizeClassify::new(self, layout_padded.size()) { + SizeClassify::Micro(ptr, size_class) => Self::dealloc(ptr, size_class, at), + SizeClassify::Small(ptr, size_class) => Self::dealloc(ptr, size_class, at), + SizeClassify::Medium(ptr, size_class) => { + cold_path(); + Self::dealloc(ptr, size_class, at) + } + SizeClassify::Large(ptr, size_class) => { + cold_path(); + Self::dealloc(ptr, size_class, at) + } + SizeClassify::Huge => Self::dealloc_huge(&self.page_alloc, layout, at), + } + } + } +} + +unsafe impl GlobalAlloc for HeapAllocator { + #[inline] + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + unsafe { self.allocate(layout).unwrap_unchecked() }.as_non_null_ptr().as_ptr() + } + + #[inline] + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { self.deallocate(NonNull::new_unchecked(ptr), layout) } + } +} diff --git a/bootinfo/Cargo.toml b/bootinfo/Cargo.toml index 4e2d839..53a9a02 100644 --- a/bootinfo/Cargo.toml +++ b/bootinfo/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" rust-version = "1.98" [dependencies] +arrayvec = { version = "0.7", default-features = false } bitflags = "2" bytemuck = { version = "1", features = ["derive", "latest_stable_rust"] } derive_more = { version = "2", default-features = false, features = [ diff --git a/bootinfo/src/lib.rs b/bootinfo/src/lib.rs index 343fcd8..132f120 100644 --- a/bootinfo/src/lib.rs +++ b/bootinfo/src/lib.rs @@ -1,63 +1,22 @@ #![feature(custom_inner_attributes)] #![rustfmt::skip] - #![no_std] pub mod paddr; -pub mod vaddr { - use super::*; - - #[derive(Debug, Clone, Copy)] - #[repr(transparent)] - pub struct VFlags(usize); - bitflags! { - impl VFlags: usize { - const WRITABLE = 1 << 0; - const USER_MODE = 1 << 1; - const WRITE_THROUGH = 1 << 2; - const CACHE_DISABLED = 1 << 3; - const ACCESSED = 1 << 4; - - /// Don't flush translation lookaside buffers when switching virtual map tables - const GLOBAL = 1 << 5; - } - } - - #[derive(Debug, Display, Clone, Copy)] - #[repr(C)] - pub enum VirtualMapError { - #[display("Couldn't allocate a page table")] - PageTable, - #[display("Couldn't map {v_addr} to {p_addr}: the virtual address is reserved")] - Reserved { p_addr: PAddr, v_addr: VAddr }, - #[display("Couldn't map {v_addr} to {p_addr}: the virtual address is already mapped to {p_addr_existing}")] - AlreadyMapped { p_addr: PAddr, v_addr: VAddr, p_addr_existing: PAddr } - } - - cfg_select! { - target_arch = "x86_64" => { - mod x86_64; - pub use x86_64::*; - } - _ => { - compile_error!("Unsupported architecture"); - } - } -} +use arrayvec::ArrayVec; use bitflags::bitflags; -use derive_more::Display; - -use core::{mem::MaybeUninit, slice}; - use paddr::PAddr; -use vaddr::{VAddr, VirtualMap}; pub const PAGE_SIZE: usize = 4096; + pub const MAX_MEMORY_REGIONS: usize = 128; +pub const MAX_IDENTITY_MAPS: usize = 32; +pub const MAX_SCRATCH: usize = 4; +pub const MAX_RELOCATIONS: usize = 8; #[allow(non_camel_case_types)] -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy)] #[repr(usize)] pub enum PixelFormat { RGB_8_BIT, @@ -66,44 +25,84 @@ pub enum PixelFormat { BLT_ONLY } +#[derive(Debug, Clone, Copy)] #[repr(C)] -#[derive(Clone, Copy)] pub struct GraphicsInfo { pub w: usize, pub h: usize, pub stride: usize, pub pixel_format: PixelFormat, - pub frame_buffer: *mut u8, + pub frame_buffer: PAddr, pub frame_buffer_size: usize } -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy)] +#[repr(C)] pub struct MemoryRegion { pub base: PAddr, - pub pages: usize + pub pages: usize, } -#[repr(C, align(4096))] -pub struct BootInfo { - pub graphics_info: GraphicsInfo, - pub virtual_map: VirtualMap, - /// Leftover identity-mapping from the bootloader, to be unmapped by the kernel - pub switcher_map: VAddr, - - pub memory_regions_base: [MaybeUninit; MAX_MEMORY_REGIONS], - pub memory_regions_size: usize, - pub v_addr_start: VAddr, - pub v_addr_end: VAddr, +impl MemoryRegion { + #[inline] + pub const fn at(base: PAddr, pages: usize) -> Self { + Self { base, pages } + } } -impl BootInfo { - #[inline] - pub fn memory_regions(&self) -> &[MemoryRegion] { - unsafe { slice::from_raw_parts(&raw const self.memory_regions_base as _, self.memory_regions_size) } +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct IdentityMap { + pub region: MemoryRegion, + pub flags: IdentityMapFlags, +} + +bitflags! { + #[derive(Debug, Clone, Copy)] + #[repr(transparent)] + pub struct IdentityMapFlags: u8 { + const EXECUTABLE = 1 << 0; + const WRITABLE = 1 << 1; + const READABLE = 1 << 2; } +} +impl IdentityMap { #[inline] - pub fn v_addr_range(&self) -> [VAddr; 2] { - [self.v_addr_start, self.v_addr_end] + pub const fn new(base: PAddr, pages: usize, flags: IdentityMapFlags) -> Self { + Self { + region: MemoryRegion { base, pages }, + flags, + } } +} + +#[derive(Debug, Clone, Copy)] +pub struct Reloc { + pub offset: usize, + pub size: usize, + pub stride: usize, +} + +#[derive(Debug, Clone, Copy)] +pub enum DeviceTree { + Acpi(PAddr), + Acpi2(PAddr), +} + +// Note: Must uphold `BootInfo: Sync` +#[derive(Debug)] +#[repr(C, align(4096))] +pub struct BootInfo { + pub graphics_info: GraphicsInfo, + pub device_tree: DeviceTree, + + /// Used to calculate slide for virtual mapping + pub kernel_elf_base: PAddr, + pub kernel_virt_base: usize, + + pub memory_regions: ArrayVec, + pub identity_maps: ArrayVec, + pub scratch_pages: ArrayVec, + pub relocations: ArrayVec, } \ No newline at end of file diff --git a/bootinfo/src/paddr.rs b/bootinfo/src/paddr.rs index 47ac40c..2653d72 100644 --- a/bootinfo/src/paddr.rs +++ b/bootinfo/src/paddr.rs @@ -24,12 +24,12 @@ impl PAddr { impl fmt::Debug for PAddr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{self}") + fmt::Pointer::fmt(self, f) } } -impl fmt::Display for PAddr { +impl fmt::Pointer for PAddr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:#018p}", self.0 as *const ()) + write!(f, "{:#018x}", self.0) } } diff --git a/bootinfo/src/vaddr/x86_64/virt.rs b/bootinfo/src/vaddr/x86_64/virt.rs deleted file mode 100644 index 360a296..0000000 --- a/bootinfo/src/vaddr/x86_64/virt.rs +++ /dev/null @@ -1,383 +0,0 @@ -use core::{cell::RefCell, fmt, hint::unreachable_unchecked, ops::DerefMut}; - -use bytemuck::Zeroable; - -use crate::{ - paddr::PAddr, - vaddr::{Entry, NodeEntry, PdEntry, PdTable, PdptEntry, PdptTable, Pml4Table, PtEntry, PtTable, UnionEntry, VFlags, VirtualMapError}, -}; - -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Zeroable)] -#[repr(transparent)] -pub struct VAddr(usize); -impl VAddr { - #[inline] - pub const fn new(virtual_address: usize) -> Self { - Self(virtual_address) - } - - #[inline] - pub const fn addr(self) -> usize { - self.0 - } - - #[inline] - pub const fn ptr(self) -> *const T { - self.0 as *const T - } - - #[inline] - pub const fn ptr_mut(self) -> *mut T { - self.0 as *mut T - } - - #[inline] - pub(crate) const fn info(self) -> VAddrInfo { - VAddrInfo { - page_offset: self.0 & 0xfff, - pt_index: (self.0 >> 12) & 0x1ff, - pd_index: (self.0 >> 21) & 0x1ff, - pdpt_index: (self.0 >> 30) & 0x1ff, - pml4_index: (self.0 >> 39) & 0x1ff, - } - } - - #[inline] - pub(crate) const fn from_info( - VAddrInfo { - page_offset, - pt_index, - pd_index, - pdpt_index, - pml4_index, - }: VAddrInfo, - ) -> Self { - let addr = - page_offset & 0xfff | (pt_index & 0x1ff) << 12 | (pd_index & 0x1ff) << 21 | (pdpt_index & 0x1ff) << 30 | (pml4_index & 0x1ff) << 39; - Self((addr.cast_signed() << 16 >> 16).cast_unsigned()) - } -} - -#[derive(Debug, Clone, Copy, Zeroable)] -#[repr(C)] -pub(crate) struct VAddrInfo { - pub page_offset: usize, - pub pt_index: usize, - pub pd_index: usize, - pub pdpt_index: usize, - pub pml4_index: usize, -} - -impl fmt::Debug for VAddr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.info()) - } -} - -impl fmt::Display for VAddr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:#018p}", self.0 as *const ()) - } -} - -#[derive(Debug)] -#[repr(C)] -pub struct VirtualMapBuilder { - map: VirtualMap, - new_page_table: fn() -> Option, -} - -impl VirtualMapBuilder { - /// # Safety - /// - `new_page_table` must return a [`PAGE_SIZE`](crate::PAGE_SIZE)-aligned physical address - /// that is: - /// - Completely zeroed out. - /// - Completely free to be written to (nothing else "owns" it). - /// - `page_table_ptr` must convert physical addresses returned by `new_page_table` into a - /// pointer that points to a page table. - #[inline] - pub const unsafe fn new(recursion_index: usize, new_page_table: fn() -> Option, page_table_ptr: unsafe fn(PAddr) -> *mut ()) -> Self { - Self { - map: VirtualMap { - mapper: LocalMapper { - table: RefCell::new(bytemuck::zeroed()), - page_table_ptr, - recursion_index, - }, - }, - new_page_table, - } - } - - #[inline] - pub fn map(&mut self, p_addr: PAddr, v_addr: VAddr, flags: VFlags) -> Result<(), VirtualMapError> { - unsafe { self.map.map(p_addr, v_addr, flags, self.new_page_table) } - } - - pub fn finish(self) -> Result<(PAddr, VirtualMap), VirtualMapError> { - let Self { - map: VirtualMap { mut mapper }, - new_page_table, - } = self; - - let pml4_phys = (new_page_table)().ok_or(VirtualMapError::PageTable)?; - match mapper.table.get_mut().pdpt_entries[mapper.recursion_index] { - e if e.is_present() => unreachable!("`recursion_index` is checked in earlier methods"), - ref mut e => *e = unsafe { NodeEntry::new(Entry::WRITABLE, pml4_phys) }, - } - - unsafe { - (mapper.page_table_ptr)(pml4_phys).cast::().write(mapper.table.into_inner()); - } - - Ok((pml4_phys, VirtualMap { - mapper: unsafe { sealed::RecursiveMapper::new(mapper.recursion_index) }, - })) - } -} - -#[derive(Debug)] -#[repr(C)] -pub struct LocalMapper { - table: RefCell, - page_table_ptr: unsafe fn(PAddr) -> *mut (), - recursion_index: usize, -} - -unsafe impl sealed::VirtualMapper for LocalMapper { - #[inline] - fn reserved(&self, v_addr: VAddr) -> bool { - let VAddrInfo { pml4_index, .. } = v_addr.info(); - pml4_index == self.recursion_index - } - - #[inline] - fn pml4(&self) -> impl DerefMut { - self.table.borrow_mut() - } - - #[inline] - unsafe fn pdpt(&self, pml4_index: usize) -> &mut PdptTable { - unsafe { - (self.page_table_ptr)(self.table.borrow_mut().pdpt_entries.get(pml4_index).unwrap_unchecked().child_addr()) - .cast::() - .as_mut_unchecked() - } - } - - #[inline] - unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> &mut PdTable { - unsafe { - (self.page_table_ptr)(match self.pdpt(pml4_index).pd_entries.get(pdpt_index).unwrap_unchecked().kind() { - UnionEntry::Node(e) => e.child_addr(), - UnionEntry::Leaf(..) => unreachable_unchecked(), - }) - .cast::() - .as_mut_unchecked() - } - } - - #[inline] - unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> &mut PtTable { - unsafe { - (self.page_table_ptr)(match self.pd(pml4_index, pdpt_index).pt_entries.get(pd_index).unwrap_unchecked().kind() { - UnionEntry::Node(e) => e.child_addr(), - UnionEntry::Leaf(..) => unreachable_unchecked(), - }) - .cast::() - .as_mut_unchecked() - } - } -} - -#[derive(Debug)] -#[repr(transparent)] -// Note: From user-facing API perspective, `VirtualMap` must have no trait bounds. -pub struct VirtualMap { - mapper: T, -} - -impl VirtualMap { - /// # Safety - /// - `new_page_table` must return a [`PAGE_SIZE`](crate::PAGE_SIZE)-aligned physical address - /// that is: - /// - Completely zeroed out. - /// - Completely free to be written to (nothing else "owns" it). - /// - There must never be concurrent (multithreaded) calls to this method that have the same - /// virtual page occupied by `v_addr`. - pub unsafe fn map( - &self, - p_addr: PAddr, - v_addr: VAddr, - flags: VFlags, - mut new_page_table: impl FnMut() -> Option, - ) -> Result<(), VirtualMapError> { - if self.mapper.reserved(v_addr) { - return Err(VirtualMapError::Reserved { p_addr, v_addr }) - } - - let VAddrInfo { - pt_index, - pd_index, - pdpt_index, - pml4_index, - .. - } = v_addr.info(); - unsafe { - match &mut self.mapper.pml4().pdpt_entries[pml4_index] { - e if !e.is_present() => *e = NodeEntry::new(Entry::WRITABLE, new_page_table().ok_or(VirtualMapError::PageTable)?), - _ => {} - } - - match &mut self.mapper.pdpt(pml4_index).pd_entries[pdpt_index] { - e if !e.is_present() => *e = PdptEntry::node(NodeEntry::new(Entry::WRITABLE, new_page_table().ok_or(VirtualMapError::PageTable)?)), - e if let UnionEntry::Leaf(e) = e.kind() => { - return Err(VirtualMapError::AlreadyMapped { - p_addr, - v_addr, - p_addr_existing: e.addr(), - }) - } - _ => {} - } - - match &mut self.mapper.pd(pml4_index, pdpt_index).pt_entries[pd_index] { - e if !e.is_present() => *e = PdEntry::node(NodeEntry::new(Entry::WRITABLE, new_page_table().ok_or(VirtualMapError::PageTable)?)), - e if let UnionEntry::Leaf(e) = e.kind() => { - return Err(VirtualMapError::AlreadyMapped { - p_addr, - v_addr, - p_addr_existing: e.addr(), - }) - } - _ => {} - } - - match &mut self.mapper.pt(pml4_index, pdpt_index, pd_index).phys_pages[pt_index] { - e if e.is_present() => { - return Err(VirtualMapError::AlreadyMapped { - p_addr, - v_addr, - p_addr_existing: e.addr(), - }) - } - e => *e = PtEntry::new(flags.into(), p_addr) | if flags.contains(VFlags::GLOBAL) { PtEntry::GLOBAL } else { PtEntry::empty() }, - } - } - - Ok(()) - } -} - -mod sealed { - use super::*; - - #[allow(unused_variables, reason = "Available for implementors, not defaults")] - pub unsafe trait VirtualMapper { - #[inline] - fn reserved(&self, v_addr: VAddr) -> bool { - false - } - - fn pml4(&self) -> impl DerefMut; - - /// # Safety - /// - [`pml4_index`] must be within `0..512` (exclusive). - unsafe fn pdpt(&self, pml4_index: usize) -> &mut PdptTable; - - /// # Safety: - /// - [`Self::pdpt()`] to the given indices must return a node entry, not leaf. - /// - [`pml4_index`] and [`pdpt_index`] must be within `0..512` (exclusive). - unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> &mut PdTable; - - /// # Safety: - /// - [`Self::pd()`] to the given indices must return a node entry, not leaf. - /// - [`pml4_index`], [`pdpt_index`], and [`pd_index`] must be within `0..512` (exclusive). - unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> &mut PtTable; - } - - #[derive(Debug)] - #[repr(transparent)] - pub struct RecursiveMapper { - recursion_index: usize, - } - - impl RecursiveMapper { - /// # Safety: - /// `recursion_index` must be N where - /// [`pdpt_entries[N]`](crate::vaddr::Pml4Table::pdpt_entries) points to the - /// physical address of the PML4 table itself (i.e. recursive slot). - #[inline] - pub const unsafe fn new(recursion_index: usize) -> Self { - Self { recursion_index } - } - } - - unsafe impl VirtualMapper for RecursiveMapper { - #[inline] - fn reserved(&self, v_addr: VAddr) -> bool { - let VAddrInfo { pml4_index, .. } = v_addr.info(); - pml4_index == self.recursion_index - } - - #[inline] - fn pml4(&self) -> impl DerefMut { - unsafe { - VAddr::from_info(VAddrInfo { - page_offset: 0, // Keep recursing so it ends up with the PML4 table itself - pt_index: self.recursion_index, - pd_index: self.recursion_index, - pdpt_index: self.recursion_index, - pml4_index: self.recursion_index, - }) - .ptr_mut::() - .as_mut_unchecked() - } - } - - #[inline] - unsafe fn pdpt(&self, pml4_index: usize) -> &mut PdptTable { - unsafe { - VAddr::from_info(VAddrInfo { - page_offset: 0, // Stop recursing at PT index so it ends up with the PDPT entry - pt_index: pml4_index, - pd_index: self.recursion_index, - pdpt_index: self.recursion_index, - pml4_index: self.recursion_index, - }) - .ptr_mut::() - .as_mut_unchecked() - } - } - - #[inline] - unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> &mut PdTable { - unsafe { - VAddr::from_info(VAddrInfo { - page_offset: 0, // Stop recursing at PD index so it ends up with the PD entry - pt_index: pdpt_index, - pd_index: pml4_index, - pdpt_index: self.recursion_index, - pml4_index: self.recursion_index, - }) - .ptr_mut::() - .as_mut_unchecked() - } - } - - #[inline] - unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> &mut PtTable { - unsafe { - VAddr::from_info(VAddrInfo { - page_offset: 0, // Stop recursing at PDPT index so it ends up with the PT entry - pt_index: pd_index, - pd_index: pdpt_index, - pdpt_index: pml4_index, - pml4_index: self.recursion_index, - }) - .ptr_mut::() - .as_mut_unchecked() - } - } - } -} diff --git a/bootloader/Cargo.toml b/bootloader/Cargo.toml index 2e655d7..06465e8 100644 --- a/bootloader/Cargo.toml +++ b/bootloader/Cargo.toml @@ -12,7 +12,6 @@ forced-target = "x86_64-unknown-uefi" elpytios-elf = { path = "../elf" } elpytios-bootinfo = { path = "../bootinfo" } -bytemuck = { version = "1", features = ["derive", "latest_stable_rust"] } +arrayvec = { version = "0.7", default-features = false } const_panic = { version = "0.2", features = ["derive", "rust_latest_stable"] } -log = "0.4" -uefi = { version = "0.37", features = ["logger", "panic_handler"] } +uefi = { version = "0.38", features = ["logger", "panic_handler"] } diff --git a/bootloader/src/lib.rs b/bootloader/src/lib.rs deleted file mode 100644 index 6f2b4ee..0000000 --- a/bootloader/src/lib.rs +++ /dev/null @@ -1,3 +0,0 @@ -#![no_std] - -pub mod region_grouper; diff --git a/bootloader/src/main.rs b/bootloader/src/main.rs index bc77cb2..36a58a2 100644 --- a/bootloader/src/main.rs +++ b/bootloader/src/main.rs @@ -1,21 +1,27 @@ -#![feature(const_cmp, const_convert, const_iter, const_trait_impl, custom_inner_attributes, fn_align)] +#![feature(const_cmp, const_convert, const_iter, const_trait_impl, custom_inner_attributes)] #![rustfmt::skip] #![no_std] #![no_main] -use core::{arch::naked_asm, mem::{self, MaybeUninit}, slice}; +use core::{arch::asm, mem::{self, MaybeUninit}}; +use arrayvec::ArrayVec; use const_panic::concat_panic; -use elpytios_elf::{Elf, Elf64, ElfSegment64, ElfSegmentType, sys::ElfProgramFlags}; -use elpytios_bootinfo::{BootInfo, GraphicsInfo, MAX_MEMORY_REGIONS, MemoryRegion, paddr::PAddr, vaddr::{VAddr, VFlags, VirtualMapBuilder}}; -use uefi::{Status, boot::{self, AllocateType, MemoryType}, entry, helpers, mem::memory_map::{MemoryMap, MemoryMapOwned}, proto::console::gop::*}; +use elpytios_elf::{Elf, Elf64, ElfSegment64, ElfSegmentType, sys::{ElfProgramFlags, ElfRela64, ElfRela64Type, ElfType}}; +use elpytios_bootinfo::{BootInfo, DeviceTree, GraphicsInfo, IdentityMap, IdentityMapFlags, MAX_SCRATCH, MemoryRegion, PAGE_SIZE, Reloc, paddr::PAddr}; +use uefi::{Status, boot::{self, AllocateType, MemoryType}, entry, helpers, mem::memory_map::MemoryMap, proto::console::gop::*, table::cfg::ConfigTableEntry}; -const PAGE_SIZE: usize = 4096; +const _: () = assert!(PAGE_SIZE == boot::PAGE_SIZE); -const ELPYTI_KERNEL_CODE: MemoryType = MemoryType::custom(0x8000_0000); -const ELPYTI_KERNEL_STACK: MemoryType = MemoryType::custom(0x8000_0001); -const ELPYTI_PAGE_TABLE: MemoryType = MemoryType::custom(0x8000_0002); +const MEM_KERNEL_CODE: MemoryType = MemoryType::custom(0x8000_0000); +const MEM_STACK: MemoryType = MemoryType::custom(0x8000_0001); +const MEM_BOOT_INFO: MemoryType = MemoryType::custom(0x8000_0002); +const MEM_SCRATCH: MemoryType = MemoryType::custom(0x8000_0003); + +const MEM_STACK_LEN: usize = 16; // Note: `opt-level = 0` makes the code consume way too much stack space +const MEM_BOOT_INFO_LEN: usize = size_of::().div_ceil(PAGE_SIZE); +const MEM_SCRATCH_LEN: usize = MAX_SCRATCH; const KERNEL_BINARY: Elf64 = match Elf::from_bytes(include_bytes!(concat!("../../target/x86_64-unknown-none/", cfg_select! { debug_assertions => "bootloader_debug", @@ -27,6 +33,11 @@ const KERNEL_BINARY: Elf64 = match Elf::from_bytes(include_bytes!(concat!("../.. }; const KERNEL_SEGMENTS: [ElfSegment64; KERNEL_BINARY.program_header_count()] = { + // Force `-fPIE` in the kernel + if !matches!(KERNEL_BINARY.prologue().elf_type, ElfType::DYNAMIC) { + panic!("Invalid kernel ELF type, must be ElfType::DYNAMIC (3)") + } + let mut out: MaybeUninit<[ElfSegment64; _]> = MaybeUninit::uninit(); let mut ptr = out.as_mut_ptr() as *mut ElfSegment64; @@ -36,7 +47,7 @@ const KERNEL_SEGMENTS: [ElfSegment64; KERNEL_BINARY.program_header_count()] = { Err(e) => concat_panic!(e), }; - if segment.alignment != PAGE_SIZE as u64 { + if let ElfSegmentType::Load(..) = segment.segment_type && segment.alignment != PAGE_SIZE as u64 { concat_panic!("Kernel segments must be aligned to ", PAGE_SIZE, "! Found: ", segment.alignment); } @@ -49,53 +60,31 @@ const KERNEL_SEGMENTS: [ElfSegment64; KERNEL_BINARY.program_header_count()] = { unsafe { out.assume_init() } }; -const KERNEL_BOOTINFO_ADDRESS: usize = { - let mut ret = 0; - for section in KERNEL_BINARY.sections() { - let section = match section { - Ok(section) => section, - Err(e) => concat_panic!(e), - }; - - if section.name(&KERNEL_BINARY) == b".bootinfo" { - ret = match usize::try_from(section.virtual_address) { - Ok(max) => max.next_multiple_of(PAGE_SIZE), - _ => concat_panic!("Integer doesn't fit: ", section.virtual_address), - }; - break - } - } - - if ret == 0 { - panic!("`.bootinfo` section not found") - } else if ret % PAGE_SIZE != 0 { - concat_panic!("`.bootinfo` section not aligned: ", ret) - } else { - ret - } -}; - /// Index 0: Lowest virtual address of the kernel. -/// Index 1: New virtual addresses can take this spot. +/// Index 1: Highest virtual address of the kernel, page-aligned. const KERNEL_VIRTUAL_ADDRESSES: [usize; 2] = { let mut min = u64::MAX; let mut max = u64::MIN; let mut i = 0; loop { - let segment = KERNEL_SEGMENTS[i]; - if segment.segment_type != ElfSegmentType::Load { continue } + if i == KERNEL_SEGMENTS.len() { break } + + let segment = &KERNEL_SEGMENTS[i]; + if !matches!(segment.segment_type, ElfSegmentType::Load(..)) { + i += 1; + continue + } min = min.min(segment.virtual_address); max = max.max(segment.virtual_address + segment.memory_size); i += 1; - if i == KERNEL_SEGMENTS.len() { break } } match (usize::try_from(min), usize::try_from(max)) { (Ok(min), Ok(max)) => [ - if min % PAGE_SIZE == 0 { + if min.is_multiple_of(PAGE_SIZE) { min } else { concat_panic!("Virtual address base (", min, ") isn't aligned to ", PAGE_SIZE) @@ -106,40 +95,24 @@ const KERNEL_VIRTUAL_ADDRESSES: [usize; 2] = { } }; -const KERNEL_STACK_PAGES: usize = 8; - struct UefiInfo { - pub memory_map: MemoryMapOwned, - pub pml4_phys: PAddr, - - pub kernel_stack_base: VAddr, - pub kernel_entry: VAddr, + pub kernel_entry: *mut u8, + pub kernel_stack_base: *mut u8, + pub boot_info: *mut BootInfo, } fn setup_uefi_and_exit() -> UefiInfo { - let memory_map: MemoryMapOwned; - let pml4_phys: PAddr; + let kernel_entry: *mut u8; + let kernel_stack_base: *mut u8; + let boot_info: *mut BootInfo; - let boot_info_ptr: *mut BootInfo; - let kernel_stack_base: VAddr; - let kernel_entry: VAddr; - helpers::init().unwrap(); { // Graphics Info Fetching - let graphics_output_protocol_handle = boot::get_handle_for_protocol::().expect("No Graphics Output Protocol"); - let mut graphics_output_protocol; - unsafe { - graphics_output_protocol = boot::open_protocol::( - boot::OpenProtocolParams { - handle: graphics_output_protocol_handle, - agent: boot::image_handle(), - controller: None - }, - boot::OpenProtocolAttributes::GetProtocol - ).expect("Error opening Graphics Output Protocol"); - } + let mut graphics_output_protocol = boot::open_protocol_exclusive::( + boot::get_handle_for_protocol::().expect("No Graphics Output Protocol") + ).expect("Error opening Graphics Output Protocol"); let mut max_area: usize = 0; let mut max_mode: Option = None; @@ -169,121 +142,133 @@ fn setup_uefi_and_exit() -> UefiInfo { let pixel_format = mode_info.pixel_format(); let frame_buffer_ptr = frame_buffer.as_mut_ptr(); - let mut virtual_map = unsafe { - VirtualMapBuilder::new( - // 511 used for higher-half addressing - 510, - // Allocate 1 page via UEFI's allocator - || boot::allocate_pages(AllocateType::AnyPages, ELPYTI_PAGE_TABLE, 1).ok().map(|ptr| { - let ptr = ptr.as_ptr(); - ptr.write_bytes(0, PAGE_SIZE); - PAddr::new(ptr.addr()) - }), - // Identity mapping is still enabled at this point - |ptr| ptr.addr() as *mut (), - ) + let graphics_info = GraphicsInfo { + w, + h, + stride, + pixel_format: match pixel_format { + PixelFormat::Rgb => elpytios_bootinfo::PixelFormat::RGB_8_BIT, + PixelFormat::Bgr => elpytios_bootinfo::PixelFormat::BGR_8_BIT, + PixelFormat::Bitmask => elpytios_bootinfo::PixelFormat::BIT_MASK, + PixelFormat::BltOnly => elpytios_bootinfo::PixelFormat::BLT_ONLY + }, + frame_buffer: PAddr::new(frame_buffer_ptr.addr()), + frame_buffer_size: frame_buffer_size, }; - let [virtual_base, mut next_v_addr] = KERNEL_VIRTUAL_ADDRESSES; - let kernel_ptr = boot::allocate_pages(AllocateType::AnyPages, ELPYTI_KERNEL_CODE, (next_v_addr - virtual_base) / PAGE_SIZE).unwrap().as_ptr(); + let [virtual_base, virtual_max] = KERNEL_VIRTUAL_ADDRESSES; + let mut identity_maps = ArrayVec::new(); + + let kernel_base_pages = (virtual_max - virtual_base) / PAGE_SIZE; + let kernel_ptr = boot::allocate_pages(AllocateType::MaxAddress(0x40_0000), MEM_KERNEL_CODE, kernel_base_pages).unwrap().as_ptr(); + for segment in KERNEL_SEGMENTS { - if segment.segment_type != ElfSegmentType::Load { continue } + let ElfSegmentType::Load(data) = segment.segment_type else { continue }; + identity_maps.push(IdentityMap::new( + PAddr::new(kernel_ptr.addr() + segment.virtual_address as usize - virtual_base), + (segment.memory_size as usize).div_ceil(PAGE_SIZE), + { + let mut flags = IdentityMapFlags::empty(); + if segment.flags.contains(ElfProgramFlags::EXECUTABLE) { flags |= IdentityMapFlags::EXECUTABLE } + if segment.flags.contains(ElfProgramFlags::READABLE) { flags |= IdentityMapFlags::READABLE } + if segment.flags.contains(ElfProgramFlags::WRITABLE) { flags |= IdentityMapFlags::WRITABLE } + + flags + }, + )); + unsafe { kernel_ptr .add(segment.virtual_address as usize - virtual_base) - .copy_from_nonoverlapping(segment.data.as_ptr(), segment.data.len()); + .copy_from_nonoverlapping(data.as_ptr(), data.len()); kernel_ptr - .add(segment.virtual_address as usize - virtual_base + segment.data.len()) - .add(segment.data.len()).write_bytes(0, segment.memory_size as usize - segment.data.len()); + .add(segment.virtual_address as usize - virtual_base + data.len()) + .write_bytes(0, segment.memory_size as usize - data.len()); } + } - for i in (0..segment.memory_size as usize).step_by(PAGE_SIZE) { - virtual_map.map( - PAddr::new(unsafe { kernel_ptr.add(segment.virtual_address as usize - virtual_base).addr() } + i), - VAddr::new(segment.virtual_address as usize + i), - VFlags::GLOBAL | match segment.flags.contains(ElfProgramFlags::WRITABLE) { - false => VFlags::empty(), - true => VFlags::WRITABLE, - }, - ).unwrap_or_else(|e| panic!("{e}")); + let mut relocations = ArrayVec::new(); + for segment in KERNEL_SEGMENTS { + let ElfSegmentType::Dynamic { offset, size, stride } = segment.segment_type else { continue }; + relocations.push(Reloc { offset, size, stride }); + for i in 0..size / stride { + unsafe { + let rela = kernel_ptr.cast::().byte_add(offset - virtual_base).add(i).read_unaligned(); + match rela.info.kind { + ElfRela64Type::X86_64_NONE => {} + ElfRela64Type::X86_64_RELATIVE => { + let slide = kernel_ptr.addr() as i64 - virtual_base as i64; + let patch_addr = kernel_ptr.add(rela.offset as usize - virtual_base); + let value = slide + rela.addend; + patch_addr.cast::().write(value); + } + kind => panic!("Unsupported Elf64_Rela kind: {}", kind.0), + } + } } } - kernel_entry = VAddr::new(KERNEL_BINARY.program_entry() as usize); - - // Identity-map the kernel switcher - let switcher_addr = (switch_to_kernel as *const ()).addr(); - assert_eq!(switcher_addr % PAGE_SIZE, 0, "`switch_to_kernel` must be page-aligned"); - virtual_map.map(PAddr::new(switcher_addr), VAddr::new(switcher_addr), VFlags::empty()).unwrap_or_else(|e| panic!("{e}")); - - let mut next_free_page = |p_addr: PAddr, page_count: usize, flags: VFlags| { - let v_addr = next_v_addr; - next_v_addr += page_count * PAGE_SIZE; - - for i in 0..page_count { - let offset = i * PAGE_SIZE; - virtual_map.map( - PAddr::new(p_addr.addr() + offset), - VAddr::new(v_addr + offset), - VFlags::GLOBAL | flags, - ).unwrap_or_else(|e| panic!("{e}")); - } - VAddr::new(v_addr) - }; + let stack_ptr = boot::allocate_pages(AllocateType::MaxAddress(0x40_0000), MEM_STACK, MEM_STACK_LEN).unwrap().as_ptr(); + identity_maps.push(IdentityMap::new(PAddr::new(stack_ptr.addr() + PAGE_SIZE), MEM_STACK_LEN - 1, IdentityMapFlags::READABLE | IdentityMapFlags::WRITABLE)); - // Map the stack pointer - let stack_ptr = boot::allocate_pages(AllocateType::AnyPages, ELPYTI_KERNEL_STACK, KERNEL_STACK_PAGES).unwrap().as_ptr(); - let stack = next_free_page(PAddr::new(stack_ptr.addr()), KERNEL_STACK_PAGES, VFlags::WRITABLE); - kernel_stack_base = VAddr::new(stack.addr() + KERNEL_STACK_PAGES * PAGE_SIZE); + kernel_entry = unsafe { kernel_ptr.add(KERNEL_BINARY.program_entry() as usize - virtual_base) }; + kernel_stack_base = unsafe { stack_ptr.add(MEM_STACK_LEN * PAGE_SIZE) }; - // Map the framebuffer - let fb_phys = frame_buffer_ptr.addr(); - let fb_size = frame_buffer_size; + boot_info = boot::allocate_pages(AllocateType::MaxAddress(0x40_0000), MEM_BOOT_INFO, MEM_BOOT_INFO_LEN).unwrap().as_ptr().cast(); + identity_maps.push(IdentityMap::new(PAddr::new(boot_info.addr()), MEM_BOOT_INFO_LEN, IdentityMapFlags::READABLE)); - let fb_phys_base = fb_phys & !(PAGE_SIZE - 1); - let fb_phys_end = (fb_phys + fb_size).next_multiple_of(PAGE_SIZE); - let fb_page_count = (fb_phys_end - fb_phys_base) / PAGE_SIZE; + unsafe { + let scratch_ptr = boot::allocate_pages(AllocateType::MaxAddress(1 << 16), MEM_SCRATCH, MEM_SCRATCH_LEN).unwrap().as_ptr(); + scratch_ptr.write_bytes(0, MEM_SCRATCH_LEN * PAGE_SIZE); + let mut scratch_pages = ArrayVec::new(); + for i in 0..MEM_SCRATCH_LEN { + scratch_pages.push(PAddr::new(scratch_ptr.addr() + i * PAGE_SIZE)); + } - let graphics_info = GraphicsInfo { - w, - h, - stride, - pixel_format: match pixel_format { - PixelFormat::Rgb => elpytios_bootinfo::PixelFormat::RGB_8_BIT, - PixelFormat::Bgr => elpytios_bootinfo::PixelFormat::BGR_8_BIT, - PixelFormat::Bitmask => elpytios_bootinfo::PixelFormat::BIT_MASK, - PixelFormat::BltOnly => elpytios_bootinfo::PixelFormat::BLT_ONLY - }, - frame_buffer: next_free_page(PAddr::new(fb_phys), fb_page_count, VFlags::WRITABLE | VFlags::WRITE_THROUGH | VFlags::CACHE_DISABLED).ptr_mut(), - frame_buffer_size: frame_buffer_size, - }; + let device_tree = uefi::system::with_config_table(|slice| { + let mut out = None; + for i in slice { + match i.guid { + ConfigTableEntry::ACPI_GUID if out.is_none() => out = Some(DeviceTree::Acpi(PAddr::new(i.address.addr()))), + ConfigTableEntry::ACPI2_GUID => { + out = Some(DeviceTree::Acpi2(PAddr::new(i.address.addr()))); + break + }, + _ => {} + } + } - let (pml4_phys_ret, virtual_map) = virtual_map.finish().unwrap(); - pml4_phys = pml4_phys_ret; - unsafe { - boot_info_ptr = kernel_ptr.add(KERNEL_BOOTINFO_ADDRESS - virtual_base).cast(); - boot_info_ptr.write(BootInfo { + out.expect("No ACPI or ACPI2 table found") + }); + + boot_info.write(BootInfo { graphics_info, - virtual_map, - switcher_map: VAddr::new(switcher_addr), + device_tree, - // Initialized after exiting UEFI boot services - memory_regions_base: [MaybeUninit::uninit(); _], - memory_regions_size: 0, + kernel_elf_base: PAddr::new(kernel_ptr.addr()), + kernel_virt_base: virtual_base, - v_addr_start: VAddr::new(next_v_addr), - v_addr_end: VAddr::new(usize::MAX), + memory_regions: ArrayVec::new(), + identity_maps, + scratch_pages, + relocations, }); } } unsafe { - memory_map = boot::exit_boot_services(Some(MemoryType::LOADER_DATA)); + let memory_map = boot::exit_boot_services(Some(MemoryType::LOADER_DATA)); + for entry in memory_map.entries() { + if matches!(entry.ty, MemoryType::ACPI_RECLAIM) { + (*boot_info).identity_maps.push(IdentityMap { + region: MemoryRegion::at(PAddr::new(entry.phys_start as usize), entry.page_count as usize), + flags: IdentityMapFlags::READABLE, + }); + } + } - let mut len = 0; let mut region = None; - for entry in memory_map.entries() { if !matches!(entry.ty, MemoryType::LOADER_CODE | MemoryType::LOADER_DATA | @@ -297,70 +282,46 @@ fn setup_uefi_and_exit() -> UefiInfo { let count = entry.page_count as usize; match region.as_mut() { - None => region = Some(MemoryRegion { - base: PAddr::new(start), - pages: count, - }), + None => region = Some(MemoryRegion::at(PAddr::new(start), count)), Some(reg) => { if reg.base.addr() + reg.pages * PAGE_SIZE == start { reg.pages += count; - } else if reg.pages >= 8 { - (&raw mut (*boot_info_ptr).memory_regions_base[len]) - .cast::() - .write(mem::replace(reg, MemoryRegion { - base: PAddr::new(start), - pages: count, - })); - - len += 1; - if len == MAX_MEMORY_REGIONS { + } else { + if (*boot_info).memory_regions.try_push(mem::replace(reg, MemoryRegion::at(PAddr::new(start), count))).is_err() { break } - } else { - region = None; } } } } - if len < MAX_MEMORY_REGIONS && let Some(region) = region { - (&raw mut (*boot_info_ptr).memory_regions_base[len]).cast::().write(region); - len += 1; + if let Some(region) = region { + _ = (*boot_info).memory_regions.try_push(region); } - slice::from_raw_parts_mut(&raw mut (*boot_info_ptr).memory_regions_base as *mut MemoryRegion, len) - .sort_unstable_by(|a, b| b.pages.cmp(&a.pages)); - - (&raw mut (*boot_info_ptr).memory_regions_size).write(len); + (*boot_info).memory_regions.sort_unstable_by(|a, b| b.pages.cmp(&a.pages)); } UefiInfo { - memory_map, - pml4_phys, - kernel_stack_base, kernel_entry, + boot_info, } } -#[rustc_align(4096)] -#[unsafe(naked)] -unsafe extern "sysv64" fn switch_to_kernel( - pml4_phys: usize, - stack_base: usize, - kernel_entry: usize, -) -> ! { - naked_asm!( - "mov cr3, rdi", - "lea rsp, [rsi - 8]", - "jmp rdx", - ) -} - #[entry] fn entry() -> Status { - let UefiInfo { memory_map, pml4_phys, kernel_stack_base, kernel_entry } = setup_uefi_and_exit(); + let UefiInfo { kernel_entry, kernel_stack_base, boot_info } = setup_uefi_and_exit(); unsafe { - switch_to_kernel(pml4_phys.addr(), kernel_stack_base.addr(), kernel_entry.addr()) + asm!( + "movq {kernel_stack_base}, %rsp", + "jmpq *{kernel_entry}", + + in("rdi") boot_info, + kernel_stack_base = in(reg) kernel_stack_base, + kernel_entry = in(reg) kernel_entry, + + options(att_syntax, noreturn) + ) } } diff --git a/bootloader/src/region_grouper.rs b/bootloader/src/region_grouper.rs deleted file mode 100644 index 26d62e5..0000000 --- a/bootloader/src/region_grouper.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub struct RegionGrouper { - -} - diff --git a/elf/src/lib.rs b/elf/src/lib.rs index 4c364d4..b47427d 100644 --- a/elf/src/lib.rs +++ b/elf/src/lib.rs @@ -2,7 +2,7 @@ //! //! ``` //! pub fn load_kernel_code() -> Result<(), ElfError> { -//! let kernel_code = include_bytes!("../Cargo.toml"); +//! let kernel_code = include_bytes!("path/to/binary.elf"); //! //! match Elf::from_bytes(kernel_code)? { //! Elf::N32(..) => unreachable!("kernel ELF is 64-bits, silly"), @@ -11,7 +11,6 @@ //! let segment = segment?; //! //! segment.segment_type; // `ElfSegmentType`: null, load, dynamic, interp, and note -//! segment.data; // `&[u8]`, program segment data //! segment.flags; // `ElfProgramFlags`: 1 = executable, 2 = writable, 4 = readable //! segment.virtual_address; // `usize`, virtual address that `segment_data` should be copied into //! segment.physical_address; // `usize`, physical address that `segment_data` could be copied into, usually ignored @@ -28,7 +27,6 @@ #![no_std] #![feature( const_clone, - const_cmp, const_convert, const_destruct, const_index, @@ -49,9 +47,9 @@ use bytemuck::AnyBitPattern; use const_panic::PanicFmt; use sys::{ElfHeader64, ElfHeaderPrologue, ElfProgramFlags, ElfProgramHeader64}; -use crate::sys::ElfSectionHeader64; +use crate::sys::{ElfDt64, ElfDyn64, ElfSectionHeader64}; -#[derive(Debug, Clone, Copy, PanicFmt)] +#[derive(Clone, Copy, PanicFmt)] pub enum ElfError { InvalidMagic([u8; 4]), InvalidArch(u8), @@ -59,9 +57,31 @@ pub enum ElfError { InvalidSegmentType(u32), IntDoesntFit, MissingStringTable, + MalformedDynHeader(&'static str), Eof, } +impl fmt::Debug for ElfError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for ElfError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidMagic(..) => write!(f, "Invalid ELF magic number"), + Self::InvalidArch(..) => write!(f, "Invalid ELF architecture"), + Self::InvalidEndian(..) => write!(f, "Invalid ELF endianness"), + Self::InvalidSegmentType(..) => write!(f, "Invalid ELF segment type"), + Self::IntDoesntFit => write!(f, "Elf integer doesn't fit"), + Self::MissingStringTable => write!(f, "Missing ELF string table"), + Self::MalformedDynHeader(_) => write!(f, "Malformed ELF DYN header"), + Self::Eof => write!(f, "End-of-File while parsing ELF"), + } + } +} + #[inline] const fn int_fit>(from: T) -> Result { match from.try_into() { @@ -96,7 +116,7 @@ impl<'a> Elf<'a> { match prologue.arch { 1 => panic!("32-bit ELF isn't supported yet"), - 2 => Ok(Self::N64(Elf64::from_bytes(file_reader, header_reader)?)), + 2 => Ok(Self::N64(Elf64::from_bytes(prologue, file_reader, header_reader)?)), arch => Err(ElfError::InvalidArch(arch)), } } @@ -104,6 +124,7 @@ impl<'a> Elf<'a> { #[derive(Debug)] pub struct Elf64<'a> { + prologue: ElfHeaderPrologue, header: ElfHeader64, file_reader: Reader<'a>, program_table_reader: Reader<'a>, @@ -112,7 +133,7 @@ pub struct Elf64<'a> { } impl<'a> Elf64<'a> { - const fn from_bytes(file_reader: Reader<'a>, mut header_reader: Reader<'a>) -> Result { + const fn from_bytes(prologue: ElfHeaderPrologue, file_reader: Reader<'a>, mut header_reader: Reader<'a>) -> Result { let header = header_reader.read::().ok_or(ElfError::Eof)?; let program_table_reader = file_reader.fork(int_fit(header.program_header_table_offset)?).ok_or(ElfError::Eof)?; let section_table_reader = file_reader.fork(int_fit(header.section_header_table_offset)?).ok_or(ElfError::Eof)?; @@ -137,6 +158,7 @@ impl<'a> Elf64<'a> { } Ok(Self { + prologue, header, file_reader, program_table_reader, @@ -145,6 +167,11 @@ impl<'a> Elf64<'a> { }) } + #[inline] + pub const fn prologue(&self) -> ElfHeaderPrologue { + self.prologue + } + #[inline] pub const fn program_entry(&self) -> u64 { self.header.program_entry_offset @@ -180,6 +207,7 @@ const impl Clone for Elf64<'_> { #[inline] fn clone(&self) -> Self { Self { + prologue: self.prologue, header: self.header, file_reader: self.file_reader.clone(), program_table_reader: self.program_table_reader.clone(), @@ -207,18 +235,33 @@ const impl<'a> Iterator for Elf64Programs<'a> { self.program_table_reader.take(self.stride as usize); Some(try { - let data = self - .file_reader - .fork(int_fit(program_header.segment_offset)?) - .ok_or(ElfError::Eof)? - .take(int_fit(program_header.segment_file_size)?) - .ok_or(ElfError::Eof)?; - + let mut data = self.file_reader.fork(int_fit(program_header.segment_offset)?).ok_or(ElfError::Eof)?; ElfSegment64 { segment_type: match program_header.segment_type { 0 => ElfSegmentType::Null, - 1 => ElfSegmentType::Load, - 2 => ElfSegmentType::Dynamic, + 1 => ElfSegmentType::Load(data.take(int_fit(program_header.segment_file_size)?).ok_or(ElfError::Eof)?), + 2 => { + let mut rela_offset = None; + let mut rela_size = None; + let mut rela_stride = None; + loop { + let dyn_entry = data.read::().ok_or(ElfError::Eof)?; + match dyn_entry.tag { + ElfDt64::NULL => break, + ElfDt64::RELA => rela_offset = Some(int_fit(dyn_entry.val)?), + ElfDt64::RELASZ => rela_size = Some(int_fit(dyn_entry.val)?), + ElfDt64::RELAENT => rela_stride = Some(int_fit(dyn_entry.val)?), + _ => {} + } + } + + // TODO `DT_RELA` isn't the only way to relocate things + let offset = rela_offset.ok_or(ElfError::MalformedDynHeader("`DT_RELA` not found"))?; + let size = rela_size.ok_or(ElfError::MalformedDynHeader("`DT_RELASZ` not found"))?; + let stride = rela_stride.ok_or(ElfError::MalformedDynHeader("`DT_RELAENT` not found"))?; + + ElfSegmentType::Dynamic { offset, size, stride } + } 3 => ElfSegmentType::Interp, 4 => ElfSegmentType::Note, 5 => ElfSegmentType::Shlib, @@ -226,7 +269,6 @@ const impl<'a> Iterator for Elf64Programs<'a> { 7 => ElfSegmentType::Tls, n => ElfSegmentType::Unknown(n), }, - data, flags: program_header.flags, virtual_address: program_header.segment_virtual_address, physical_address: program_header.segment_physical_address, @@ -252,10 +294,9 @@ impl ExactSizeIterator for Elf64Programs<'_> { impl FusedIterator for Elf64Programs<'_> {} -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub struct ElfSegment64<'a> { - pub segment_type: ElfSegmentType, - pub data: &'a [u8], + pub segment_type: ElfSegmentType<'a>, pub flags: ElfProgramFlags, /// [`Self::data`] should be copied to this v-address pub virtual_address: u64, @@ -267,15 +308,15 @@ pub struct ElfSegment64<'a> { pub alignment: u64, } -#[derive(Debug, Clone, Copy, Hash)] +#[derive(Debug, Clone)] #[repr(u32)] -pub enum ElfSegmentType { +pub enum ElfSegmentType<'a> { /// Ignore the entry Null = 0, /// Clear p_memsz bytes at p_vaddr to 0, then copy p_filesz bytes from p_offset to p_vaddr - Load = 1, + Load(&'a [u8]) = 1, /// Requires dynamic linking - Dynamic = 2, + Dynamic { offset: usize, size: usize, stride: usize } = 2, /// Contains a file path to an executable to use as an interpreter for the segment Interp = 3, /// Note section. There are more values, but mostly contain architecture/environment specific @@ -291,25 +332,6 @@ pub enum ElfSegmentType { Unknown(u32) = u32::MAX, } -const impl Eq for ElfSegmentType {} -const impl PartialEq for ElfSegmentType { - #[inline] - fn eq(&self, other: &Self) -> bool { - match (*self, *other) { - (Self::Null, Self::Null) - | (Self::Load, Self::Load) - | (Self::Dynamic, Self::Dynamic) - | (Self::Interp, Self::Interp) - | (Self::Note, Self::Note) - | (Self::Shlib, Self::Shlib) - | (Self::Header, Self::Header) - | (Self::Tls, Self::Tls) => true, - (Self::Unknown(l), Self::Unknown(r)) if l == r => true, - _ => false, - } - } -} - #[derive(Debug)] pub struct Elf64Sections<'a> { len: u16, diff --git a/elf/src/sys/elf_header.rs b/elf/src/sys/elf_header.rs index 83ba97a..0d52dc3 100644 --- a/elf/src/sys/elf_header.rs +++ b/elf/src/sys/elf_header.rs @@ -1,7 +1,7 @@ -use bytemuck::{Pod, Zeroable}; +use bytemuck::AnyBitPattern; /// The ELF header is always found at the start of the file. -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(C)] pub struct ElfHeaderPrologue { /// Magic number - 0x7F, then 'ELF' in ASCII 0-3 @@ -15,17 +15,28 @@ pub struct ElfHeaderPrologue { /// OS ABI - usually 0 for System V 7 pub os_abi: u8, /// Unused/padding 8-15 - pub _padding: [u8; 8], + _padding: [u8; 8], /// Type (1 = relocatable, 2 = executable, 3 = shared, 4 = core) 16-17 - pub elf_type: u16, + pub elf_type: ElfType, /// Instruction set - see table below 18-19 pub instruction_set: u16, /// ELF Version (currently 1) 20-23 pub elf_version: u32, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, AnyBitPattern)] +#[repr(transparent)] +pub struct ElfType(u16); +impl ElfType { + pub const NONE: Self = Self(0); + pub const RELOCATABLE: Self = Self(1); + pub const EXECUTABLE: Self = Self(2); + pub const DYNAMIC: Self = Self(3); + pub const CORE_DUMP: Self = Self(4); +} + /// Continuation of [`ElfHeaderPrologue`] in 64-bit format (arch == 2). -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(C)] pub struct ElfHeader64 { /// Program entry offset 24-31 diff --git a/elf/src/sys/mod.rs b/elf/src/sys/mod.rs index ddef6bd..19c17c7 100644 --- a/elf/src/sys/mod.rs +++ b/elf/src/sys/mod.rs @@ -1,6 +1,8 @@ mod elf_header; mod program_header; +mod relocations; mod section_header; pub use elf_header::*; pub use program_header::*; +pub use relocations::*; pub use section_header::*; diff --git a/elf/src/sys/program_header.rs b/elf/src/sys/program_header.rs index 8e8d5c9..8ba4a81 100644 --- a/elf/src/sys/program_header.rs +++ b/elf/src/sys/program_header.rs @@ -1,7 +1,7 @@ use bitflags::bitflags; -use bytemuck::{Pod, Zeroable}; +use bytemuck::AnyBitPattern; -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(C)] pub struct ElfProgramHeader64 { /// Type of segment (see below) 0-3 @@ -22,7 +22,7 @@ pub struct ElfProgramHeader64 { pub section_alignment: u64, } -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(transparent)] pub struct ElfProgramFlags(u32); bitflags! { diff --git a/elf/src/sys/relocations.rs b/elf/src/sys/relocations.rs new file mode 100644 index 0000000..55b1a0e --- /dev/null +++ b/elf/src/sys/relocations.rs @@ -0,0 +1,44 @@ +use bytemuck::AnyBitPattern; + +#[derive(Debug, Clone, Copy, AnyBitPattern)] +#[repr(C)] +pub struct ElfDyn64 { + pub tag: ElfDt64, + pub val: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, AnyBitPattern)] +#[repr(transparent)] +pub struct ElfDt64(pub i64); +impl ElfDt64 { + pub const NULL: Self = Self(0); + pub const RELA: Self = Self(7); + pub const RELASZ: Self = Self(8); + pub const RELAENT: Self = Self(9); +} + +#[derive(Debug, Clone, Copy, AnyBitPattern)] +#[repr(C)] +pub struct ElfRela64 { + pub offset: u64, + pub info: ElfRela64Info, + pub addend: i64, +} + +#[derive(Debug, Clone, Copy, AnyBitPattern)] +#[repr(C, align(8))] +pub struct ElfRela64Info { + #[cfg(target_endian = "little")] + pub kind: ElfRela64Type, + pub index: u32, + #[cfg(target_endian = "big")] + pub kind: ElfRela64Type, +} + +#[derive(Debug, Clone, Copy, AnyBitPattern, PartialEq, Eq)] +#[repr(transparent)] +pub struct ElfRela64Type(pub u32); +impl ElfRela64Type { + pub const X86_64_NONE: Self = Self(0); + pub const X86_64_RELATIVE: Self = Self(8); +} diff --git a/elf/src/sys/section_header.rs b/elf/src/sys/section_header.rs index 6a2e0fa..c896ad0 100644 --- a/elf/src/sys/section_header.rs +++ b/elf/src/sys/section_header.rs @@ -1,7 +1,7 @@ use bitflags::bitflags; -use bytemuck::{Pod, Zeroable}; +use bytemuck::AnyBitPattern; -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(C)] pub struct ElfSectionHeader64 { /// Offset into the section-name string table 0-3 @@ -26,7 +26,7 @@ pub struct ElfSectionHeader64 { pub entry_size: u64, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, AnyBitPattern)] #[repr(transparent)] pub struct ElfSectionType(pub u32); impl ElfSectionType { @@ -44,7 +44,7 @@ impl ElfSectionType { pub const DYNSYM: Self = Self(11); } -#[derive(Debug, Clone, Copy, Pod, Zeroable)] +#[derive(Debug, Clone, Copy, AnyBitPattern)] #[repr(transparent)] pub struct ElfSectionFlags(u64); bitflags! { diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index abd61e9..e39c82e 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -10,4 +10,12 @@ forced-target = "x86_64-unknown-none" [dependencies] elpytios-abi = { path = "../abi" } +elpytios-alloc = { path = "../alloc" } elpytios-bootinfo = { path = "../bootinfo" } +elpytios-elf = { path = "../elf" } + +arrayvec = { version = "0.7", default-features = false } +bitflags = "2" +bytemuck = { version = "1", features = ["derive", "latest_stable_rust"] } +log = "*" +nonmax = { version = "0.5", default-features = false } diff --git a/kernel/src/allocator/mod.rs b/kernel/src/allocator/mod.rs new file mode 100644 index 0000000..c58d8c6 --- /dev/null +++ b/kernel/src/allocator/mod.rs @@ -0,0 +1,31 @@ +use core::ptr::NonNull; + +use elpytios_alloc::PageAllocator; + +use crate::{ + statics::{get_phys_alloc, phys_to_virt, virt_to_phys}, + vaddr::VAddr, +}; + +mod phys; +mod tree; +pub use phys::*; +pub use tree::*; + +#[derive(Debug, Clone, Copy)] +pub struct KernelPageAllocator; +unsafe impl PageAllocator for KernelPageAllocator { + #[inline] + fn alloc(&self, order: u32) -> Option> { + get_phys_alloc() + .lock() + .alloc(order) + .ok() + .map(|addr| unsafe { NonNull::new_unchecked(phys_to_virt(addr).ptr_mut::()) }) + } + + #[inline] + unsafe fn dealloc(&self, ptr: NonNull, order: u32) { + unsafe { get_phys_alloc().lock().dealloc(virt_to_phys(VAddr::new(ptr.as_ptr() as usize)), order) } + } +} diff --git a/kernel/src/allocator/phys.rs b/kernel/src/allocator/phys.rs new file mode 100644 index 0000000..056bbb1 --- /dev/null +++ b/kernel/src/allocator/phys.rs @@ -0,0 +1,96 @@ +use core::{ + hint::cold_path, + sync::atomic::{AtomicBool, Ordering::Relaxed}, +}; + +use arrayvec::ArrayVec; +use elpytios_bootinfo::{PAGE_SIZE, paddr::PAddr}; + +use crate::allocator::{AllocTree, TreeAllocError}; + +#[derive(Debug)] +pub struct PhysicalPageAllocator { + trees: ArrayVec() }>, +} + +unsafe impl Send for PhysicalPageAllocator {} +unsafe impl Sync for PhysicalPageAllocator {} + +impl PhysicalPageAllocator { + /// # Safety + /// [`Self::sort_tree`] must be called before allocating. + #[inline] + pub unsafe fn new() -> PhysicalPageAllocator { + static CREATED: AtomicBool = AtomicBool::new(false); + + if CREATED.compare_exchange(false, true, Relaxed, Relaxed).is_err() { + panic!("Only one `PhysicalPageAllocator` instance may be created") + } + + Self { + trees: ArrayVec::new_const(), + } + } + + #[inline] + pub const fn tree_count(&self) -> usize { + self.trees.len() + } + + /// # Safety + /// The resulting tree's allocations must be aligned to + /// [`ALLOC_ALIGNMENT`](elpytios_abi::ALLOC_ALIGNMENT). + #[inline] + pub unsafe fn push_tree(&mut self, base: PAddr, tree: *mut AllocTree) { + self.trees.push(Entry { base, tree }); + } + + #[inline] + pub fn sort_tree(&mut self) { + self.trees.sort_unstable_by_key(|e| e.base); + } + + pub fn alloc(&mut self, order: u32) -> Result { + let mut last_error = TreeAllocError::InsufficientSpace { requested_order: order }; + for &Entry { base, tree } in &self.trees { + let tree = unsafe { tree.as_mut_unchecked() }; + match tree.alloc(order) { + Ok(index) => return Ok(base.byte_add(index as usize * PAGE_SIZE)), + Err(e @ TreeAllocError::InsufficientSpace { .. }) => last_error = e, + } + } + + Err(last_error) + } + + /// # Safety + /// - `addr` must have been obtained through [`Self::alloc`]. + /// - `order` must be the same value passed through the same [`Self::alloc`] invocation. + pub unsafe fn dealloc(&mut self, addr: PAddr, order: u32) { + let tree_index = match self.trees.binary_search_by_key(&addr, |e| e.base) { + Ok(i) => i, + Err(0) => return, + Err(i) => i - 1, + }; + + unsafe { + let &Entry { base, tree } = self.trees.get_unchecked(tree_index); + let tree = tree.as_mut_unchecked(); + let index = u32::try_from((addr.addr() - base.addr()) / PAGE_SIZE).unwrap_unchecked(); + + // `dealloc` *may* be called for pages that didn't originally come with this allocator + // But in the case that they do, callers must ensure the safety invariants + if index < tree.node_count() { + tree.dealloc(index, order); + } else { + cold_path(); + } + } + } +} + +#[derive(Debug, Clone, Copy)] +struct Entry { + base: PAddr, + tree: *mut AllocTree, +} diff --git a/kernel/src/allocator/tree.rs b/kernel/src/allocator/tree.rs new file mode 100644 index 0000000..d7494e8 --- /dev/null +++ b/kernel/src/allocator/tree.rs @@ -0,0 +1,325 @@ +use core::{ + alloc::{Layout, LayoutError}, + fmt, + hint::assert_unchecked, + mem::MaybeUninit, + ptr, slice, +}; + +use nonmax::NonMaxU32; + +#[derive(Clone, Copy)] +pub enum TreeAllocError { + InsufficientSpace { requested_order: u32 }, +} + +impl fmt::Debug for TreeAllocError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for TreeAllocError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InsufficientSpace { requested_order } => write!(f, "Tree can no longer contain allocation of size 2^{requested_order}"), + } + } +} + +/// A binary buddy tree, implemented with a split bitset and free lists. The tree operates on number +/// of "order," not leaf counts; i.e., the leaf count must be `2 ^ order`. +#[repr(C)] +pub struct AllocTree { + max_order: u32, + // The offsets here are relative to the offset of `data`... + nodes_offset: usize, + split_bitset_offset: usize, + data: AllocTreeData, +} + +unsafe impl Send for AllocTree {} +unsafe impl Sync for AllocTree {} + +struct AllocTreeFields<'a> { + max_order: u32, + free_lists: FreeLists<'a>, + list_nodes: &'a mut [ListNode], + split_bitset: &'a mut AllocBitset, +} + +impl AllocTree { + pub unsafe fn new(at: *mut (), layout: AllocTreeLayout) -> *mut Self { + let this = ptr::from_raw_parts_mut::(at, layout.size() - layout.free_nodes_offset_abs); + unsafe { + (&raw mut (*this).max_order).write(layout.max_order); + (&raw mut (*this).nodes_offset).write(layout.nodes_offset_abs - layout.free_nodes_offset_abs); + (&raw mut (*this).split_bitset_offset).write(layout.split_bitset_offset_abs - layout.free_nodes_offset_abs); + + let free_lists = (&raw mut (*this).data.0).as_mut_ptr().cast::(); + for i in 0..=layout.max_order as usize { + free_lists.add(i).write(ListHead { + head: if i == layout.max_order as usize { Some(NonMaxU32::ZERO) } else { None }, + }); + } + + let list_nodes = (&raw mut (*this).data.0) + .as_mut_ptr() + .add(layout.nodes_offset_abs - layout.free_nodes_offset_abs) + .cast::(); + for i in 0..1 << layout.max_order { + list_nodes.add(i).write(ListNode { prev: None, next: None }); + } + + (&raw mut (*this).data.0) + .as_mut_ptr() + .add(layout.split_bitset_offset_abs - layout.free_nodes_offset_abs) + .cast::() + .write_bytes(0, AllocBitset::size_for((1 << layout.max_order) - 1)); + } + + this + } + + #[inline] + fn fields(&mut self) -> AllocTreeFields<'_> { + let max_order = self.max_order; + let data = self.data.0.as_mut_ptr(); + unsafe { + AllocTreeFields { + max_order, + free_lists: FreeLists { + heads: slice::from_raw_parts_mut(data.cast(), max_order as usize + 1), + }, + list_nodes: slice::from_raw_parts_mut(data.add(self.nodes_offset).cast(), 1 << max_order), + split_bitset: ptr::from_raw_parts_mut::(data.add(self.split_bitset_offset), AllocBitset::size_for((1 << max_order) - 1)) + .as_mut_unchecked(), + } + } + } + + #[inline] + pub fn node_count(&self) -> u32 { + 1 << self.max_order + } + + pub fn alloc(&mut self, order: u32) -> Result { + let AllocTreeFields { + max_order, + free_lists, + list_nodes, + split_bitset, + } = self.fields(); + + if order > max_order { + return Err(TreeAllocError::InsufficientSpace { requested_order: order }) + } + + let mut current = None; + for i in order..=max_order { + if let Some(head) = free_lists.heads[i as usize].head.take() { + if let Some(new_head) = list_nodes[head.get() as usize].next.take() { + list_nodes[new_head.get() as usize].prev = None; + free_lists.heads[i as usize].head = Some(new_head); + } + + current = Some((head.get(), i)); + break + } + } + + let Some((index, mut current_order)) = current else { + return Err(TreeAllocError::InsufficientSpace { requested_order: order }) + }; + + if current_order < max_order { + unsafe { + split_bitset.get_and_toggle(Self::bit_index(index, current_order, max_order)); + } + } + + while current_order > order { + let next_order = current_order - 1; + + let free = NonMaxU32::new(index + (1 << next_order)).expect("Allocation index >= u32::MAX"); + list_nodes[free.get() as usize].prev = None; + + if let Some(prev_head) = free_lists.heads[next_order as usize].head.replace(free) { + list_nodes[prev_head.get() as usize].prev = Some(free); + list_nodes[free.get() as usize].next = Some(prev_head); + } else { + list_nodes[free.get() as usize].next = None; + } + + current_order = next_order; + unsafe { + // False: Either both buddies are occupied or both are allocated + // True: Exactly one buddy is occupied + split_bitset.get_and_toggle(Self::bit_index(index, current_order, max_order)); + } + } + + Ok(index) + } + + pub unsafe fn dealloc(&mut self, mut index: u32, mut order: u32) { + let AllocTreeFields { + max_order, + free_lists, + list_nodes, + split_bitset, + } = self.fields(); + + unsafe { + assert_unchecked(index.is_multiple_of(1 << order)); + assert_unchecked(index < 1 << max_order); + assert_unchecked(order <= max_order); + } + + while order < max_order { + // Was false, now true: Can't merge, exactly one buddy is still occupied + // Was true, now false: Can merge, no buddies are occupied + let can_merge = unsafe { split_bitset.get_and_toggle(Self::bit_index(index, order, max_order)) }; + if can_merge { + let buddy_index = index ^ (1 << order); + + // Remove the buddy from the free list + let buddy_node = &mut list_nodes[buddy_index as usize]; + match [buddy_node.prev.take(), buddy_node.next.take()] { + // `prev.is_none()` means this is the head in the free list` + [None, new_head] => { + free_lists.heads[order as usize].head = new_head; + if let Some(new_head) = new_head { + list_nodes[new_head.get() as usize].prev = None; + } + } + [Some(prev), next] => { + list_nodes[prev.get() as usize].next = next; + if let Some(next) = next { + list_nodes[next.get() as usize].prev = Some(prev); + } + } + } + + index &= !(1 << order); + order += 1; + } else { + break + } + } + + list_nodes[index as usize].prev = None; + if let Some(prev_head) = free_lists.heads[order as usize].head.replace(unsafe { NonMaxU32::new_unchecked(index) }) { + list_nodes[prev_head.get() as usize].prev = NonMaxU32::new(index); + list_nodes[index as usize].next = Some(prev_head); + } + } + + #[inline] + fn bit_index(index: u32, order: u32, max_order: u32) -> u32 { + debug_assert!(order < max_order, "`order` ({order}) must be less than `max_order` ({max_order})"); + + let layer_base = (1 << (max_order - order - 1)) - 1; + let pair_index = index >> (order + 1); + + layer_base + pair_index + } + + pub const fn layout(count: usize) -> Result { + let max_order = count.ilog2(); + let taken = 1 << max_order; + + // `order` + let layout = Layout::new::(); + // `nodes_offset` + let (layout, ..) = layout.extend(Layout::new::())?; + // `split_bitset_offset` + let (layout, ..) = layout.extend(Layout::new::())?; + + // `free_nodes` + let (layout, free_nodes_offset_abs) = layout.extend(Layout::array::((max_order + 1) as usize)?)?; + // `nodes`split_bitset_offset + let (layout, nodes_offset_abs) = layout.extend(Layout::array::(taken)?)?; + // `split_bitset` + let (layout, split_bitset_offset_abs) = layout.extend(Layout::array::(AllocBitset::size_for(taken - 1))?)?; + let layout = layout.pad_to_align(); + + Ok(AllocTreeLayout { + layout, + free_nodes_offset_abs, + nodes_offset_abs, + split_bitset_offset_abs, + max_order, + }) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct AllocTreeLayout { + layout: Layout, + // ..while the offsets here are absolute. + free_nodes_offset_abs: usize, + nodes_offset_abs: usize, + split_bitset_offset_abs: usize, + max_order: u32, +} + +impl AllocTreeLayout { + #[inline] + pub const fn node_count(&self) -> usize { + 1 << self.max_order + } + + #[inline] + pub const fn size(&self) -> usize { + self.layout.size() + } + + #[inline] + pub const fn align(&self) -> usize { + self.layout.align() + } +} + +// Safety notes: The repr and align must be the same as `ListHead`. +#[repr(C, align(4))] +struct AllocTreeData([MaybeUninit]); + +#[repr(transparent)] +struct FreeLists<'a> { + heads: &'a mut [ListHead], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, align(4))] +struct ListHead { + head: Option, +} + +#[derive(Clone, Copy)] +#[repr(C)] +struct ListNode { + prev: Option, + next: Option, +} + +#[repr(transparent)] +struct AllocBitset([u64]); +impl AllocBitset { + #[inline] + const fn size_for(bits: usize) -> usize { + bits.div_ceil(u64::BITS as usize) + } + + #[inline] + unsafe fn get_and_toggle(&mut self, bit: u32) -> bool { + let block_index = bit / u64::BITS; + let block_bit = 1 << (bit & (u64::BITS - 1)); + + let block = unsafe { self.0.get_unchecked_mut(block_index as usize) }; + let old_block = *block; + + *block = old_block ^ block_bit; + old_block & block_bit != 0 + } +} diff --git a/kernel/src/arch/mod.rs b/kernel/src/arch/mod.rs new file mode 100644 index 0000000..3e4818f --- /dev/null +++ b/kernel/src/arch/mod.rs @@ -0,0 +1,8 @@ +cfg_select! { + target_arch = "x86_64" => { + pub mod x86_64; + } + _ => { + compile_error!("Unsupported architecture"); + } +} diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs new file mode 100644 index 0000000..8402fa9 --- /dev/null +++ b/kernel/src/arch/x86_64/mod.rs @@ -0,0 +1,6 @@ +mod msr; +mod pmio; +mod reg; +pub use msr::*; +pub use pmio::*; +pub use reg::*; diff --git a/kernel/src/arch/x86_64/msr.rs b/kernel/src/arch/x86_64/msr.rs new file mode 100644 index 0000000..5dcedea --- /dev/null +++ b/kernel/src/arch/x86_64/msr.rs @@ -0,0 +1,95 @@ +use core::arch::asm; + +#[derive(Debug, Clone, Copy)] +#[repr(u32)] +pub enum Msr { + /// - Read-write register. + /// - Bit 10: x2APIC enable (turns off MMIO, maps registers to MSR). + /// - Bit 11: APIC global enable (must be 1 for APIC functionality). + /// - Bit 12-51: [`PAddr::addr()`](elpytios_bootinfo::PAddr::addr) without bits 0..=11 (i.e. + /// must be page-aligned). + Ia32ApicBase = 0x001b, + /// - Read-write register. + /// - Bit 0-63: Pointer to CPU-local data. + Ia32FsBase = 0xc0000100, + /// - Read-write register. + /// - Bit 0-63: Pointer to CPU-local data. + Ia32GsBase = 0xc0000101, + /// - Read-write register. + /// - Bit 0: `syscall` and `sysret` enable. + /// - Bit 8: Long-mode enable. + Ia32Efer = 0xc000_0080, + /// - Read-write register. + /// - Bit 32-47: `KERNEL_CODE` GDT selector. + /// - Bit 48-63: USER_BASE GDT selector (+8 must be `USER_DATA`, +16 must be `USER_CODE`). + Ia32Star = 0xc000_0081, + /// - Read-write register. + /// - Bit 0-63: `syscall` entry stub naked function pointer. + Ia32Lstar = 0xc000_0082, + /// - Read-write register. + /// - Bit 9: Interrupt flag (`cli`). + /// - Bit 10: Direction flag (`cld`). + /// - Bit 18: Alignment check. + Ia32Fmask = 0xc000_0084, + /// - Read-write register. + /// - Bit 0-63: TSC timestamp for timer tick. + Ia32TscDeadline = 0x6e0, + /// - Read-only register. + /// - Bit 0-31: Unique 32-bit physical hardware ID. + Ia32X2ApicId = 0x802, + /// - Read-only register. + /// - Bit 0-7: Version number. + /// - Bit 16-23: Max LVT entries. + Ia32X2ApicVersion = 0x803, + /// - Write-only register. + /// - Write a dummy 0 to clear in-service flag. + Ia32X2ApicEoi = 0x80b, + /// - Read-write register. + /// - Bit 0-7: Set fallback handler vector. + /// - Bit 8: APIC enable in software. + Ia32X2ApicSivr = 0x80f, + /// - Read-write register. + /// - Bit 0-7: Vector. + /// - Bit 8-10: Delivery mode (100=NMI, 101=Init, 110=Startup). + /// - Bit 14: Assert flag. + /// - Bit 32-63: Target core destination APIC ID (as specified in [`Self::Ia32X2ApicId`]). + Ia32X2ApicIcr = 0x830, + /// - Read-write register. + /// - Bit 0-7: Vector. + /// - Bit 17-18: Mode (0=One-shot, 1=Periodic, 2=TSC-deadline, 3=Reserved). + Ia32X2ApicLvtTimer = 0x832, +} + +#[inline(always)] +pub unsafe fn rdmsr(address: Msr) -> u64 { + let low: u32; + let high: u32; + unsafe { + asm!( + "rdmsr", + + in("ecx") address as u32, + out("eax") low, + out("edx") high, + + options(att_syntax, nomem, nostack, preserves_flags) + ); + } + + ((high as u64) << 32) | (low as u64) +} + +#[inline(always)] +pub unsafe fn wrmsr(address: Msr, value: u64) { + unsafe { + asm!( + "wrmsr", + + in("ecx") address as u32, + in("eax") value as u32, + in("edx") (value >> 32) as u32, + + options(att_syntax, nomem, nostack, preserves_flags) + ); + } +} diff --git a/kernel/src/arch/x86_64/pmio.rs b/kernel/src/arch/x86_64/pmio.rs new file mode 100644 index 0000000..41f5905 --- /dev/null +++ b/kernel/src/arch/x86_64/pmio.rs @@ -0,0 +1,29 @@ +use core::arch::asm; + +#[inline(always)] +pub unsafe fn outb(port: u16, value: u8) { + unsafe { + asm!( + "outb %al, %dx", + in("al") value, + in("dx") port, + + options(att_syntax, nomem, nostack, preserves_flags) + ) + } +} + +#[inline(always)] +pub unsafe fn inb(port: u16) -> u8 { + unsafe { + let value: u8; + asm!( + "inb %dx, %al", + in("dx") port, + out("al") value, + + options(att_syntax, nomem, nostack, preserves_flags), + ); + value + } +} diff --git a/kernel/src/arch/x86_64/reg.rs b/kernel/src/arch/x86_64/reg.rs new file mode 100644 index 0000000..874545f --- /dev/null +++ b/kernel/src/arch/x86_64/reg.rs @@ -0,0 +1,85 @@ +use core::{ + alloc::Layout, + arch::{ + asm, + x86_64::{__cpuid_count, _xrstor64, _xsave64, _xsavec64, _xsaveopt64, _xsetbv}, + }, + mem::{Alignment, MaybeUninit}, +}; + +#[derive(Debug, Clone, Copy)] +pub struct ExtendedRegisterLayout { + layout: Layout, + save: unsafe fn(to: *mut u8, save_mask: u64), + mask: ExtendedRegisterMask, +} + +impl !Send for ExtendedRegisterLayout {} +impl !Sync for ExtendedRegisterLayout {} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(transparent)] +pub struct ExtendedRegisterMask(u64); + +#[repr(C, align(64))] +pub struct ExtendedRegisters([MaybeUninit]); + +impl ExtendedRegisterLayout { + pub const ALIGNMENT: Alignment = unsafe { Alignment::new_unchecked(1 << 6) }; + + /// # Safety + /// Must only be called once per CPU core. + #[inline] + pub unsafe fn new() -> Self { + // Enable `xsave` and `xstor` + // x86_64 guarantees support for these instructions, so no need to check + unsafe { + asm!( + "movq %cr4, {tmp}", + "orq $(1 << 18), {tmp}", + "movq {tmp}, %cr4", + + tmp = out(reg) _, + options(att_syntax, nomem, nostack, preserves_flags), + ); + } + + // Enable all supported features to xcr0 + let leaf0 = __cpuid_count(0xd, 0); + let mask = ExtendedRegisterMask((leaf0.edx as u64) << 32 | (leaf0.eax as u64)); + unsafe { _xsetbv(0, mask.0) } + + // Query how many bytes the extended registers would take + let leaf1 = __cpuid_count(0xd, 1); + let (size, save): (u32, unsafe fn(to: *mut u8, save_mask: u64)) = if leaf1.eax & (1 << 1) != 0 { + (leaf1.ebx, _xsavec64) + } else if leaf1.eax & (1 << 0) != 0 { + (leaf0.ebx, _xsaveopt64) + } else { + (leaf0.ebx, _xsave64) + }; + + let layout = Layout::from_size_alignment(size as usize, Self::ALIGNMENT).expect("Extended register buffer size too large"); + Self { layout, save, mask } + } + + #[inline] + pub fn layout(&self) -> Layout { + self.layout + } + + #[inline] + pub fn mask(&self) -> ExtendedRegisterMask { + self.mask + } + + #[inline(always)] + pub unsafe fn save(&self, mask: ExtendedRegisterMask, to: *mut ExtendedRegisters) { + unsafe { (self.save)((&raw mut (*to).0).as_mut_ptr().cast(), mask.0) } + } + + #[inline(always)] + pub unsafe fn load(&self, mask: ExtendedRegisterMask, from: *const ExtendedRegisters) { + unsafe { _xrstor64((&raw const (*from).0).as_ptr().cast(), mask.0) } + } +} diff --git a/kernel/src/device/acpi/madt.rs b/kernel/src/device/acpi/madt.rs new file mode 100644 index 0000000..7e6e490 --- /dev/null +++ b/kernel/src/device/acpi/madt.rs @@ -0,0 +1,347 @@ +#![allow(unused, reason = "Not all ACPI features are used, but they are needed for parsing")] + +use core::{iter, ops::RangeInclusive}; + +use bitflags::bitflags; + +use crate::device::{ + SystemTable, + acpi::{PackedPtr, TypedSystemTable}, +}; + +#[derive(Clone, Copy)] +pub struct Madt<'root> { + pub local_interrupt_control_addr: u32, + pub flags: ApicFlags, + payload: PackedPtr<'root>, +} + +impl<'root> IntoIterator for Madt<'root> { + type IntoIter = impl Iterator + 'root; + type Item = ::Item; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + let mut payload = self.payload; + iter::from_fn(move || { + while payload.len() != 0 { + const RESERVED_FOR_OSPM_START: u8 = *RESERVED_FOR_OSPM.start(); + const RESERVED_FOR_OSPM_END: u8 = *RESERVED_FOR_OSPM.end(); + const RESERVED_FOR_OEM_START: u8 = *RESERVED_FOR_OEM.start(); + const RESERVED_FOR_OEM_END: u8 = *RESERVED_FOR_OEM.end(); + + unsafe { + let kind: u8 = payload.read(); + let len: u8 = payload.read(); + + return Some(match kind { + PROCESSOR_LOCAL => Pic::ProcessorLocal(payload.read()), + IO => Pic::Io(payload.read()), + INTERRUPT_SOURCE_OVERRIDE => Pic::InterruptSourceOverride(payload.read()), + NMI_SOURCE => Pic::NmiSource(payload.read()), + LOCAL_NMI => Pic::LocalNmi(payload.read()), + LOCAL_ADDR_OVERRIDE => Pic::LocalAddrOverride(payload.read()), + IO_STREAMLINED => Pic::IoStreamlined(payload.read()), + PROCESSOR_LOCAL_STREAMLINED => { + let data = payload.read(); + let _acpi_processor_uid_string = payload.slice(len as usize - 2 - size_of::()); + Pic::ProcessorLocalStreamlined(data) + } + PLATFORM_INTERRUPT_SOURCES => Pic::PlatformInterruptSources(payload.read()), + PROCESSOR_LOCAL_X2 => Pic::ProcessLocalX2(payload.read()), + LOCAL_NMI_X2 => Pic::LocalNmiX2(payload.read()), + GIC_CPU_INTERFACE => Pic::GicCpuInterface(payload.read()), + GIC_DISTRIBUTOR => Pic::GicDistributor(payload.read()), + GIC_MSI_FRAME => Pic::GicMsiFrame(payload.read()), + GIC_REDISTRIBUTOR => Pic::GicRedistributor(payload.read()), + GIC_INTERRUPT_TRANSLATION_SERVICE => Pic::GicInterruptTranslationService(payload.read()), + RESERVED_FOR_OSPM_START..=RESERVED_FOR_OSPM_END | RESERVED_FOR_OEM_START..=RESERVED_FOR_OEM_END => { + _ = payload.slice(len as usize - 2); + continue + }, + }) + } + } + None + }) + } +} + +unsafe impl TypedSystemTable for Madt<'_> { + const SIGNATURE: [u8; 4] = *b"APIC"; + type Out<'root> = Madt<'root>; + + #[inline] + unsafe fn from_table<'root>(table: &SystemTable<'root>) -> Madt<'root> { + let mut payload = PackedPtr::new(table.entries); + unsafe { + Madt { + local_interrupt_control_addr: payload.read(), + flags: payload.read(), + payload, + } + } + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct ApicFlags(u32); +bitflags! { + impl ApicFlags: u32 { + const PCAT_COMPAT = 1 << 0; + } +} + +pub const PROCESSOR_LOCAL: u8 = 0x00; +pub const IO: u8 = 0x01; +pub const INTERRUPT_SOURCE_OVERRIDE: u8 = 0x02; +pub const NMI_SOURCE: u8 = 0x03; +pub const LOCAL_NMI: u8 = 0x04; +pub const LOCAL_ADDR_OVERRIDE: u8 = 0x05; +pub const IO_STREAMLINED: u8 = 0x06; +pub const PROCESSOR_LOCAL_STREAMLINED: u8 = 0x07; +pub const PLATFORM_INTERRUPT_SOURCES: u8 = 0x08; +pub const PROCESSOR_LOCAL_X2: u8 = 0x09; +pub const LOCAL_NMI_X2: u8 = 0x0a; +pub const GIC_CPU_INTERFACE: u8 = 0x0b; +pub const GIC_DISTRIBUTOR: u8 = 0x0c; +pub const GIC_MSI_FRAME: u8 = 0x0d; +pub const GIC_REDISTRIBUTOR: u8 = 0x0e; +pub const GIC_INTERRUPT_TRANSLATION_SERVICE: u8 = 0x0f; +pub const RESERVED_FOR_OSPM: RangeInclusive = 0x10..=0x7f; +pub const RESERVED_FOR_OEM: RangeInclusive = 0x80..=0xff; + +#[derive(Debug, Clone, Copy)] +#[repr(u8)] +pub enum Pic { + ProcessorLocal(ProcessorLocal) = PROCESSOR_LOCAL, + Io(Io) = IO, + InterruptSourceOverride(InterruptSourceOverride) = INTERRUPT_SOURCE_OVERRIDE, + NmiSource(NmiSource) = NMI_SOURCE, + LocalNmi(LocalNmi) = LOCAL_NMI, + LocalAddrOverride(LocalAddrOverride) = LOCAL_ADDR_OVERRIDE, + IoStreamlined(IoStreamlined) = IO_STREAMLINED, + ProcessorLocalStreamlined(ProcessorLocalStreamlined) = PROCESSOR_LOCAL_STREAMLINED, + PlatformInterruptSources(PlatformInterruptSources) = PLATFORM_INTERRUPT_SOURCES, + ProcessLocalX2(ProcessLocalX2) = PROCESSOR_LOCAL_X2, + LocalNmiX2(LocalNmiX2) = LOCAL_NMI_X2, + GicCpuInterface(GicCpuInterface) = GIC_CPU_INTERFACE, + GicDistributor(GicDistributor) = GIC_DISTRIBUTOR, + GicMsiFrame(GicMsiFrame) = GIC_MSI_FRAME, + GicRedistributor(GicRedistributor) = GIC_REDISTRIBUTOR, + GicInterruptTranslationService(GicInterruptTranslationService) = GIC_INTERRUPT_TRANSLATION_SERVICE, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct ProcessorLocal { + pub acpi_processor_uid: u8, + pub apic_id: u8, + pub flags: LocalApicFlags, +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct LocalApicFlags(u32); +bitflags! { + impl LocalApicFlags: u32 { + const ENABLED = 1 << 0; + const ONLINE_CAPABLE = 1 << 1; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct Io { + pub io_apic_id: u8, + pub reserved: [u8; 1], + pub io_apic_addr: u32, + pub global_system_interrupt_base: u32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct InterruptSourceOverride { + pub bus: u8, + pub source: u8, + pub global_system_interrupt: u32, + pub flags: MpsIntiFlags, +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct MpsIntiFlags(u16); +bitflags! { + impl MpsIntiFlags: u16 { + const ACTIVE_HIGH = 0b01; + const ACTIVE_LOW = 0b11; + const POLARITY = 0b11; + + const EDGE_TRIGGERED = 0b01 << 2; + const LEVEL_TRIGGERED = 0b11 << 2; + const TRIGGER_MODE = 0b11 << 2; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct NmiSource { + pub flags: MpsIntiFlags, + pub global_system_interrupt: u32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct LocalNmi { + pub acpi_processor_uid: u8, + pub flags: MpsIntiFlags, + pub local_apic_lint_num: u8, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct LocalAddrOverride { + pub reserved: [u8; 2], + pub local_apic_addr: u64, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct IoStreamlined { + pub io_apic_id: u8, + pub reserved: [u8; 1], + pub global_system_interrupt_base: u32, + pub io_sapic_addr: u64, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct ProcessorLocalStreamlined { + pub acpi_processor_id: u8, + pub local_sapic_id: u8, + pub local_sapic_eid: u8, + pub reserved: [u8; 3], + pub flags: LocalApicFlags, + pub acpi_processor_uid_value: u32, + // Note: `acpi_processor_uid_string: [u8; leftover]` +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct PlatformInterruptSources { + pub flags: MpsIntiFlags, + pub interrupt_type: u8, + pub processor_id: u8, + pub processor_eid: u8, + pub io_sapic_vector: u8, + pub global_system_interrupt: u32, + pub platform_interrupt_source_flags: PlatformInterruptSourceFlags, +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct PlatformInterruptSourceFlags(u32); +bitflags! { + impl PlatformInterruptSourceFlags: u32 { + const CPEI_PROCESSOR_OVERRIDE = 1 << 0; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct ProcessLocalX2 { + pub reserved: [u8; 2], + pub x2apic_id: u32, + pub flags: LocalApicFlags, + pub acpi_processor_uid: u32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct LocalNmiX2 { + pub flags: MpsIntiFlags, + pub acpi_processor_uid: u32, + pub local_x2apic_lint_num: u8, + pub reserved: [u8; 3], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct GicCpuInterface { + pub reserved: [u8; 2], + pub cpu_interface_num: u32, + pub acpi_processor_uid: u32, + pub flags: GicCpuInterfaceFlags, + pub parking_protocol_version: u32, + pub performance_interrupt_gsiv: u32, + pub parked_addr: u64, + pub physical_base_addr: u64, + pub gicv: u64, + pub gich: u64, + pub vgic_maintenance_interrupt: u32, + pub gicr_base_addr: u64, + pub mpidr: u64, + pub processor_power_efficiency_class: u8, + pub reserved2: [u8; 1], + pub spe_overflow_interrupt: u16, +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct GicCpuInterfaceFlags(u32); +bitflags! { + impl GicCpuInterfaceFlags: u32 { + const ENABLED = 1 << 0; + const PERFORMANCE_INTERRUPT_MODE = 1 << 1; + const VGIC_MAINTENANCE_INTERRUPT_MODE = 1 << 2; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct GicDistributor { + pub reserved: [u8; 2], + pub gic_id: u32, + pub physical_base_addr: u64, + pub system_vector_base: u32, + pub gic_version: u8, + pub reserved2: [u8; 3], +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct GicMsiFrame { + pub reserved: [u8; 2], + pub gic_msi_frame_id: u32, + pub physical_base_addr: u64, + pub flags: GicMsiFrameFlags, + pub spi_count: u16, + pub spi_base: u16, +} + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct GicMsiFrameFlags(u32); +bitflags! { + impl GicMsiFrameFlags: u32 { + const SPI_COUNT_BASE_SELECT = 1 << 0; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct GicRedistributor { + pub reserved: [u8; 2], + pub discovery_range_base_addr: u64, + pub discovery_range_len: u32, +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct GicInterruptTranslationService { + pub reserved: [u8; 2], + pub gic_its_id: u32, + pub physical_base_addr: u64, + pub reserved2: [u8; 4], +} diff --git a/kernel/src/device/acpi/mod.rs b/kernel/src/device/acpi/mod.rs new file mode 100644 index 0000000..2eca441 --- /dev/null +++ b/kernel/src/device/acpi/mod.rs @@ -0,0 +1,308 @@ +use core::{any::type_name, fmt, iter::FusedIterator, marker::PhantomData, mem::offset_of, num::NonZeroU8, ptr, slice}; + +mod madt; +mod root; +pub use madt::*; +pub use root::*; + +pub const ROOT_SIGNATURE: [u8; 8] = *b"RSD PTR "; + +pub type AcpiResult = Result; + +#[derive(Clone, Copy)] +pub struct SignatureFmt<'a>(pub &'a [u8]); +impl fmt::Debug for SignatureFmt<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for SignatureFmt<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for chunk in self.0.utf8_chunks() { + for ch in chunk.valid().chars() { + write!(f, "{}", ch.escape_debug())?; + } + for byte in chunk.invalid() { + write!(f, "\\x{:02x}", byte)?; + } + } + Ok(()) + } +} + +#[derive(Clone, Copy)] +pub enum AcpiError { + InvalidRootSignature { found: [u8; 8] }, + InvalidTableSignature { expected: [u8; 4], found: [u8; 4] }, + InvalidChecksum(NonZeroU8), +} + +impl fmt::Debug for AcpiError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for AcpiError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidRootSignature { found } => write!(f, "Expected 'RSD PTR ', found '{}'", SignatureFmt(found)), + Self::InvalidTableSignature { expected, found } => write!(f, "Expected '{}', found '{}'", SignatureFmt(expected), SignatureFmt(found)), + Self::InvalidChecksum(checksum) => write!(f, "Invalid checksum, found {checksum}"), + } + } +} + +trait AcpiParse: Sized { + type Input; + + unsafe fn parse(input: *const Self::Input) -> AcpiResult; +} + +#[repr(C, packed)] +pub struct Rsdp { + signature: [u8; 8], + checksum: u8, + oem_id: [u8; 6], + revision: u8, + rsdt_addr: u32, // Note: Explicitly use `u32` because it's a 32-bit address regardless of architecture +} + +impl Rsdp { + /// # Safety + /// - The pointer must be valid for unaligned reads of `Rsdp`. + /// - The pointer must point to a firmware-provided data to ensure validations. + /// - The resulting output must be dropped before identity-mapping is disabled. + #[inline] + pub unsafe fn new(this: *const Self) -> AcpiResult { + unsafe { Self::parse(this) } + } + + #[inline] + pub fn rsdt(&self) -> AcpiResult> { + unsafe { SystemTable::parse(self.rsdt_addr as usize as *const SystemTableHeader).and_then(|table| table.typed()) } + } +} + +impl AcpiParse for Rsdp { + type Input = Self; + + unsafe fn parse(input: *const Self::Input) -> AcpiResult { + unsafe { + ((*input).signature == ROOT_SIGNATURE).ok_or(AcpiError::InvalidRootSignature { found: (*input).signature })?; + let mut checksum = 0u8; + for &byte in slice::from_raw_parts(&raw const *input as *const u8, size_of::()) { + checksum = checksum.wrapping_add(byte); + } + + match NonZeroU8::new(checksum) { + None => Ok(input.read_unaligned()), + Some(invalid) => Err(AcpiError::InvalidChecksum(invalid)), + } + } + } +} + +#[repr(C, packed)] +pub struct Xsdp { + rsdp: Rsdp, + len: u32, + xsdt_addr: u64, // Note: Explicitly use `u64` because it's a 64-bit address regardless of architecture + ext_checksum: u8, + _reserved: [u8; 3], +} + +impl Xsdp { + /// # Safety + /// - The pointer must be valid for unaligned reads of `Xsdp`. + /// - The pointer must point to a firmware-provided data to ensure validations. + /// - The resulting output must be dropped before identity-mapping is disabled. + #[inline] + pub unsafe fn new(this: *const Self) -> AcpiResult { + unsafe { Self::parse(this) } + } + + #[inline] + pub fn xsdt(&self) -> AcpiResult> { + unsafe { SystemTable::parse(self.xsdt_addr as usize as *const SystemTableHeader).and_then(|table| table.typed()) } + } +} + +impl AcpiParse for Xsdp { + type Input = Self; + + unsafe fn parse(input: *const Self::Input) -> AcpiResult { + unsafe { + Rsdp::parse(&raw const (*input).rsdp)?; + + let mut checksum = 0u8; + for &byte in slice::from_raw_parts(&raw const (*input).len as *const u8, size_of::() - offset_of!(Self, len)) { + checksum = checksum.wrapping_add(byte); + } + + match NonZeroU8::new(checksum) { + None => Ok(input.read_unaligned()), + Some(invalid) => Err(AcpiError::InvalidChecksum(invalid)), + } + } + } +} + +#[repr(C, packed)] +pub struct SystemTableHeader { + signature: [u8; 4], + length: u32, + revision: u8, + checksum: u8, + oem_id: [u8; 6], + oem_table_id: u64, + oem_revision: u32, + creator_id: u32, + creator_revision: u32, + system_table: (), +} + +#[derive(Clone, Copy)] +pub struct SystemTable<'root> { + signature: [u8; 4], + entries: *const [u8], + _marker: PhantomData<&'root ()>, +} + +impl<'root> SystemTable<'root> { + #[inline] + pub fn typed = T>>(&self) -> AcpiResult { + (self.signature == T::SIGNATURE) + .then(|| unsafe { T::from_table(self) }) + .ok_or(AcpiError::InvalidTableSignature { + expected: T::SIGNATURE, + found: self.signature, + }) + } +} + +impl fmt::Debug for SystemTable<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct(type_name::()) + .field("signature", &SignatureFmt(&self.signature)) + .field("byte_len", &self.entries.len()) + .finish() + } +} + +impl AcpiParse for SystemTable<'_> { + type Input = SystemTableHeader; + + unsafe fn parse(input: *const Self::Input) -> AcpiResult { + unsafe { + let entries = slice::from_raw_parts( + &raw const (*input).system_table as *const u8, + (*input).length as usize - size_of::(), + ); + + let mut checksum = 0u8; + for &byte in slice::from_raw_parts(&raw const *input as *const u8, size_of::()) { + checksum = checksum.wrapping_add(byte); + } + + for &byte in entries { + checksum = checksum.wrapping_add(byte); + } + + match NonZeroU8::new(checksum) { + None => Ok(Self { + signature: (*input).signature, + entries, + _marker: PhantomData, + }), + Some(invalid) => Err(AcpiError::InvalidChecksum(invalid)), + } + } + } +} + +pub unsafe trait TypedSystemTable: Sized { + const SIGNATURE: [u8; 4]; + type Out<'root>: TypedSystemTable + 'root; + + unsafe fn from_table<'root>(table: &SystemTable<'root>) -> Self::Out<'root>; +} + +#[derive(Clone, Copy)] +struct PackedPtr<'root> { + ptr: *const [u8], + _marker: PhantomData<&'root ()>, +} + +impl<'root> PackedPtr<'root> { + #[inline] + fn new(ptr: *const [u8]) -> Self { + Self { ptr, _marker: PhantomData } + } + + #[inline] + fn len(self) -> usize { + self.ptr.len() + } + + #[inline] + unsafe fn slice(&mut self, len: usize) -> &'root [u8] { + unsafe { + let new_len = self.ptr.len().checked_sub(len).expect("Not enough bytes"); + let first = self.ptr.as_ptr(); + let result = slice::from_raw_parts(first, len); + + self.ptr = ptr::slice_from_raw_parts(first.add(len), new_len); + result + } + } + + #[inline] + unsafe fn read(&mut self) -> T { + unsafe { + let new_len = self.ptr.len().checked_sub(size_of::()).expect("Not enough bytes"); + let first = self.ptr.cast::(); + let result = first.read_unaligned(); + + self.ptr = ptr::slice_from_raw_parts(first.add(1).cast(), new_len); + result + } + } +} + +struct UnalignedPtrIter<'root, T: 'root> { + entries: PackedPtr<'root>, + _marker: PhantomData<&'root [T]>, +} + +impl<'root, T: 'root> UnalignedPtrIter<'root, T> { + fn new(entries: PackedPtr<'root>) -> Self { + Self { + entries, + _marker: PhantomData, + } + } +} + +impl<'root, T: 'root> Iterator for UnalignedPtrIter<'root, T> { + type Item = T; + + fn next(&mut self) -> Option { + (self.entries.len() != 0).then(|| unsafe { self.entries.read() }) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (self.entries.len() / size_of::(), Some(self.entries.len() / size_of::())) + } +} + +impl<'root, T: 'root> ExactSizeIterator for UnalignedPtrIter<'root, T> { + #[inline] + fn len(&self) -> usize { + self.entries.len() / size_of::() + } +} + +impl<'root, T: 'root> FusedIterator for UnalignedPtrIter<'root, T> {} diff --git a/kernel/src/device/acpi/root.rs b/kernel/src/device/acpi/root.rs new file mode 100644 index 0000000..bfde65b --- /dev/null +++ b/kernel/src/device/acpi/root.rs @@ -0,0 +1,76 @@ +use core::{any::type_name, fmt, iter::FusedIterator}; + +use crate::device::{ + AcpiResult, SystemTable, SystemTableHeader, + acpi::{AcpiParse, PackedPtr, TypedSystemTable, UnalignedPtrIter}, +}; + +#[derive(Clone, Copy)] +pub struct Rsdt<'root> { + entries: PackedPtr<'root>, +} + +impl fmt::Debug for Rsdt<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct(type_name::()) + .field_with("entries", |f| f.debug_list().entries(self.into_iter()).finish()) + .finish() + } +} + +impl<'root> IntoIterator for Rsdt<'root> { + type IntoIter = impl Iterator>> + ExactSizeIterator + FusedIterator + 'root; + type Item = ::Item; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + UnalignedPtrIter::::new(self.entries).map(|addr| unsafe { SystemTable::parse(addr as usize as *const SystemTableHeader) }) + } +} + +unsafe impl TypedSystemTable for Rsdt<'_> { + const SIGNATURE: [u8; 4] = *b"RSDT"; + type Out<'root> = Rsdt<'root>; + + #[inline] + unsafe fn from_table<'root>(table: &SystemTable<'root>) -> Rsdt<'root> { + Rsdt { + entries: PackedPtr::new(table.entries), + } + } +} + +#[derive(Clone, Copy)] +pub struct Xsdt<'root> { + entries: PackedPtr<'root>, +} + +impl fmt::Debug for Xsdt<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct(type_name::()) + .field_with("entries", |f| f.debug_list().entries(self.into_iter()).finish()) + .finish() + } +} + +impl<'root> IntoIterator for Xsdt<'root> { + type IntoIter = impl Iterator>> + ExactSizeIterator + FusedIterator + 'root; + type Item = ::Item; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + UnalignedPtrIter::::new(self.entries).map(|addr| unsafe { SystemTable::parse(addr as usize as *const SystemTableHeader) }) + } +} + +unsafe impl TypedSystemTable for Xsdt<'_> { + const SIGNATURE: [u8; 4] = *b"XSDT"; + type Out<'root> = Xsdt<'root>; + + #[inline] + unsafe fn from_table<'root>(table: &SystemTable<'root>) -> Xsdt<'root> { + Xsdt { + entries: PackedPtr::new(table.entries), + } + } +} diff --git a/kernel/src/device/imp/trampolines/x86_64.s b/kernel/src/device/imp/trampolines/x86_64.s new file mode 100644 index 0000000..302295f --- /dev/null +++ b/kernel/src/device/imp/trampolines/x86_64.s @@ -0,0 +1,124 @@ +.section .ap_trampoline, "aw" + +.global __ap_trampoline_start +.global __ap_trampoline_size + +__ap_trampoline_start: +.code16 +ap_entry_16: + # Clear interrupts and zero out segments + cli + xorw %ax, %ax + mov %ax, %ds + mov %ax, %es + mov %ax, %ss + + # Get the allocated address of `__ap_trampoline_start` from `cs` + xorl %ebx, %ebx + movw %cs, %bx + shll $4, %ebx + + # Set `.gdt_desc + 2` to `.gdt_start` relative to `bx` + leaw (.gdt_start - __ap_trampoline_start)(%bx), %ax + movw %ax, (.gdt_desc - __ap_trampoline_start + 2)(%bx) + lgdt (.gdt_desc - __ap_trampoline_start)(%bx) + + # Set `.jmp_32 + 2` to `ap_entry_32` relative to `bx` + leaw (ap_entry_32 - __ap_trampoline_start)(%bx), %ax + movw %ax, (.jmp_32 - __ap_trampoline_start + 2)(%bx) + + # Enable protected mode + movl %cr0, %eax + orl $0x00000001, %eax + movl %eax, %cr0 + + # Raw bytes for a far jump; necessary since the address is dynamically written +.jmp_32: + .byte 0x66, 0xea + .long 0x00000000 + .word 0x0008 +.align 4 +.gdt_start: + .quad 0x0000000000000000 # Null + .quad 0x00cf9a000000ffff # Kernel code 32-bit + .quad 0x00cf92000000ffff # Kernel data + .quad 0x00af9a000000ffff # Kernel code 64-bit +.gdt_end: +.gdt_desc: + .word .gdt_end - .gdt_start - 1 + .long 0x00000000 + +.code32 +ap_entry_32: + # Reinitialize segments and zero-extend base address + movw $0x10, %ax + movw %ax, %ds + movw %ax, %es + movw %ax, %ss + + # Zero out unused segments + xorw %ax, %ax + movw %ax, %fs + movw %ax, %gs + + # Set physical page table + movl (__ap_cr3 - __ap_trampoline_start)(%ebx), %eax + movl %eax, %cr3 + # Copy CR4 (which importantly includes Page Address Extension) + movl (__ap_cr4 - __ap_trampoline_start)(%ebx), %eax + movl %eax, %cr4 + + # Enable 64-bit long mode + movl $0xc0000080, %ecx # `IA32_EFER` + rdmsr + orl $(1 << 8), %eax + orl $(1 << 11), %eax + wrmsr + + # Set `.jmp_64 + 1` to `ap_entry_64` relative to `ebx` + movl $(ap_entry_64 - __ap_trampoline_start), %eax + addl %ebx, %eax + movl %eax, (.jmp_64 - __ap_trampoline_start + 1)(%ebx) + + # Enable virtual paging + movl %cr0, %eax + orl $(1 << 31), %eax + movl %eax, %cr0 + + # Raw bytes for a far jump; necessary since the address is dynamically written +.jmp_64: + .byte 0xea + .long 0x00000000 + .word 0x0018 + +.code64 +ap_entry_64: + # Setup stack pointer to the stack given by BSP + movq __ap_stack(%rip), %rsp + andq $-16, %rsp + + # Call an `extern "sysv64"` function given by the kernel + movq __ap_kernel_arg0(%rip), %rdi + movq __ap_kernel_arg1(%rip), %rsi + movq __ap_kernel_arg2(%rip), %rdx + jmpq *__ap_kernel_entry(%rip) + +.global __ap_cr3 +.global __ap_cr4 +.global __ap_stack +.global __ap_kernel_entry +.global __ap_kernel_arg0 +.global __ap_kernel_arg1 +.global __ap_kernel_arg2 + +.align 8 +__ap_cr3: .long 0x00000000 +__ap_cr4: .long 0x00000000 +__ap_stack: .quad 0x0000000000000000 +__ap_kernel_entry: .quad 0x0000000000000000 +__ap_kernel_arg0: .quad 0x0000000000000000 +__ap_kernel_arg1: .quad 0x0000000000000000 +__ap_kernel_arg2: .quad 0x0000000000000000 + +__ap_trampoline_end: +__ap_trampoline_size: .quad __ap_trampoline_end - __ap_trampoline_start \ No newline at end of file diff --git a/kernel/src/device/imp/x86_64.rs b/kernel/src/device/imp/x86_64.rs new file mode 100644 index 0000000..bca98fb --- /dev/null +++ b/kernel/src/device/imp/x86_64.rs @@ -0,0 +1,349 @@ +use alloc::boxed::Box; +use core::{ + arch::{asm, global_asm, x86_64::__cpuid_count}, + cell::{Cell, UnsafeCell}, + hint::spin_loop, + mem::{ManuallyDrop, offset_of}, + ptr::{self, NonNull}, + sync::atomic::{ + AtomicBool, AtomicU32, + Ordering::{AcqRel, Acquire, Relaxed, Release}, + }, + time::Duration, +}; + +use elpytios_bootinfo::{PAGE_SIZE, paddr::PAddr}; +use log::{debug, error, info}; + +use crate::{ + ScratchPages, + arch::x86_64::{ExtendedRegisterLayout, Msr, rdmsr, wrmsr}, + device::acpi::{LocalApicFlags, Madt, Pic}, + interrupt::{ + init_interrupts, + x86_64::{IdtIndex, InterruptStack, Tss}, + }, + statics::{get_phys_alloc, get_virtual_map, phys_to_virt}, + task::Task, + timer::{Timer, TimerX86_64}, + vaddr::{VAddr, VFlags}, +}; + +global_asm!(include_str!("trampolines/x86_64.s"), options(att_syntax, raw)); +unsafe extern "sysv64" { + static __ap_trampoline_start: u8; + static __ap_trampoline_size: usize; + + static mut __ap_cr3: u32; + static mut __ap_cr4: u32; + static mut __ap_stack: *mut u8; + static mut __ap_kernel_entry: unsafe extern "sysv64" fn(*const (), ApicDriver, *const [u32; 2]) -> !; + static mut __ap_kernel_arg0: *const (); + static mut __ap_kernel_arg1: ApicDriver; + static mut __ap_kernel_arg2: *const [u32; 2]; +} + +#[derive(Clone, Copy)] +pub enum ApicDriver { + XApic { mmr: NonNull }, + X2Apic, +} + +impl ApicDriver { + #[inline] + pub fn apic_id(self) -> u32 { + match self { + Self::XApic { mmr } => unsafe { (mmr.byte_add(0x20).read_volatile() >> 24) & 0xff }, + Self::X2Apic => unsafe { rdmsr(Msr::Ia32X2ApicId) as u32 }, + } + } + + #[inline] + pub unsafe fn end_of_interrupt(self) { + match self { + Self::XApic { .. } => unimplemented!("Sending End-of-Interrupts via legacy xAPIC isn't implemented yet"), + Self::X2Apic => unsafe { wrmsr(Msr::Ia32X2ApicEoi, 0) }, + } + } + + #[inline] + fn init_timers(self, timer: &Timer) { + match self { + Self::XApic { .. } => unimplemented!("Initializing timers via legacy xAPIC isn't implemented yet"), + Self::X2Apic => unsafe { + // - Bit 0-7: Set fallback handler vector + // - Bit 8: APIC enable in software + wrmsr(Msr::Ia32X2ApicSivr, IdtIndex::Spurious as u64 | (1 << 8)); + + match timer.inner { + TimerX86_64::Tsc { .. } => { + // - Bit 0-7: Vector + // - Bit 17-18: Mode (0=One-shot, 1=Periodic, 2=TSC-deadline, 3=Reserved) + wrmsr(Msr::Ia32X2ApicLvtTimer, IdtIndex::ScheduleTimer as u64 | (2 << 17)); + } + } + }, + } + } + + #[inline] + unsafe fn init_core(self, apic_id: u32) { + match self { + Self::XApic { .. } => unimplemented!("Waking up cores via legacy xAPIC isn't implemented yet"), + Self::X2Apic => unsafe { + let id = (apic_id as u64) << 32; + let assert = 1 << 14; + + wrmsr(Msr::Ia32X2ApicIcr, (5 << 8) | assert | id); + }, + } + } + + #[inline] + unsafe fn startup_core(self, scratch_timer: &Timer, send_init: bool, trampoline_phys: PAddr, apic_id: u32) { + match self { + Self::XApic { .. } => unimplemented!("Waking up cores via legacy xAPIC isn't implemented yet"), + Self::X2Apic => unsafe { + // IA32_X2APIC_ICR: + // - Bit 0-7: Vector + // - Bit 8-10: Delivery mode (4=NMI, 5=Init, 6=Startup) + // - Bit 14: Assert flag + // - Bit 32-63: Target core destination APIC ID + let id = (apic_id as u64) << 32; + let assert = 1 << 14; + + if send_init { + self.init_core(apic_id); + scratch_timer.busy_wait(Duration::from_millis(10)); + } + + wrmsr( + Msr::Ia32X2ApicIcr, + ((trampoline_phys.addr() / PAGE_SIZE) & 0xff) as u64 | (6 << 8) | assert | id, + ); + }, + } + } +} + +#[repr(C)] +pub struct CpuContext { + // Common fields across all architectures + this: *const Self, + pub is_bootstrap: bool, + pub apic_id: u32, + pub cpu_id: u32, + pub timer: Timer, + pub current_task: UnsafeCell>, + // x86_64-specific fields + apic: ApicDriver, + pub timer_callback: Cell>, + pub registers: ExtendedRegisterLayout, + /// Task state segment + pub tss: Tss, +} + +impl CpuContext { + unsafe fn install(apic: ApicDriver, is_bootstrap: bool, apic_id: u32, cpu_id: u32) { + if let ApicDriver::X2Apic = apic { + unsafe { wrmsr(Msr::Ia32ApicBase, rdmsr(Msr::Ia32ApicBase) | (1 << 10) | (1 << 11)) } + } + + let timer = Timer::new(); + apic.init_timers(&timer); + + let this = Box::into_raw(Box::new(Self { + this: ptr::null(), + is_bootstrap, + apic_id, + cpu_id, + timer, + current_task: UnsafeCell::new(None), + apic, + timer_callback: Cell::new(None), + registers: unsafe { ExtendedRegisterLayout::new() }, + tss: Tss::new(), + })); + + unsafe { + (*this).this = this; + wrmsr(Msr::Ia32GsBase, this as u64); + + init_interrupts(Self::get()) + } + } + + #[inline(always)] + pub fn get() -> &'static Self { + let ptr: *const Self; + unsafe { + asm!( + "movq %gs:[0], {}", + + out(reg) ptr, + options(att_syntax, pure, readonly, nostack, preserves_flags), + ); + ptr.as_ref_unchecked() + } + } + + #[inline] + pub unsafe fn end_of_interrupt(&self) { + unsafe { self.apic.end_of_interrupt() } + } + + #[inline] + pub const fn stack_addr(index: InterruptStack) -> usize { + offset_of!(Self, tss) + Tss::stack_addr(index) + } +} + +pub unsafe fn init_device_tree ! + Clone + Send>(scratch_pages: &mut ScratchPages, processor_entry: F, madt: Madt) -> ! { + unsafe { + let v_map = get_virtual_map(); + let driver = if __cpuid_count(0x01, 0x00).ecx & (1 << 21) != 0 { + info!("x2APIC is supported on this hardware; using Model-Specific Registers for APIC"); + + wrmsr(Msr::Ia32ApicBase, rdmsr(Msr::Ia32ApicBase) | (1 << 10) | (1 << 11)); + ApicDriver::X2Apic + } else { + info!("x2APIC is unsupported on this hardware; falling back to legacy memory-mapped xAPIC"); + let mmr_phys = PAddr::new(madt.local_interrupt_control_addr as usize); + let mmr = phys_to_virt(mmr_phys); + + v_map + .map( + mmr_phys, + mmr, + 1, + VFlags::GLOBAL | VFlags::WRITABLE | VFlags::CACHE_DISABLED | VFlags::EXECUTE_DISABLE, + ) + .expect("Couldn't virtual-map xAPIC MMR"); + + ApicDriver::XApic { + mmr: NonNull::new_unchecked(mmr.ptr_mut::()), + } + }; + + let trampoline_phys = scratch_pages.take().expect("Not enough scratch pages for AP trampoline entry"); + let trampoline = phys_to_virt(trampoline_phys); + v_map + .map(trampoline_phys, VAddr::new(trampoline_phys.addr()), 1, VFlags::empty()) + .expect("Couldn't identity-map trampoline code"); + v_map + .map(trampoline_phys, trampoline, 1, VFlags::WRITABLE | VFlags::EXECUTE_DISABLE) + .expect("Couldn't virtual-map trampoline code"); + let trampoline = trampoline.ptr_mut::(); + debug!("\tCopying {__ap_trampoline_size} bytes into {trampoline:p} (physical address at {trampoline_phys:p}) for AP cores entry"); + + // Setup properties common to all AP cores + let cr3: usize; + asm!("movq %cr3, {}", out(reg) cr3, options(att_syntax, nomem, nostack, preserves_flags)); + let cr4: usize; + asm!("movq %cr4, {}", out(reg) cr4, options(att_syntax, nomem, nostack, preserves_flags)); + + __ap_cr3 = u32::try_from(cr3).expect("Page table physical address must be within 32-bit address"); + __ap_cr4 = u32::try_from(cr4).expect("Page table physical address must be within 32-bit address"); + __ap_kernel_entry = ap_kernel_entry::; + + static AP_INIT: AtomicBool = AtomicBool::new(false); + static AP_PROCEED: AtomicU32 = AtomicU32::new(0); + + unsafe extern "sysv64" fn ap_kernel_entry ! + Clone + Send>( + processor_entry: *const (), + apic: ApicDriver, + ids: *const [u32; 2], + ) -> ! { + let processor_entry = unsafe { + let [apic_id, cpu_id] = ids.read_unaligned(); + CpuContext::install(apic, false, apic_id, cpu_id); + + (processor_entry as *const F).read_unaligned() + }; + + AP_INIT.store(true, Release); + let cpu_count = loop { + match AP_PROCEED.load(Acquire) { + 0 => { + spin_loop(); + continue + } + n => break n, + } + }; + + processor_entry(cpu_count) + } + + let bsp_id = driver.apic_id(); + let mut cpu_id = 0; + + let scratch_timer = Timer::new(); + let mut init_cpu = |apic_id: u32| { + if bsp_id == apic_id { + CpuContext::install(driver, true, apic_id, cpu_id); + cpu_id += 1; + } else { + const STACK_PAGES: usize = 16; + + let stack = get_phys_alloc() + .lock() + .alloc(STACK_PAGES.ilog2()) + .expect("Couldn't allocate stack for AP core"); + let stack_top = phys_to_virt(stack).byte_add(STACK_PAGES * PAGE_SIZE).ptr_mut(); + let processor_entry = ManuallyDrop::new(processor_entry.clone()); + let ids = [apic_id, cpu_id]; + + // Setup properties for specific AP cores + __ap_stack = stack_top; + __ap_kernel_arg0 = (&raw const processor_entry).cast(); + __ap_kernel_arg1 = driver; + __ap_kernel_arg2 = &raw const ids; + + trampoline.copy_from_nonoverlapping(&raw const __ap_trampoline_start, __ap_trampoline_size); + + AP_INIT.store(false, Release); + driver.startup_core(&scratch_timer, true, trampoline_phys, apic_id); + + for i in 0..2 { + scratch_timer.busy_wait(Duration::from_millis(1)); + match AP_INIT.compare_exchange(true, false, AcqRel, Relaxed) { + Ok(..) => { + cpu_id += 1; + debug!("\tAP core {apic_id} is up and running"); + break + } + Err(..) => { + if i == 0 { + driver.startup_core(&scratch_timer, false, trampoline_phys, apic_id); + error!("\tCouldn't start up AP core {apic_id}, retrying one more time"); + } else { + // Send one last INIT IPI to ensure the AP core isn't doing anything + driver.init_core(apic_id); + error!("\tCouldn't start up AP core {apic_id} even after retrying, giving up"); + + // Wait 10 milliseconds just to absolutely ensure the AP core isn't running + scratch_timer.busy_wait(Duration::from_millis(10)); + } + } + } + } + } + }; + + for pic in madt { + match pic { + Pic::ProcessorLocal(proc) if (*&raw const proc.flags).contains(LocalApicFlags::ENABLED) => { + init_cpu(proc.apic_id as u32); + } + Pic::ProcessLocalX2(proc) if (*&raw const proc.flags).contains(LocalApicFlags::ENABLED) => { + init_cpu(proc.x2apic_id as u32); + } + _ => {} + } + } + + AP_PROCEED.store(cpu_id, Release); + processor_entry(cpu_id) + } +} diff --git a/kernel/src/device/mod.rs b/kernel/src/device/mod.rs new file mode 100644 index 0000000..8f8b8ba --- /dev/null +++ b/kernel/src/device/mod.rs @@ -0,0 +1,79 @@ +mod imp { + cfg_select! { + target_arch = "x86_64" => { + mod x86_64; + pub use x86_64::*; + } + _ => { + compile_error!("Unsupported architecture"); + } + } +} + +use elpytios_bootinfo::DeviceTree; +pub use imp::CpuContext; +use log::info; + +mod acpi; +use acpi::*; + +use crate::ScratchPages; + +impl !Send for CpuContext {} +impl !Sync for CpuContext {} + +unsafe fn init_device_tree_impl( + scratch_pages: &mut ScratchPages, + processor_entry: impl FnOnce(u32) -> ! + Clone + Send, + system_tables: impl IntoIterator>, +) -> ! { + let system_tables = system_tables.into_iter(); + info!("Initializing device tree: found {} system tables", system_tables.len()); + + macro_rules! tables { + ($($output:ident: $type:ty;)*) => { + $(let mut $output = None::<$type>;)* + for system_table in system_tables { + let system_table = system_table.expect("Couldn't parse system table"); + $(match system_table.typed::<$type>() { + Ok(_table) => { + if $output.replace(_table).is_some() { + panic!("Duplicate '{}' entries", SignatureFmt(&<$type as TypedSystemTable>::SIGNATURE)); + } + } + Err(AcpiError::InvalidTableSignature { .. }) => {} + Err(e) => panic!("Couldn't parse typed system table: {e}"), + })* + } + $(let $output = $output.unwrap_or_else(|| panic!("Missing '{}' entries", SignatureFmt(&<$type as TypedSystemTable>::SIGNATURE)));)* + }; + } + + tables! { + madt: Madt; + } + + unsafe { imp::init_device_tree(scratch_pages, processor_entry, madt) } +} + +/// # Safety +/// - Only call this once in setup phase after higher-half addressing is finished. +/// - Identity-mapping must still be available. +pub unsafe fn init_device_tree( + device_tree: DeviceTree, + scratch_pages: &mut ScratchPages, + processor_entry: impl FnOnce(u32) -> ! + Clone + Send, +) -> ! { + match device_tree { + DeviceTree::Acpi(addr) => { + let rsdp = unsafe { Rsdp::new(addr.addr() as *const Rsdp) }.expect("Couldn't parse RSDP"); + let rsdt = rsdp.rsdt().expect("Couldn't parse RSDT"); + unsafe { init_device_tree_impl(scratch_pages, processor_entry, rsdt) } + } + DeviceTree::Acpi2(addr) => { + let xsdp = unsafe { Xsdp::new(addr.addr() as *const Xsdp) }.expect("Couldn't parse XSDP"); + let xsdt = xsdp.xsdt().expect("Couldn't parse XSDT"); + unsafe { init_device_tree_impl(scratch_pages, processor_entry, xsdt) } + } + }; +} diff --git a/kernel/src/framebuffer.rs b/kernel/src/framebuffer.rs new file mode 100644 index 0000000..d4e5367 --- /dev/null +++ b/kernel/src/framebuffer.rs @@ -0,0 +1,10 @@ +use elpytios_bootinfo::PixelFormat; + +#[derive(Debug)] +pub struct FrameBuffer { + pub width: usize, + pub height: usize, + pub stride: usize, + pub format: PixelFormat, + pub pointer: *mut u8, +} diff --git a/kernel/src/interrupt/mod.rs b/kernel/src/interrupt/mod.rs new file mode 100644 index 0000000..7878417 --- /dev/null +++ b/kernel/src/interrupt/mod.rs @@ -0,0 +1,41 @@ +use elpytios_abi::FileHandle; + +cfg_select! { + target_arch = "x86_64" => { + pub mod x86_64; + pub use x86_64::init_interrupts; + } + _ => { + compile_error!("Unsupported architecture"); + } +} + +#[inline] +unsafe fn page_fault( + ptr: *mut (), + missing_or_protected: bool, + caused_by_write: bool, + triggered_by_user: bool, + overwritten_reserved_bits: bool, + instruction_fetch_violation: bool, +) { + panic!( + "Page fault at address {ptr:p}\n\ + Missing/access\t: {missing_or_protected}\n\ + Caused by write\t: {caused_by_write}\n\ + From userland\t: {triggered_by_user}\n\ + Reserved bits\t: {overwritten_reserved_bits}\n\ + Inst fetch\t: {instruction_fetch_violation}" + ) +} + +#[inline] +unsafe fn mem_map(file: FileHandle, offset: usize, page_count: usize, flags: usize) -> *mut u8 { + panic!( + "SYSCALL mem_map\n\ + File\t: {file:?}\n\ + Offset\t: {offset}\n\ + Count\t: {page_count}\n\ + Flags\t: {flags}" + ) +} diff --git a/kernel/src/interrupt/x86_64/gdt.rs b/kernel/src/interrupt/x86_64/gdt.rs new file mode 100644 index 0000000..d100324 --- /dev/null +++ b/kernel/src/interrupt/x86_64/gdt.rs @@ -0,0 +1,85 @@ +use alloc::boxed::Box; +use core::arch::asm; + +use bitflags::bitflags; + +use crate::{device::CpuContext, interrupt::x86_64::TssEntry}; + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct GdtEntry(pub(super) u64); +bitflags! { + impl GdtEntry: u64 { + const ACCESSED = 1 << 40; + const WRITABLE = 1 << 41; + const EXECUTABLE = 1 << 43; + const DESCRIPTOR_TYPE = 1 << 44; + + const DPL_0 = 0 << 45; + const DPL_3 = 3 << 45; + + const PRESENT = 1 << 47; + const LONG_MODE = 1 << 53; + } +} + +impl GdtEntry { + pub const NULL: Self = Self(0); + + pub const KERNEL_CODE: Self = Self(Self::PRESENT.0 | Self::DESCRIPTOR_TYPE.0 | Self::EXECUTABLE.0 | Self::LONG_MODE.0 | Self::DPL_0.0); + pub const KERNEL_DATA: Self = Self(Self::PRESENT.0 | Self::DESCRIPTOR_TYPE.0 | Self::WRITABLE.0 | Self::DPL_0.0); + + pub const USER_CODE: Self = Self(Self::PRESENT.0 | Self::DESCRIPTOR_TYPE.0 | Self::EXECUTABLE.0 | Self::LONG_MODE.0 | Self::DPL_3.0); + pub const USER_DATA: Self = Self(Self::PRESENT.0 | Self::DESCRIPTOR_TYPE.0 | Self::WRITABLE.0 | Self::DPL_3.0); +} + +pub unsafe fn init_gdt(cpu: &'static CpuContext) { + #[repr(C, packed)] + struct GdtPointer { + limit: u16, + base: *mut GdtEntry, + } + + let [tss_lower, tss_upper] = TssEntry::new(&cpu.tss).to_gdt_entries(); + unsafe { + let entries = Box::leak(Box::new([ + GdtEntry::NULL, // 0x00 + GdtEntry::KERNEL_CODE, // 0x08 + GdtEntry::KERNEL_DATA, // 0x10 + GdtEntry::USER_DATA, // 0x18 + GdtEntry::USER_CODE, // 0x20 + tss_lower, // 0x28 + tss_upper, + ])); + + let ptr = GdtPointer { + limit: u16::try_from(size_of_val(entries) - 1).unwrap(), + base: (&raw mut *entries).cast(), + }; + + asm!( + "lgdt ({ptr})", + // TSS selector at 0x28 + "movw $0x28, %ax", + "ltr %ax", + + // `KERNEL_DATA` selector is 0x10 + "movw $0x10, %ax", + "movw %ax, %ds", + "movw %ax, %es", + "movw %ax, %ss", + // `KERNEL_CODE` selector is 0x08 + "push $0x08", + // Perform a far jump, loading the GDT entries + "leaq 2f(%rip), %rax", + "push %rax", + "lretq", + "2:", + + ptr = in(reg) &ptr, + out("rax") _, + + options(att_syntax), + ); + } +} diff --git a/kernel/src/interrupt/x86_64/idt.rs b/kernel/src/interrupt/x86_64/idt.rs new file mode 100644 index 0000000..2d33f54 --- /dev/null +++ b/kernel/src/interrupt/x86_64/idt.rs @@ -0,0 +1,208 @@ +use core::{ + arch::{asm, naked_asm}, + mem::size_of_val_raw, +}; + +use bitflags::bitflags; +use bytemuck::Zeroable; + +use super::InterruptFrame; +use crate::{ + device::CpuContext, + interrupt::{page_fault, x86_64::InterruptStack}, + spin_sync::SpinOnce, +}; + +#[derive(Copy, Clone, Zeroable)] +#[repr(C, packed)] +pub struct IdtEntry { + pointer_low: u16, + gdt_selector: u16, + options: IdtOptions, + pointer_middle: u16, + pointer_high: u32, + reserved: u32, +} + +impl IdtEntry { + #[inline] + pub unsafe fn new(handler: unsafe extern "sysv64" fn() -> !, stack: InterruptStack) -> Self { + let addr = handler as usize; + Self { + pointer_low: addr as u16, + pointer_middle: (addr >> 16) as u16, + pointer_high: (addr >> 32) as u32, + gdt_selector: 0x08, // `KERNEL_CODE` selector + options: IdtOptions( + IdtOptions::PRESENT.0 | IdtOptions::DPL_RING_0.0 | IdtOptions::TYPE_INTERRUPT.0 | (stack as u16) & IdtOptions::IST_MASK.0, + ), + reserved: 0, + } + } +} + +#[derive(Debug, Clone, Copy, Zeroable)] +#[repr(transparent)] +pub struct IdtOptions(u16); +bitflags! { + impl IdtOptions: u16 { + const PRESENT = 1 << 15; + + const IST_MASK = 7; + + const DPL_RING_0 = 0 << 12; + const DPL_RING_3 = 3 << 12; + + const TYPE_INTERRUPT = 0xe << 8; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(u8)] +pub enum IdtIndex { + // Hard-coded by CPU + DoubleFault = 0x08, + PageFault = 0x0e, + // Programmable interrupts; must send End-of-Interrupt before returning + ScheduleTimer = 0x20, + // Spurious vector + Spurious = 0xff, +} + +#[macro_export] +macro_rules! interrupt { + (#[$($has_error:tt)*] $handle:ident) => { + { + const _: unsafe extern "sysv64" fn(&interrupt!(type => #[$($has_error)*])) = $handle; + naked_asm!( + r#" + callq {save} + callq {handle} + callq {load} + iretq + "#, + + save = sym #[$($has_error)*])>::save, + handle = sym $handle, + load = sym #[$($has_error)*])>::load, + + options(att_syntax), + ) + } + }; + (type => #[error]) => { + InterruptFrame + }; + (type => #[not(error)]) => { + InterruptFrame<()> + }; +} + +#[unsafe(naked)] +pub unsafe extern "sysv64" fn int_double_fault() -> ! { + unsafe extern "sysv64" fn handle(frame: &InterruptFrame) { + panic!("Double-fault caught (Hardware error code: {})", frame.error) + } + + interrupt!( + #[error] + handle + ) +} + +#[unsafe(naked)] +pub unsafe extern "sysv64" fn int_page_fault() -> ! { + #[repr(transparent)] + struct ErrorCode(u64); + bitflags! { + impl ErrorCode: u64 { + // 0=protection violation, 1=not present + const NOT_PRESENT = 1 << 0; + // 0=caused by read, 1=caused by read + const IS_WRITE = 1 << 1; + // 0=triggered in ring 0, 1=triggered in ring 3 + const IS_USER = 1 << 2; + // overwrote reserved bits in page table + const RESERVED = 1 << 3; + // instruction fetch violation + const EXECUTE = 1 << 4; + } + } + + unsafe extern "sysv64" fn handle(frame: &InterruptFrame) { + unsafe { + let ptr: *mut (); + asm!("mov {ptr}, cr2", ptr = out(reg) ptr); + + let code = ErrorCode(frame.error); + page_fault( + ptr, + code.contains(ErrorCode::NOT_PRESENT), + code.contains(ErrorCode::IS_WRITE), + code.contains(ErrorCode::IS_USER), + code.contains(ErrorCode::RESERVED), + code.contains(ErrorCode::EXECUTE), + ) + } + } + + interrupt!( + #[error] + handle + ) +} + +#[unsafe(naked)] +pub unsafe extern "sysv64" fn int_timer() -> ! { + unsafe extern "sysv64" fn handle(_frame: &InterruptFrame) { + let cpu = CpuContext::get(); + if let Some(func) = cpu.timer_callback.get() { + func() + } + + unsafe { cpu.end_of_interrupt() } + } + + interrupt!( + #[not(error)] + handle + ) +} + +#[unsafe(naked)] +pub unsafe extern "sysv64" fn int_spurious() -> ! { + naked_asm!("iretq", options(att_syntax)) +} + +pub unsafe fn init_idt() { + static mut IDT_ENTRIES: [IdtEntry; 256] = [bytemuck::zeroed(); 256]; + static IDT_INIT: SpinOnce = SpinOnce::new(); + + IDT_INIT.call_once(|| unsafe { + IDT_ENTRIES[IdtIndex::DoubleFault as usize] = IdtEntry::new(int_double_fault, InterruptStack::DoubleFault); + IDT_ENTRIES[IdtIndex::PageFault as usize] = IdtEntry::new(int_page_fault, InterruptStack::Task); + + IDT_ENTRIES[IdtIndex::ScheduleTimer as usize] = IdtEntry::new(int_timer, InterruptStack::Task); + IDT_ENTRIES[IdtIndex::Spurious as usize] = IdtEntry::new(int_spurious, InterruptStack::Task); + }); + + #[repr(C, packed)] + struct IdtPointer { + limit: u16, + base: *mut IdtEntry, + } + + unsafe { + let ptr = IdtPointer { + limit: u16::try_from(size_of_val_raw(&raw const IDT_ENTRIES) - 1).unwrap(), + base: (&raw mut IDT_ENTRIES).cast(), + }; + + asm!( + "lidt ({ptr})", + ptr = in(reg) &ptr, + + options(att_syntax), + ); + } +} diff --git a/kernel/src/interrupt/x86_64/mod.rs b/kernel/src/interrupt/x86_64/mod.rs new file mode 100644 index 0000000..0de1029 --- /dev/null +++ b/kernel/src/interrupt/x86_64/mod.rs @@ -0,0 +1,196 @@ +use core::{arch::naked_asm, mem::offset_of}; + +use crate::device::CpuContext; + +mod gdt; +mod idt; +mod syscall; +mod tss; +pub use gdt::*; +pub use idt::*; +pub use syscall::*; +pub use tss::*; + +mod sealed { + pub trait InterruptError: Sized { + const STACK_ADJUST: usize; + } + + impl InterruptError for () { + const STACK_ADJUST: usize = 0; + } + + impl InterruptError for u64 { + const STACK_ADJUST: usize = 8; + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct InterruptFrame { + // General-purpose registers, pushed by software + pub rax: u64, + pub rcx: u64, + pub rdx: u64, + pub rsi: u64, + pub rdi: u64, + pub r8: u64, + pub r9: u64, + pub r10: u64, + pub r11: u64, + pub rbx: u64, + pub rbp: u64, + pub r12: u64, + pub r13: u64, + pub r14: u64, + pub r15: u64, + // Interupt info, pushed by hardware + pub error: Error, + pub rip: u64, + pub cs: u64, + pub rflags: u64, + pub rsp: u64, + pub ss: u64, +} + +impl InterruptFrame { + pub const STACK_ADJUST: usize = Error::STACK_ADJUST; +} + +impl InterruptFrame { + pub const INTERRUPT_BUFFER_SIZE: usize = size_of::() - offset_of!(Self, error); + + /// # Safety + /// - Invoke this function with `call` instruction directly. + /// - `%rsp` must point to [`Self::error`] before the `call` instruction, which is guaranteed + /// inside interrupt handlers. + /// - After this function returns, `%rsp` is now a 16 bytes-aligned address, and `%rdi` is now a + /// pointer to [`InterruptFrame`]. + #[unsafe(naked)] + pub unsafe extern "sysv64" fn save() -> ! { + naked_asm!( + "subq $({error} - {rax} + {adj} - 8), %rsp", // `%rsp->r15` is currently the return address + + "movq %rax, ({adj} + {rax})(%rsp)", + "movq ({adj} + {r15})(%rsp), %rax", // `%rax` is now clobbered, containing the return address + "movq %rcx, ({adj} + {rcx})(%rsp)", + "movq %rdx, ({adj} + {rdx})(%rsp)", + "movq %rsi, ({adj} + {rsi})(%rsp)", + "movq %rdi, ({adj} + {rdi})(%rsp)", + "movq %r8, ({adj} + {r8})(%rsp)", + "movq %r9, ({adj} + {r9})(%rsp)", + "movq %r10, ({adj} + {r10})(%rsp)", + "movq %r11, ({adj} + {r11})(%rsp)", + "movq %rbx, ({adj} + {rbx})(%rsp)", + "movq %rbp, ({adj} + {rbp})(%rsp)", + "movq %r12, ({adj} + {r12})(%rsp)", + "movq %r13, ({adj} + {r13})(%rsp)", + "movq %r14, ({adj} + {r14})(%rsp)", + "movq %r15, ({adj} + {r15})(%rsp)", + + "testq $3, ({adj} + {cs})(%rsp)", + "jz 2f", + "swapgs", + + "2:", + "leaq {adj}(%rsp), %rdi", + "jmpq *%rax", + + adj = const Error::STACK_ADJUST, + + rax = const offset_of!(Self, rax), + rcx = const offset_of!(Self, rcx), + rdx = const offset_of!(Self, rdx), + rsi = const offset_of!(Self, rsi), + rdi = const offset_of!(Self, rdi), + r8 = const offset_of!(Self, r8), + r9 = const offset_of!(Self, r9), + r10 = const offset_of!(Self, r10), + r11 = const offset_of!(Self, r11), + rbx = const offset_of!(Self, rbx), + rbp = const offset_of!(Self, rbp), + r12 = const offset_of!(Self, r12), + r13 = const offset_of!(Self, r13), + r14 = const offset_of!(Self, r14), + r15 = const offset_of!(Self, r15), + error = const offset_of!(Self, error), + cs = const offset_of!(Self, cs), + + options(att_syntax), + ) + } + + /// # Safety + /// - Invoke this function with `call` instruction directly. + /// - `%rsp` must be restored to wherever it points to after [`Self::push`] returns. + /// - After this function returns, `%rsp` now points to [`Self::error`], which is convenient for + /// `iretq`s (in case the interrupt has an error, you must `addq $8, %rsp` as well to clear + /// the error code). + #[unsafe(naked)] + pub unsafe extern "sysv64" fn load() -> ! { + naked_asm!( + "addq $8, %rsp", + + "movq ({adj} + {rax})(%rsp), %rax", + "movq ({adj} + {rcx})(%rsp), %rcx", + "movq ({adj} + {rdx})(%rsp), %rdx", + "movq ({adj} + {rsi})(%rsp), %rsi", + "movq ({adj} + {rdi})(%rsp), %rdi", + "movq ({adj} + {r8})(%rsp), %r8", + "movq ({adj} + {r9})(%rsp), %r9", + "movq ({adj} + {r10})(%rsp), %r10", + "movq ({adj} + {r11})(%rsp), %r11", + "movq ({adj} + {rbx})(%rsp), %rbx", + "movq ({adj} + {rbp})(%rsp), %rbp", + "movq ({adj} + {r12})(%rsp), %r12", + "movq ({adj} + {r13})(%rsp), %r13", + "movq ({adj} + {r14})(%rsp), %r14", + "movq ({adj} + {r15})(%rsp), %r15", + + "testq $3, ({adj} + {cs})(%rsp)", + "jz 2f", + "movq %rsp, %gs:{stack}", + "addq $({adj} + {error} - {rax} + {frame}), %gs:{stack}", + "swapgs", + + "2:", + "addq $({adj} + {rip} - {rax}), %rsp", + "jmpq *-({adj} + {rip} - {rax} + 8)(%rsp)", + + adj = const Error::STACK_ADJUST, + stack = const CpuContext::stack_addr(InterruptStack::Task), + frame = const Self::INTERRUPT_BUFFER_SIZE, + + rax = const offset_of!(Self, rax), + rcx = const offset_of!(Self, rcx), + rdx = const offset_of!(Self, rdx), + rsi = const offset_of!(Self, rsi), + rdi = const offset_of!(Self, rdi), + r8 = const offset_of!(Self, r8), + r9 = const offset_of!(Self, r9), + r10 = const offset_of!(Self, r10), + r11 = const offset_of!(Self, r11), + rbx = const offset_of!(Self, rbx), + rbp = const offset_of!(Self, rbp), + r12 = const offset_of!(Self, r12), + r13 = const offset_of!(Self, r13), + r14 = const offset_of!(Self, r14), + r15 = const offset_of!(Self, r15), + error = const offset_of!(Self, error), + rip = const offset_of!(Self, rip), + cs = const offset_of!(Self, cs), + + options(att_syntax), + ) + } +} + +/// # Safety +/// Only call this once per CPU core in setup phase after higher-half addressing is finished. +pub unsafe fn init_interrupts(cpu: &'static CpuContext) { + unsafe { + init_gdt(cpu); + init_idt(); + init_syscalls(); + } +} diff --git a/kernel/src/interrupt/x86_64/syscall.rs b/kernel/src/interrupt/x86_64/syscall.rs new file mode 100644 index 0000000..e149645 --- /dev/null +++ b/kernel/src/interrupt/x86_64/syscall.rs @@ -0,0 +1,106 @@ +use core::arch::naked_asm; + +use elpytios_abi::{FileHandle, Syscall, SyscallArg, SyscallEntry}; + +use crate::{ + arch::x86_64::{Msr, rdmsr, wrmsr}, + device::CpuContext, + interrupt::x86_64::InterruptStack, + spin_sync::SpinOnce, +}; + +pub unsafe extern "sysv64" fn write(_file: FileHandle, _buffer: *const u8, _len: usize) -> usize { + Syscall::INVALID +} + +pub unsafe extern "sysv64" fn read(_file: FileHandle, _buffer: *mut u8, _len: usize) -> usize { + Syscall::INVALID +} + +pub unsafe extern "sysv64" fn mem_map(file: FileHandle, offset: usize, page_count: usize, flags: usize) -> usize { + unsafe { super::super::mem_map(file, offset, page_count, flags) }.into_usize() +} + +pub unsafe extern "sysv64" fn yield_now() -> usize { + crate::task::schedule(); + 0 +} + +pub unsafe fn init_syscalls() { + static mut SYSCALL_ENTRIES: [SyscallEntry; Syscall::MAX_ENTRIES] = [SyscallEntry::MISSING; Syscall::MAX_ENTRIES]; + static SYSCALL_INIT: SpinOnce = SpinOnce::new(); + + SYSCALL_INIT.call_once(|| unsafe { + SYSCALL_ENTRIES[Syscall::Write as usize] = SyscallEntry { write }; + SYSCALL_ENTRIES[Syscall::Read as usize] = SyscallEntry { read }; + + SYSCALL_ENTRIES[Syscall::MemMap as usize] = SyscallEntry { mem_map }; + + SYSCALL_ENTRIES[Syscall::YieldNow as usize] = SyscallEntry { yield_now }; + }); + + unsafe { + // Enable `syscall` and `sysret` + wrmsr(Msr::Ia32Efer, rdmsr(Msr::Ia32Efer) | (1 << 0)); + // 0x08: KERNEL_CODE + // 0x10 + 8 = 0x18: USER_DATA + // 0x10 + 16 = 0x20: USER_CODE + wrmsr(Msr::Ia32Star, (0x08 << 32) | (0x10 << 48)); + wrmsr(Msr::Ia32Lstar, syscall as *const () as u64); + // Bit 9: Interrupt flag (`cli`) + // Bit 10: Direction flag (`cld`) + // Bit 18: Alignment check + wrmsr(Msr::Ia32Fmask, (1 << 9) | (1 << 10) | (1 << 18)); + + #[unsafe(naked)] + pub unsafe extern "sysv64" fn syscall() -> ! { + naked_asm!( + // `rax` is the `syscall` entry, immediately bail if invalid + "cmpq ${max_entries}, %rax", + "jae 2f", + + // `rax` is now address of the handler, bail if not set (null) + // `r12` is supposed to be treated as caller-saved (breaking the traditional Sys V, but it's not a strict requirement anyway) + "leaq {entries}(%rip), %r12", + "movq (%r12, %rax, 8), %rax", + "testq %rax, %rax", + "jz 2f", + + // Switch to kernel stack + "swapgs", + "movq %rsp, %r12", + "movq %gs:{stack}, %rsp", + + "push %r12", + "push %r11", + "push %rcx", + "subq $8, %rsp", + + // User uses `r10` instead of `rcx`, but Sys V expects `rcx` to be 4th arg + "movq %r10, %rcx", + // After this, `rax` is now the return value of the handler + "callq *%rax", + + "addq $8, %rsp", + "pop %rcx", + "pop %r11", + // Switch back to user stack + "pop %rsp", + "swapgs", + + "sysretq", + + "2:", + "movq ${invalid}, %rax", + "sysretq", + + max_entries = const Syscall::MAX_ENTRIES, + entries = sym SYSCALL_ENTRIES, + stack = const CpuContext::stack_addr(InterruptStack::Task), + invalid = const Syscall::INVALID, + + options(att_syntax), + ) + } + } +} diff --git a/kernel/src/interrupt/x86_64/tss.rs b/kernel/src/interrupt/x86_64/tss.rs new file mode 100644 index 0000000..73367a9 --- /dev/null +++ b/kernel/src/interrupt/x86_64/tss.rs @@ -0,0 +1,105 @@ +use core::{ + cell::UnsafeCell, + mem::{self, offset_of}, +}; + +use super::GdtEntry; + +#[derive(Debug, Clone, Copy)] +#[repr(u8)] +pub enum InterruptStack { + /// Use [`Tss::rsp0`]. + Task = 0, + /// Use [`Tss::ist`]`[0]`. + DoubleFault = 1, +} + +#[repr(C, packed)] +pub struct Tss { + reserved0: u32, + /// Switch to this stack only when intercepting an interrupt from Ring 3. Set this to + /// [`Task`](crate::task::Task)'s general-purpose registers address. + pub rsp0: UnsafeCell, + pub rsp1: UnsafeCell, + pub rsp2: UnsafeCell, + reserved1: u64, + ist: [UnsafeCell; 7], + reserved2: u64, + reserved3: u16, + iopb_offset: u16, +} + +impl Tss { + #[inline] + pub const fn new() -> Self { + Self { + reserved0: 0, + rsp0: UnsafeCell::new(0), + rsp1: UnsafeCell::new(0), + rsp2: UnsafeCell::new(0), + reserved1: 0, + ist: [const { UnsafeCell::new(0) }; 7], + reserved2: 0, + reserved3: 0, + iopb_offset: 0xffff, + } + } + + #[inline] + pub const fn set_stack(&self, index: InterruptStack, addr: u64) { + unsafe { self.stack_ptr(index).write_unaligned(addr) } + } + + #[inline] + pub const fn stack_ptr(&self, index: InterruptStack) -> *mut u64 { + UnsafeCell::raw_get(unsafe { (self as *const Self).cast::>().byte_add(Self::stack_addr(index)) }) + } + + #[inline] + pub const fn stack_addr(index: InterruptStack) -> usize { + match (index as usize).checked_sub(1) { + None => offset_of!(Self, rsp0), + Some(i) => offset_of!(Self, ist) + i * size_of::(), + } + } +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct TssEntry { + limit_low: u16, + base_low: u16, + base_mid: u8, + type_flags: u8, + limit_high_flags: u8, + base_high: u8, + base_upper: u32, + reserved: u32, +} + +impl TssEntry { + #[inline] + pub fn new(tss: &'static Tss) -> Self { + let addr = tss as *const Tss as u64; + let limit = (size_of::() - 1) as u64; + + Self { + limit_low: limit as u16, + base_low: addr as u16, + base_mid: (addr >> 16) as u8, + // 0x89: Present (1), DPL (00), System (0), Type (1001 = 64-bit TSS Available) + type_flags: 0x89, + // The upper 4 bits of the limit + limit_high_flags: ((limit >> 16) & 0x0f) as u8, + base_high: (addr >> 24) as u8, + base_upper: (addr >> 32) as u32, + reserved: 0, + } + } + + #[inline] + pub const fn to_gdt_entries(self) -> [GdtEntry; 2] { + let [lower, upper] = unsafe { mem::transmute(self) }; + [GdtEntry(lower), GdtEntry(upper)] + } +} diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index 316c44b..00541ee 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -1,20 +1,140 @@ -#![feature(custom_inner_attributes)] -#![rustfmt::skip] - +#![forbid(unfulfilled_lint_expectations)] +#![feature( + anonymous_lifetime_in_impl_trait, + arbitrary_self_types_pointers, + const_trait_impl, + const_try, + debug_closure_helpers, + impl_trait_in_assoc_type, + layout_for_ptr, + negative_impls, + never_type, + ptr_alignment_type, + ptr_metadata, + slice_ptr_get, + sync_unsafe_cell +)] #![no_std] -pub mod page_alloc; -pub mod page_alloc_tree; +extern crate alloc; + +pub mod allocator; +pub mod arch; +pub mod device; +pub mod framebuffer; +pub mod interrupt; pub mod rendering; +pub mod serial; +pub mod spin_sync; +pub mod task; +pub mod timer; +pub mod vaddr; + +use core::{mem::MaybeUninit, ops::Range}; + +use allocator::{KernelPageAllocator, PhysicalPageAllocator}; +use elpytios_alloc::HeapAllocator; +use elpytios_bootinfo::paddr::PAddr; +use framebuffer::FrameBuffer; +use spin_sync::SpinMutex; +use vaddr::{VAddr, VirtualMap}; + +pub const LOWER_HALF_ADDRESSES: Range = VAddr::new(0x0000_0000_0000_1000)..VAddr::new(0x0000_8000_0000_0000); +pub const HIGHER_HALF_ADDRESSES: Range = VAddr::new(0xffff_8000_0000_0000)..VAddr::new(0xffff_ffff_ffff_ffff); + +#[repr(transparent)] +pub struct ScratchPages<'a> { + pub pages: &'a [PAddr], +} + +impl ScratchPages<'_> { + pub fn take(&mut self) -> Option { + loop { + match self.pages.split_at_checked(1) { + Some((&[next], pages)) => { + self.pages = pages; + if next.addr() == 0 { continue } else { break Some(next) } + } + _ => break None, + } + } + } +} + +/// # Safety +/// Every single one of these statics must be set by their corresponding `set_*` functions below in +/// the setup-phase of the kernel. +/// +/// See `main.rs`. +pub mod statics { + use super::*; + + static mut DIRECT_MAP_OFFSET: MaybeUninit = MaybeUninit::uninit(); + static mut VIRTUAL_MAP: MaybeUninit = MaybeUninit::uninit(); + static mut PHYS_ALLOC: MaybeUninit> = MaybeUninit::uninit(); + static mut FRAME_BUFFER: MaybeUninit = MaybeUninit::uninit(); + + #[global_allocator] + static ALLOC: HeapAllocator = HeapAllocator::new(KernelPageAllocator); + + #[inline] + pub unsafe fn set_direct_map_offset(offset: usize) { + unsafe { + DIRECT_MAP_OFFSET = MaybeUninit::new(offset); + } + } + + #[inline] + pub unsafe fn set_virtual_map(virtual_map: VirtualMap) { + unsafe { + VIRTUAL_MAP = MaybeUninit::new(virtual_map); + } + } + + #[inline] + pub unsafe fn set_phys_alloc(phys_alloc: PhysicalPageAllocator) { + unsafe { + PHYS_ALLOC = MaybeUninit::new(SpinMutex::new(phys_alloc)); + } + } + + #[inline] + pub unsafe fn set_frame_buffer(frame_buffer: FrameBuffer) { + unsafe { + FRAME_BUFFER = MaybeUninit::new(frame_buffer); + } + } + + #[inline] + pub fn phys_to_virt(p_addr: PAddr) -> VAddr { + VAddr::new( + p_addr + .addr() + .wrapping_add(unsafe { (&raw const DIRECT_MAP_OFFSET as *const usize).read() }), + ) + } -use core::mem::MaybeUninit; + #[inline] + pub fn virt_to_phys(v_addr: VAddr) -> PAddr { + PAddr::new( + v_addr + .addr() + .wrapping_sub(unsafe { (&raw const DIRECT_MAP_OFFSET as *const usize).read() }), + ) + } -use elpytios_bootinfo::BootInfo; + #[inline] + pub fn get_virtual_map() -> &'static VirtualMap { + unsafe { (&raw const VIRTUAL_MAP as *const VirtualMap).as_ref_unchecked() } + } -#[unsafe(link_section = ".bootinfo")] -#[used] -static mut BOOT_INFO: MaybeUninit = MaybeUninit::uninit(); + #[inline] + pub fn get_phys_alloc() -> &'static SpinMutex { + unsafe { (&raw const PHYS_ALLOC as *const SpinMutex).as_ref_unchecked() } + } -pub fn boot_info() -> &'static BootInfo { - unsafe { (&raw const BOOT_INFO as *const BootInfo).as_ref_unchecked() } -} \ No newline at end of file + #[inline] + pub fn get_frame_buffer() -> &'static FrameBuffer { + unsafe { (&raw const FRAME_BUFFER as *const FrameBuffer).as_ref_unchecked() } + } +} diff --git a/kernel/src/linker.ld b/kernel/src/linker.ld index 18ac51c..8a12af0 100644 --- a/kernel/src/linker.ld +++ b/kernel/src/linker.ld @@ -2,15 +2,16 @@ ENTRY(_start) PHDRS { - text PT_LOAD FLAGS(5); - rodata PT_LOAD FLAGS(4); - data PT_LOAD FLAGS(6); + text PT_LOAD FLAGS(5); + rodata PT_LOAD FLAGS(4); + data PT_LOAD FLAGS(6); + dynamic PT_DYNAMIC FLAGS(4); } SECTIONS { __kernel_start = .; - . = 0xffffffff80000000; + . = 0; .text : ALIGN(4K) { @@ -27,13 +28,6 @@ SECTIONS __rodata_end = .; } :rodata - .bootinfo (NOLOAD) : ALIGN(4K) - { - __bootinfo_start = .; - KEEP(*(.bootinfo)) - __bootinfo_end = .; - } :rodata - .data : ALIGN(4K) { __data_start = .; @@ -49,5 +43,10 @@ SECTIONS __bss_end = .; } :data + .dynamic : ALIGN(8) + { + *(.dynamic) + } :data :dynamic + __kernel_end = .; } \ No newline at end of file diff --git a/kernel/src/main.rs b/kernel/src/main.rs index ffac2e5..cd1207c 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -1,71 +1,514 @@ -#![feature(custom_inner_attributes)] -#![rustfmt::skip] - +#![forbid(unfulfilled_lint_expectations)] +#![feature(core_float_math, ptr_alignment_type)] #![no_std] #![no_main] -use core::{arch::naked_asm, fmt::Write, panic::PanicInfo}; +extern crate alloc; +use core::{ + arch::{asm, naked_asm}, + cell::RefCell, + fmt::Write, + iter::once, + panic::PanicInfo, + slice, +}; -use elpytios_bootinfo::PAGE_SIZE; -use elpytios_kernel::{boot_info, rendering::DisplayWriter}; +use elpytios_abi::ALLOC_ALIGNMENT; +use elpytios_bootinfo::{BootInfo, IdentityMapFlags, MemoryRegion, PAGE_SIZE, Reloc, paddr::PAddr}; +use elpytios_elf::sys::{ElfRela64, ElfRela64Type}; +use elpytios_kernel::{ + HIGHER_HALF_ADDRESSES, ScratchPages, + allocator::{AllocTree, PhysicalPageAllocator}, + device::{CpuContext, init_device_tree}, + framebuffer::FrameBuffer, + serial::{Com, Serial, serial_init}, + statics::{get_virtual_map, phys_to_virt, set_direct_map_offset, set_frame_buffer, set_phys_alloc, set_virtual_map}, + vaddr::{VAddr, VFlags, VirtualMapBuilder}, +}; +use log::{debug, error, info}; #[panic_handler] -fn hanic_pandler(_info: &PanicInfo) -> ! { +fn panic_handler(info: &PanicInfo) -> ! { + error!("{info}"); loop {} } -#[cfg(debug_assertions)] -#[unsafe(no_mangle)] -#[used] -static mut DEBUG_HALT: u8 = 1; - -#[cfg(debug_assertions)] -#[inline(never)] -fn pause() { - loop { - if unsafe { DEBUG_HALT } == 0 { - break +#[unsafe(naked)] +#[unsafe(export_name = "_start")] // Tell the linker that this is our entry point +unsafe extern "sysv64" fn jump_from_bootloader(info: &'static BootInfo) -> ! { + naked_asm!( + // Clear interrupt handlers, will be reinitialized by `setup_virtual_mapped()` + "cli", + "jmp {setup_identity_mapped}", + + setup_identity_mapped = sym setup_identity_mapped, + + options(att_syntax), + ) +} + +struct SerialLogger(Com); +impl log::Log for SerialLogger { + fn enabled(&self, _metadata: &log::Metadata) -> bool { + true + } + + fn log(&self, record: &log::Record) { + if self.enabled(record.metadata()) { + _ = match (record.file(), record.line()) { + (Some(file), Some(line)) => writeln!(Serial(self.0), "[{}]\t{}:{}\t> {}", record.level(), file, line, record.args()), + _ => writeln!(Serial(self.0), "[{}]\t{}\t> {}", record.level(), record.target(), record.args()), + } } + } + + fn flush(&self) {} +} - core::hint::spin_loop(); +static LOGGER: SerialLogger = SerialLogger(Com::Com3); + +struct MemoryRegions<'a> { + available: &'a [MemoryRegion], + head: MemoryRegion, +} + +impl<'a> MemoryRegions<'a> { + fn new(source: &'a [MemoryRegion]) -> Self { + let &[ref available @ .., head] = source else { panic!("Not enough memory to start the kernel") }; + Self { available, head } + } + + fn take_head(&mut self) -> Option { + let head = loop { + match self.head.pages { + 0 => { + let &[ref available @ .., head] = self.available else { return None }; + self.available = available; + self.head = head; + } + n => { + self.head.pages = n - 1; + break self.head.base.byte_add((n - 1) * PAGE_SIZE) + } + } + }; + Some(head) } } -#[unsafe(naked)] -#[unsafe(export_name = "_start")] -unsafe extern "sysv64" fn jump_from_bootloader() -> ! { - naked_asm!( - "jmp {main}", - main = sym main - ) +/// # Safety +/// - Available memory regions must *not* include the kernel code, stack, and boot info itself; +/// i.e., they must be usable immediately. +/// - Any references must point to the defined custom `MEM_*` memory types in the bootloader. +/// - See safety notes of [`setup_virtual_mapped`]. +unsafe extern "sysv64" fn setup_identity_mapped(info: &'static BootInfo) -> ! { + // Notes: + // - `log` mustn't be setup here; wait until symbols are relocated + + let regions = RefCell::new(MemoryRegions::new(&info.memory_regions)); + let mut scratch_pages = ScratchPages { pages: &info.scratch_pages }; + + let kernel_base = info + .identity_maps + .iter() + .min_by_key(|map| map.region.base) + .expect("Didn't find any identity maps") + .region + .base; + let v_slide = HIGHER_HALF_ADDRESSES + .start + .addr() + .checked_sub(kernel_base.addr()) + .expect("Kernel physical address somehow higher than higher-half addressing base"); + + let page_table_phys = scratch_pages.take().expect("Not enough scratch pages for page table"); + let setup_virtual_mapped = { + let mut v_map = unsafe { VirtualMapBuilder::new(page_table_phys, || regions.borrow_mut().take_head(), |p_addr| p_addr.addr() as *mut ()) }; + + let mut direct_map_offset = usize::MIN; + for map in &info.identity_maps { + let mut flags = VFlags::GLOBAL; + if map.flags.contains(IdentityMapFlags::WRITABLE) { + flags |= VFlags::WRITABLE; + } + if !map.flags.contains(IdentityMapFlags::EXECUTABLE) { + flags |= VFlags::EXECUTE_DISABLE; + } + + v_map + .map(map.region.base, VAddr::new(map.region.base.addr()), map.region.pages, flags) + .expect("Couldn't identity map kernel segment"); + + v_map + .map(map.region.base, VAddr::new(map.region.base.addr() + v_slide), map.region.pages, flags) + .expect("Couldn't virtual map kernel segment"); + + direct_map_offset = direct_map_offset.max(map.region.base.addr() + v_slide + map.region.pages * PAGE_SIZE); + } + + // Direct map *all* of RAM to the specified direct-map offset + let direct_map_offset = direct_map_offset.next_multiple_of(2 << 30); // Align to a gigabyte + unsafe { set_direct_map_offset(direct_map_offset) } + + v_map + .map( + page_table_phys, + VAddr::new(page_table_phys.addr() + direct_map_offset), + 1, + VFlags::GLOBAL | VFlags::WRITABLE | VFlags::EXECUTE_DISABLE, + ) + .unwrap(); + + for region in &info.memory_regions { + v_map + .map( + region.base, + VAddr::new(region.base.addr() + direct_map_offset), + region.pages, + VFlags::GLOBAL | VFlags::WRITABLE | VFlags::EXECUTE_DISABLE, + ) + .unwrap(); + } + + let virtual_map = unsafe { v_map.finish() }; + let setup_virtual_mapped = (setup_virtual_mapped as *const ()) + .addr() + .checked_add(v_slide) + .expect("`setup_virtual_mapped()` virtual address overflowed"); + + unsafe { set_virtual_map(virtual_map) } + setup_virtual_mapped + }; + + unsafe { + let tmp = 0usize; + asm!( + // Enable `GLOBAL` mapping, i.e. pages in TLB that don't get flushed + "movq %cr4, {tmp}", + "orq $(1 << 7), {tmp}", + "movq {tmp}, %cr4", + + "movq {page_table_phys}, %cr3", + "addq {v_slide}, %rsp", + "andq $-16, %rsp", + "jmpq *{setup_virtual_mapped}", + + tmp = in(reg) tmp, + page_table_phys = in(reg) page_table_phys.addr(), + v_slide = in(reg) v_slide, + setup_virtual_mapped = in(reg) setup_virtual_mapped, + in("rdi") (info as *const BootInfo).byte_add(v_slide), + in("rsi") ®ions.into_inner(), + in("rdx") scratch_pages.pages.as_ptr(), + in("rcx") scratch_pages.pages.len(), + in("r8") kernel_base.addr(), + + options(att_syntax, noreturn), + ) + } } -unsafe extern "sysv64" fn main() -> ! { +unsafe extern "sysv64" fn setup_virtual_mapped( + info: &'static BootInfo, + regions: &MemoryRegions, + scratch_pages_ptr: *const PAddr, + scratch_pages_len: usize, + kernel_base: PAddr, +) -> ! { + let mut scratch_pages = ScratchPages { + pages: unsafe { slice::from_raw_parts(scratch_pages_ptr, scratch_pages_len) }, + }; + + // Relocate all symbols to higher-half addressing + // Identity-mapping is still present at this point, so it is okay to cast `PAddr` into pointers + let kernel_ptr = info.kernel_elf_base; + let v_slide = HIGHER_HALF_ADDRESSES + .start + .addr() + .checked_sub(kernel_base.addr()) + .expect("Kernel physical address somehow higher than higher-half addressing base") + .cast_signed() as i64; + + for &Reloc { offset, size, stride } in &info.relocations { + for i in 0..size / stride { + unsafe { + let rela = (kernel_ptr.addr() as *mut u8) + .cast::() + .byte_add(offset - info.kernel_virt_base) + .add(i) + .read_unaligned(); + + match rela.info.kind { + ElfRela64Type::X86_64_RELATIVE => { + let slide = v_slide + kernel_ptr.addr() as i64 - info.kernel_virt_base as i64; + let patch_addr = (kernel_ptr.addr() as *mut u8).add(rela.offset as usize - info.kernel_virt_base); + let value = slide + rela.addend; + patch_addr.cast::().write(value); + } + kind => panic!("Unsupported Elf64_Rela kind: {}", kind.0), + } + } + } + } + + // Setup `log` here, symbols have been relocated + unsafe { + serial_init(Com::Com3); + log::set_logger_racy(&LOGGER).expect("Log already setup before symbol relocations"); + log::set_max_level_racy(cfg_select! { + debug_assertions => log::LevelFilter::Trace, + not(debug_assertions) => log::LevelFilter::Info, + }); + + debug!( + "Setting up kernel at {kernel_ptr:p} -> {:p}", + VAddr::new(kernel_ptr.addr().wrapping_add_signed(v_slide as isize)) + ); + } + + // When running through `x qemu run --debug`, wait until a corresponding GDB client executes this: + // + // target remote [host, usuallty `localhost`]:[port, usually `1234`] + // add-symbol-file [path/to]/elpytios-kernel -o [offset; see "Setting up kernel at ..." log] + // + // set language c + // set *(unsigned char*)&__DEBUG_HALT = 0 + // set language rust + // continue + // + // This is to ensure the kernel has been loaded to memory at higher-half address before inserting + // software breakpoints and looking up symbols at the same offset #[cfg(debug_assertions)] - pause(); + { + #[unsafe(no_mangle)] + #[used] + static mut __DEBUG_HALT: u8 = 1; - let mut display_writer = DisplayWriter { - graphics_info: &boot_info().graphics_info, - line: 0, - col: 0, - }; + debug!("Waiting for debugger..."); + while unsafe { (&raw const __DEBUG_HALT).read_volatile() } != 0 { + core::hint::spin_loop(); + } - writeln!(&mut display_writer, "Hello World from the Kernel, calling at address {:p}!!!!", main as *const ()).unwrap(); - - let regions = boot_info().memory_regions(); - writeln!(&mut display_writer, "Found {} usable physical memory regions!", regions.len()).unwrap(); - for region in regions { - writeln!( - &mut display_writer, - "Usable physical memory in {}..{}, {} pages!", - region.base, - region.base.byte_add(region.pages * PAGE_SIZE), - region.pages, - ).unwrap(); + debug!("Continuing!"); } - let [virt_start, virt_end] = boot_info().v_addr_range(); - writeln!(&mut display_writer, "Higher-half virtual addressing available in range {virt_start}..{virt_end}").unwrap(); + // Setup global physical page allocator + { + info!( + "Initializing physical page allocator: found {} usable memory regions", + info.memory_regions.len() + ); + + let mut phys_alloc = unsafe { PhysicalPageAllocator::new() }; + for MemoryRegion { base, pages } in regions.available.iter().copied().chain(once(regions.head)) { + debug!( + "\tAvailable memory region found at [{base:p}..{:p}], {pages} pages", + base.byte_add(pages * PAGE_SIZE) + ); + + let mut usable_start = base.addr(); + let mut usable_end = usable_start + pages * PAGE_SIZE; + + // Trees need to be aligned to `PHYS_ALLOC_ALIGNMENT`, so round down and manually fill "ghost" pages + let mut tree_start = usable_start & !(ALLOC_ALIGNMENT.as_usize() - 1); + let mut tree_end = usable_end & !(ALLOC_ALIGNMENT.as_usize() - 1); + + while tree_start < tree_end { + let layout = AllocTree::layout((tree_end - tree_start) / PAGE_SIZE).expect("`AllocTree` layout error"); + let meta_pages = layout.size().div_ceil(PAGE_SIZE); + let tree_end_actual = tree_start + layout.node_count() * PAGE_SIZE; + + let (tree, ghost_pages) = if (usable_end - tree_end_actual) / PAGE_SIZE >= meta_pages { + // Store tree metadata to the right, outside the tree + usable_end -= meta_pages * PAGE_SIZE; + unsafe { + ( + AllocTree::new(phys_to_virt(PAddr::new(usable_end)).ptr_mut(), layout), + usable_start + .checked_sub(tree_start) + .and_then(|ghost_size| (ghost_size != 0).then_some(ghost_size / PAGE_SIZE)), + ) + } + } else if (tree_end_actual - usable_start) / PAGE_SIZE >= meta_pages { + // Store tree metadata to the left, adding along ghost pages + unsafe { + ( + AllocTree::new(phys_to_virt(PAddr::new(usable_start)).ptr_mut(), layout), + Some((usable_start + meta_pages * PAGE_SIZE - tree_start) / PAGE_SIZE), + ) + } + } else { + tree_end -= (tree_end - tree_start) / 2; + continue + }; + + debug!( + "\t\tBuilding tree at [{tree_start:#018x}..{:#018x}], {} pages", + tree_start + layout.node_count() * PAGE_SIZE, + layout.node_count(), + ); + + if let Some(mut ghost_pages) = ghost_pages { + debug!("\t\t\tReserving {ghost_pages} ghost pages"); + + #[cfg(debug_assertions)] + let [mut prev, mut prev_len] = [0, 0]; + + while ghost_pages != 0 { + let order = ghost_pages.ilog2(); + ghost_pages -= 1 << order; + + #[cfg_attr(not(debug_assertions), expect(unused))] + let index = unsafe { (*tree).alloc(order).expect("Couldn't reserve ghost pages") }; + + #[cfg(debug_assertions)] + if index == prev + prev_len { + prev = index; + prev_len = 1 << order; + } else { + panic!("Ghost page reservation wasn't contiguous"); + } + } + } + + unsafe { phys_alloc.push_tree(PAddr::new(tree_start), tree) } + tree_start += layout.node_count() * PAGE_SIZE; + usable_start = tree_start; + tree_end = usable_end & !(ALLOC_ALIGNMENT.as_usize() - 1); + } + } + + phys_alloc.sort_tree(); + match phys_alloc.tree_count() { + 0 => panic!("Couldn't initialize physical page allocator, found no usable memory regions"), + n => { + info!("Initialized physical page allocator with {n} trees"); + unsafe { set_phys_alloc(phys_alloc) } + } + } + } + + // Virtual-map the framebuffer + { + let v_map = get_virtual_map(); + let fb_phys = info.graphics_info.frame_buffer.addr(); + let fb_size = info.graphics_info.frame_buffer_size; + + let fb_phys_base = fb_phys & !(PAGE_SIZE - 1); + let fb_phys_end = (fb_phys + fb_size).next_multiple_of(PAGE_SIZE); + let fb_page_count = (fb_phys_end - fb_phys_base) / PAGE_SIZE; + + unsafe { + let p_addr = PAddr::new(fb_phys_base); + v_map + .map( + p_addr, + phys_to_virt(p_addr), + fb_page_count, + VFlags::GLOBAL | VFlags::WRITABLE | VFlags::WRITE_THROUGH | VFlags::EXECUTE_DISABLE, + ) + .unwrap(); + } + + unsafe { + set_frame_buffer(FrameBuffer { + width: info.graphics_info.w, + height: info.graphics_info.h, + stride: info.graphics_info.stride, + format: info.graphics_info.pixel_format, + pointer: phys_to_virt(PAddr::new(fb_phys)).ptr_mut(), + }) + } + } + + // Setup device tree, which includes waking up all AP and setting up interrupt handlers + // This calls the closure once for every CPU cores locally + unsafe { init_device_tree(info.device_tree, &mut scratch_pages, main) } +} + +/// # Safety +/// - All [`statics`](elpytios_kernel::statics) must have been initialized prior to calling this +/// function. +/// - This function must be able to be run in parallel with itself on other threads +fn main(core_count: u32) -> ! { + let ctx = CpuContext::get(); + if ctx.is_bootstrap { + info!("Hello, world! Kernel is now up and running on {core_count} logical processors!"); + } + + { + use elpytios_bootinfo::PixelFormat; + use elpytios_kernel::statics::get_frame_buffer; + + let fbo = get_frame_buffer(); + let fbo_div = fbo.height.div_ceil(core_count as usize); + let cpu_id = ctx.cpu_id as usize; + + match fbo.format { + fmt @ (PixelFormat::RGB_8_BIT | PixelFormat::BGR_8_BIT) => { + let invert_br = matches!(fmt, PixelFormat::BGR_8_BIT); + for y in (cpu_id * fbo_div)..((cpu_id + 1) * fbo_div).min(fbo.height) { + for x in 0..fbo.width { + let fx = x as f32 / (fbo.width - 1) as f32; + let fy = y as f32 / (fbo.height - 1) as f32; + + let r = (fx * 255.) as u8; + let g = (fy * 255.) as u8; + let b = (core::f32::math::sqrt((fx * 2. - 1.).abs() * (fy * 2. - 1.).abs()) * 255.) as u8; + let a = 255; + + unsafe { + fbo.pointer.cast::<[u8; 4]>().add(y * fbo.stride + x).write_volatile(match invert_br { + false => [r, g, b, a], + true => [b, g, r, a], + }); + } + } + } + } + _ => {} + } + } + + if ctx.is_bootstrap { + use core::time::Duration; + + use elpytios_elf::Elf; + use elpytios_kernel::{ + interrupt::x86_64::InterruptStack, + task::{Process, TASK_QUEUE, schedule}, + }; + + { + let shell = include_bytes!("../../target/x86_64-unknown-elpytios/release/elpytios-shell"); + let Elf::N64(elf) = Elf::from_bytes(shell).unwrap(); + + let (process, task) = Process::from_elf(elf).unwrap(); + core::mem::forget(process); + TASK_QUEUE.push_back(task); + } + + const SCHED_TIMER: Duration = Duration::from_millis(10); + unsafe { + asm!( + "movq %rsp, ({stack})", + "jmp {schedule}", + + stack = in(reg) ctx.tss.stack_ptr(InterruptStack::DoubleFault), + schedule = label { + ctx.timer_callback.set(Some(|| { + CpuContext::get().timer.schedule(SCHED_TIMER); + schedule(); + })); + + ctx.timer.schedule(SCHED_TIMER); + schedule(); + }, + + options(att_syntax, noreturn) + ) + } + } loop {} -} \ No newline at end of file +} diff --git a/kernel/src/page_alloc.rs b/kernel/src/page_alloc.rs deleted file mode 100644 index 0554168..0000000 --- a/kernel/src/page_alloc.rs +++ /dev/null @@ -1,171 +0,0 @@ -/*use core::{fmt::{Display, Formatter}, mem::offset_of, ptr::null_mut}; - -use uefi::{boot::MemoryType, mem::memory_map::{MemoryMap, MemoryMapOwned}}; - -use crate::page_alloc_tree::{BinaryBuddyTree, PAGE_SIZE}; - -const MINIMUM_PAGES_TO_MANAGE: usize = 4; -const PAGES_RESERVED_FOR_TREE_POINTERS: usize = 2; // TODO Fix, This is STUPID - -#[repr(C)] -pub struct PhysicalPageAllocator { - size: usize, - trees: [*mut BinaryBuddyTree; 0] -} - -impl PhysicalPageAllocator { - unsafe fn push_tree(&mut self, tree: *mut BinaryBuddyTree) { - unsafe { - (self as *mut Self) - .byte_add(offset_of!(Self, trees)) - .cast::<*mut BinaryBuddyTree>() - .add(self.size) - .write_volatile(tree); - } - - self.size += 1; - } - - unsafe fn get_tree(&self, n: usize) -> *mut BinaryBuddyTree { - unsafe { - (self as *const Self) - .byte_add(offset_of!(Self, trees)) - .cast::<*mut BinaryBuddyTree>() - .add(n) - .cast_mut() - .read_volatile() - } - } - - unsafe fn create_trees_(&mut self, start: usize, pages: usize, loader_memory: bool) -> Result<(), ()> { - if pages < MINIMUM_PAGES_TO_MANAGE { - return Err(()); - } - - let mut region_size = 1 << (pages.ilog2() as usize); - let free_pages = pages - region_size; - - let mut required_pages_for_tree = BinaryBuddyTree::required_pages_for_tree(region_size); - - while required_pages_for_tree > free_pages { - region_size >>= 1; - - if region_size < MINIMUM_PAGES_TO_MANAGE { - return Err(()); - } - - required_pages_for_tree = BinaryBuddyTree::required_pages_for_tree(region_size); - } - - let memory_region_start = start + required_pages_for_tree * PAGE_SIZE; - - unsafe { - if let Some(tree) = BinaryBuddyTree::new( - start as *mut BinaryBuddyTree, - memory_region_start as *mut u8, - region_size, - loader_memory - ) { - self.push_tree(tree); - - if let Ok(_) = self.create_trees_( - memory_region_start + region_size * PAGE_SIZE, - pages - region_size - required_pages_for_tree, - loader_memory - ) { - return Ok(()); - } else { - return Err(()); - } - } else { - return Err(()); - } - } - } - - pub fn new(memory_map: &MemoryMapOwned) -> Result<&mut Self, ()> { - let mut allocator: *mut Self = null_mut(); - - for i in memory_map.entries() { - let valid; - let loader_memory; - match i.ty { - MemoryType::CONVENTIONAL | - MemoryType::PERSISTENT_MEMORY => { - valid = true; - loader_memory = false; - }, - MemoryType::BOOT_SERVICES_CODE | - MemoryType::BOOT_SERVICES_DATA | - MemoryType::LOADER_CODE | - MemoryType::LOADER_DATA => { - valid = false; - loader_memory = true; - } - _ => { - valid = false; - loader_memory = false; - } - } - - if valid { - let mut start = i.phys_start as usize; - let mut size = i.page_count as usize; - - if allocator.is_null() && start.is_multiple_of(align_of::()) && - size == PAGES_RESERVED_FOR_TREE_POINTERS { - allocator = start as *mut Self; - - unsafe { - allocator.write_volatile(PhysicalPageAllocator { size: 0, trees: [] }); - } - - start = start + PAGE_SIZE * PAGES_RESERVED_FOR_TREE_POINTERS; - size -= PAGES_RESERVED_FOR_TREE_POINTERS; - } - - if !allocator.is_null() && size >= MINIMUM_PAGES_TO_MANAGE { - unsafe { - let _ = (&mut*allocator).create_trees_(start, size, loader_memory); - } - } - } - } - - unsafe { - if allocator.is_null() { - Err(()) - } else { - Ok(&mut*allocator) - } - } - } - - pub unsafe fn alloc(&mut self, pages: usize) -> Option<*mut u8> { - for i in 0..self.size { - unsafe { - if let Some(address) = (*self.get_tree(i)).alloc(pages) { - return Some(address); - } - } - } - - return None; - } - - pub unsafe fn free(&mut self, _region: *mut u8) -> Result<(), ()> { - Ok(()) - } -} - -impl Display for PhysicalPageAllocator { - fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { - for i in 0..self.size { - unsafe { - writeln!(f, "{:?}", self.get_tree(i))?; - } - } - - Ok(()) - } -}*/ diff --git a/kernel/src/page_alloc_tree.rs b/kernel/src/page_alloc_tree.rs deleted file mode 100644 index 8e70741..0000000 --- a/kernel/src/page_alloc_tree.rs +++ /dev/null @@ -1,280 +0,0 @@ -/*use core::mem::offset_of; - -#[repr(C)] -#[derive(Debug)] -pub struct BinaryBuddyTree { - base: *mut u8, - pages: usize, - pub loader_memory: bool, - children: [BinaryBuddyTreeNode; 0] -} - -const HAS_CHILDREN_FLAG: u8 = 0b0000_0001; -const IS_OCCUPIED_FLAG: u8 = 0b0000_0010; -pub const PAGE_SIZE: usize = 4096; - -#[derive(Debug, Clone, Copy)] -struct BinaryBuddyTreeNode { - flags: u8 -} - -const _: () = assert!( - offset_of!(BinaryBuddyTree, children).is_multiple_of(align_of::()) -); - -impl BinaryBuddyTree { - fn get_child_start_and_pages(&self, node: usize) -> (usize, usize) { - let level = (node + 1).ilog2() as usize; - - let first_in_level = (1 << level) - 1; - let position_in_level = node - first_in_level; - - let pages = self.pages >> level; - let start = position_in_level * pages; - - debug_assert!(pages.is_power_of_two()); - - (start, pages) - } - - unsafe fn write_child(&mut self, offset: usize, node: &BinaryBuddyTreeNode) { - unsafe { - (self as *mut BinaryBuddyTree) - .byte_add(offset_of!(Self, children)) - .cast::() - .add(offset) - .write_volatile(*node); - } - } - - unsafe fn get_child(&self, offset: usize) -> *mut BinaryBuddyTreeNode { - unsafe { - (self as *const BinaryBuddyTree) - .byte_add(offset_of!(Self, children)) - .cast::() - .cast_mut() - .add(offset) - } - } - - /// Get the amount of necessary pages to store a tree that keeps track of `pages` pages. - /// - /// * `pages`: The amount of pages the tree will keep track of. - pub fn required_pages_for_tree(pages: usize) -> usize { - let required_size_bytes = size_of::() + (pages * 2 - 1) * size_of::(); - - return (required_size_bytes + (PAGE_SIZE - 1)) / PAGE_SIZE; - } - - /// Create a new tree. - /// - /// * `tree_base`: Where to store the tree. - /// * `memory_base`: Where the memory the tree keeps track of starts at. - /// * `pages`: The amount of pages to keep track of. - /// * `loader_memory`: Whether or not this tree is keeping memory in the UEFI LoaderCode or - /// LoaderData regions. - pub unsafe fn new(tree_base: *mut BinaryBuddyTree, memory_base: *mut u8, pages: usize, loader_memory: bool) -> Option<&'static mut Self> { - unsafe { - if pages == 0 || !pages.is_power_of_two() { - return None; - } - - if !(tree_base as usize).is_multiple_of(align_of::()) { - return None; - } - - tree_base.write_volatile(BinaryBuddyTree { - base: memory_base, - pages: pages, - loader_memory: loader_memory, - children: [] - }); - - (&mut *tree_base).write_child(0, &BinaryBuddyTreeNode { flags: 0 }); - - Some(&mut *tree_base) - } - } - - unsafe fn find_fitting_leaf_(&mut self, pages: usize, node: usize, min_pages: usize) -> Option<(usize, usize)> { - unsafe { - let child = self.get_child(node); - - if (*child).flags & HAS_CHILDREN_FLAG == 0 { - let (_, child_pages) = self.get_child_start_and_pages(node); - - if (*child).flags & IS_OCCUPIED_FLAG == 0 && - child_pages >= pages && child_pages < min_pages { - return Some((node, child_pages)); - } else { - return None; - } - } else { - let result_left = self.find_fitting_leaf_(pages, node * 2 + 1, min_pages); - - let mut found_node = None; - let mut size_to_beat = min_pages; - if let Some((node_left, min_pages_left)) = result_left { - size_to_beat = min_pages_left; - found_node = Some(node_left); - } - - let result_right = self.find_fitting_leaf_(pages, node * 2 + 2, size_to_beat); - - if let Some((node_right, min_pages_right)) = result_right { - size_to_beat = min_pages_right; - found_node = Some(node_right); - } - - if let Some(found_node) = found_node { - return Some((found_node, size_to_beat)); - } else { - return None; - } - } - } - } - - unsafe fn find_fitting_leaf(&mut self, pages: usize) -> Option { - unsafe { - if let Some((node, _)) = self.find_fitting_leaf_(pages, 0, usize::MAX) { - return Some(node); - } - } - - return None; - } - - unsafe fn split_node(&mut self, node: usize) -> (usize, usize) { - unsafe { - let base_child = self.get_child(node); - (*base_child).flags |= HAS_CHILDREN_FLAG; - - self.write_child(node * 2 + 1, &BinaryBuddyTreeNode { flags: 0 }); - self.write_child(node * 2 + 2, &BinaryBuddyTreeNode { flags: 0 }); - - (node * 2 + 1, node * 2 + 2) - } - } - - unsafe fn find_node_(&mut self, start: usize, node: usize) -> Option { - unsafe { - let child = self.get_child(node); - - let (child_start, child_pages) = self.get_child_start_and_pages(node); - - if start >= child_start && start < child_start + child_pages { - if (*child).flags & HAS_CHILDREN_FLAG != 0 { - if start < child_start + child_pages / 2 { - return self.find_node_(start, node * 2 + 1); - } else { - return self.find_node_(start, node * 2 + 2); - } - } else { - if child_start == start { - return Some(node); - } else { - return None; - } - } - } - - return None; - } - } - - unsafe fn find_node(&mut self, start: usize) -> Option { - unsafe { - return self.find_node_(start, 0); - } - } - - unsafe fn try_merge(&mut self, node: usize) { - if node == 0 { - return; - } - - unsafe { - let child = self.get_child(node); - if (*child).flags & (HAS_CHILDREN_FLAG | IS_OCCUPIED_FLAG) != 0 { - return; - } - - let parent_node = (node - 1) / 2; - - let sibling; - if node % 2 == 1 { // Left child - sibling = self.get_child(node + 1); - } else { - sibling = self.get_child(node - 1); - } - - if (*sibling).flags & (HAS_CHILDREN_FLAG | IS_OCCUPIED_FLAG) == 0 { - (*self.get_child(parent_node)).flags &= !HAS_CHILDREN_FLAG; - - self.try_merge(parent_node); - } - } - } - - /// Try to allocate `pages` consecutive pages. - /// - /// * `pages`: The amount of consecutive pages to allocate. - pub unsafe fn alloc(&mut self, pages: usize) -> Option<*mut u8> { - if pages == 0 { - return None; - } - - unsafe { - if let Some(node) = self.find_fitting_leaf(pages) { - let mut child_node = node; - let mut child = self.get_child(node); - - let (mut child_start, mut child_pages) = self.get_child_start_and_pages(node); - - while child_pages / 2 >= pages { - let (next_child, _) = self.split_node(child_node); - - child_node = next_child; - child = self.get_child(child_node); - - (child_start, child_pages) = self.get_child_start_and_pages(child_node); - } - - (*child).flags |= IS_OCCUPIED_FLAG; - - return Some((child_start * PAGE_SIZE + self.base as usize) as *mut u8); - } - } - - return None; - } - - /// Try to free a region beginning at `page`. - /// - /// * `page`: The position of the region to free. - pub unsafe fn free(&mut self, page: *mut u8) -> Result<(), ()> { - debug_assert_eq!(page as usize % PAGE_SIZE, 0); - - let start = (page as usize - self.base as usize) / PAGE_SIZE; - unsafe { - if (page as usize) < (self.base as usize) || (page as usize) >= (self.base as usize) + self.pages * PAGE_SIZE { - return Err(()); - } - - if let Some(node) = self.find_node(start) { - let child = self.get_child(node); - - if (*child).flags & IS_OCCUPIED_FLAG != 0 { - (*child).flags &= !IS_OCCUPIED_FLAG; - - self.try_merge(node); - - return Ok(()); - } - } - } - - Err(()) - } -}*/ diff --git a/kernel/src/rendering.rs b/kernel/src/rendering.rs index fa031c9..ca6e63b 100644 --- a/kernel/src/rendering.rs +++ b/kernel/src/rendering.rs @@ -1,4 +1,4 @@ -#![rustfmt::skip] +/*#![rustfmt::skip] use core::fmt::Write; @@ -145,7 +145,7 @@ impl<'a> DisplayWriter<'a> { if self.col >= (self.graphics_info.w / (FONT_WIDTH + HORIZONTAL_SPACING)) { self.col = 0; - + self.advance_line(); } } @@ -195,3 +195,4 @@ impl<'a> Write for DisplayWriter<'a> { Ok(()) } } +*/ diff --git a/kernel/src/serial/mod.rs b/kernel/src/serial/mod.rs new file mode 100644 index 0000000..2fdb15d --- /dev/null +++ b/kernel/src/serial/mod.rs @@ -0,0 +1,9 @@ +cfg_select! { + target_arch = "x86_64" => { + mod uart; + pub use uart::*; + } + _ => { + compile_error!("Unsupported architecture"); + } +} diff --git a/kernel/src/serial/uart.rs b/kernel/src/serial/uart.rs new file mode 100644 index 0000000..db3c60d --- /dev/null +++ b/kernel/src/serial/uart.rs @@ -0,0 +1,69 @@ +use core::{fmt, hint::spin_loop}; + +use crate::arch::x86_64::{inb, outb}; + +#[derive(Debug, Clone, Copy)] +#[repr(u16)] +pub enum Com { + Com1 = 0x3f8, + Com2 = 0x2f8, + Com3 = 0x3e8, + Com4 = 0x2e8, +} + +pub unsafe fn serial_init(port: Com) { + let port = port as u16; + unsafe { + // Disable interrupts + outb(port + 1, 0x00); + + // Enable DLAB + outb(port + 3, 0x80); + + // Baud divisor = 3 (38400 baud assuming 115200 clock) + outb(port + 0, 0x03); + outb(port + 1, 0x00); + + // 8 bits, no parity, one stop bit + outb(port + 3, 0x03); + + // Enable FIFO, clear them, 14-byte threshold + outb(port + 2, 0xC7); + + // IRQs disabled, RTS/DSR set + outb(port + 4, 0x03); + } +} + +#[inline] +fn tx_ready(port: Com) -> bool { + unsafe { inb(port as u16 + 5) & 0x20 != 0 } +} + +pub fn serial_write_byte(port: Com, byte: u8) { + while !tx_ready(port) { + spin_loop(); + } + + unsafe { + outb(port as u16, byte); + } +} + +pub fn serial_write(port: Com, s: &str) { + for b in s.bytes() { + if b == b'\n' { + serial_write_byte(port, b'\r'); + } + + serial_write_byte(port, b); + } +} + +pub struct Serial(pub Com); +impl fmt::Write for Serial { + fn write_str(&mut self, s: &str) -> fmt::Result { + serial_write(self.0, s); + Ok(()) + } +} diff --git a/kernel/src/spin_sync/mod.rs b/kernel/src/spin_sync/mod.rs new file mode 100644 index 0000000..0dbc85a --- /dev/null +++ b/kernel/src/spin_sync/mod.rs @@ -0,0 +1,4 @@ +mod mutex; +mod once; +pub use mutex::*; +pub use once::*; diff --git a/kernel/src/spin_sync/mutex.rs b/kernel/src/spin_sync/mutex.rs new file mode 100644 index 0000000..bf91851 --- /dev/null +++ b/kernel/src/spin_sync/mutex.rs @@ -0,0 +1,80 @@ +use core::{ + cell::SyncUnsafeCell, + hint::spin_loop, + ops::{Deref, DerefMut}, + sync::atomic::{AtomicBool, Ordering::*}, +}; + +#[derive(Debug)] +pub struct SpinMutex { + locked: AtomicBool, + value: SyncUnsafeCell, +} + +impl SpinMutex { + #[inline] + pub const fn new(value: T) -> Self + where T: Sized { + Self { + locked: AtomicBool::new(false), + value: SyncUnsafeCell::new(value), + } + } + + #[inline] + pub fn lock(&self) -> SpinMutexGuard<'_, T> { + loop { + match self.locked.compare_exchange_weak(false, true, Acquire, Relaxed) { + Ok(..) => { + break SpinMutexGuard { + locked: &self.locked, + value: unsafe { self.value.get().as_mut_unchecked() }, + } + } + Err(..) => spin_loop(), + } + } + } + + #[inline] + pub fn get_mut(&mut self) -> &mut T { + self.value.get_mut() + } +} + +#[derive(Debug)] +pub struct SpinMutexGuard<'a, T: ?Sized> { + locked: &'a AtomicBool, + value: &'a mut T, +} + +impl Deref for SpinMutexGuard<'_, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + self.value + } +} + +impl DerefMut for SpinMutexGuard<'_, T> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + self.value + } +} + +impl Drop for SpinMutexGuard<'_, T> { + fn drop(&mut self) { + cfg_select! { + debug_assertions => { + if !self.locked.swap(false, Release) { + unreachable!("Spin-mutex incorrectly unlocked!") + } + } + not(debug_assertions) => { + self.locked.store(false, Release); + } + } + } +} diff --git a/kernel/src/spin_sync/once.rs b/kernel/src/spin_sync/once.rs new file mode 100644 index 0000000..b5878fb --- /dev/null +++ b/kernel/src/spin_sync/once.rs @@ -0,0 +1,41 @@ +use core::{ + hint::{cold_path, spin_loop, unreachable_unchecked}, + sync::atomic::{ + AtomicU8, + Ordering::{Acquire, Release}, + }, +}; + +#[derive(Debug)] +pub struct SpinOnce(AtomicU8); +impl SpinOnce { + const UNINIT: u8 = 0; + const LOCKED: u8 = 1; + const INIT: u8 = 2; + + #[inline] + pub const fn new() -> Self { + Self(AtomicU8::new(Self::UNINIT)) + } + + #[inline] + pub fn call_once(&self, f: impl FnOnce() -> T) -> Option { + loop { + match self.0.compare_exchange_weak(Self::UNINIT, Self::LOCKED, Acquire, Acquire) { + Ok(..) => { + cold_path(); + let result = f(); + + self.0.store(Self::INIT, Release); + break Some(result) + } + Err(Self::LOCKED) => { + cold_path(); + spin_loop(); + } + Err(Self::INIT) => break None, + Err(..) => unsafe { unreachable_unchecked() }, + } + } + } +} diff --git a/kernel/src/task/imp/x86_64.rs b/kernel/src/task/imp/x86_64.rs new file mode 100644 index 0000000..5615e5e --- /dev/null +++ b/kernel/src/task/imp/x86_64.rs @@ -0,0 +1,133 @@ +use alloc::{ + alloc::{alloc_zeroed, handle_alloc_error}, + boxed::Box, +}; +use core::{ + arch::{asm, naked_asm}, + cell::UnsafeCell, + ptr, +}; + +use crate::{ + arch::x86_64::{ExtendedRegisterMask, ExtendedRegisters}, + device::CpuContext, + interrupt::x86_64::InterruptFrame, + task::TASK_QUEUE, + vaddr::VAddr, +}; + +pub struct Task { + curr_kernel_stack: *mut u8, + curr_schedule_inst: *const u8, + register_mask: ExtendedRegisterMask, + registers: Box>, +} + +impl Task { + pub unsafe fn new(entry: VAddr, _user_stack_top: *mut u8, user_stack_lower_top: VAddr, kernel_stack_top: *mut u8) -> Self { + let cpu = CpuContext::get(); + unsafe { + let registers = alloc_zeroed(cpu.registers.layout()); + if registers.is_null() { + handle_alloc_error(cpu.registers.layout()) + } + let registers = Box::from_raw(ptr::from_raw_parts_mut(registers, cpu.registers.layout().size())); + let register_mask = cpu.registers.mask(); + + let frame = kernel_stack_top.cast::().sub(1); + frame.write(InterruptFrame { + rax: 0, + rcx: 0, + rdx: 0, + rsi: 0, + rdi: 0, + r8: 0, + r9: 0, + r10: 0, + r11: 0, + rbx: 0, + rbp: 0, + r12: 0, + r13: 0, + r14: 0, + r15: 0, + error: (), + rip: entry.addr() as u64, + cs: 0x20 + 3, + rflags: 0x202, + rsp: user_stack_lower_top.addr() as u64, + ss: 0x18 + 3, + }); + + Self { + curr_kernel_stack: (&raw mut (*frame).rax).cast(), + curr_schedule_inst: Self::init as *const u8, + register_mask, + registers, + } + } + } + + /// # Safety + /// - Call with a `jmp` instruction. + /// - `%rsp` must point to [`InterruptFrame::rax`]. + #[unsafe(naked)] + unsafe extern "sysv64" fn init() -> ! { + naked_asm!( + "callq {load}", + "iretq", + + load = sym InterruptFrame::<()>::load, + + options(att_syntax) + ) + } +} + +pub fn schedule() { + let cpu = CpuContext::get(); + unsafe { + match TASK_QUEUE.pop_front() { + Some(next) => { + let next_v_map = next.virtual_map_phys; + + let next_kernel_stack = next.inner.curr_kernel_stack; + let next_schedule_inst = next.inner.curr_schedule_inst; + let next_register_mask = next.inner.register_mask; + let next_registers_ptr = Box::as_ptr(&next.inner.registers); + + // Note: `None` means `schedule()` is *just* called, so `%rsp` still points to the bootstrap stack + if let Some(mut curr) = cpu.current_task.get().replace(Some(next)) { + cpu.registers.save(curr.inner.register_mask, curr.inner.registers.get_mut()); + asm!( + "movq %rsp, {stack}", + "leaq {ret}(%rip), {inst}", + + stack = in(reg) &raw mut curr.inner.curr_kernel_stack, + inst = in(reg) &raw mut curr.inner.curr_schedule_inst, + ret = label { + // Early-return `schedule()` + return + }, + + options(att_syntax, nostack) + ); + } + + cpu.registers.load(next_register_mask, UnsafeCell::raw_get(next_registers_ptr)); + asm!( + "movq {v_map}, %cr3", + "movq {stack}, %rsp", + "jmpq *{inst}", + + v_map = in(reg) next_v_map.addr(), + stack = in(reg) next_kernel_stack, + inst = in(reg) next_schedule_inst, + + options(att_syntax) + ) + } + None => return, + } + } +} diff --git a/kernel/src/task/mod.rs b/kernel/src/task/mod.rs new file mode 100644 index 0000000..3d306b5 --- /dev/null +++ b/kernel/src/task/mod.rs @@ -0,0 +1,133 @@ +use alloc::alloc::{alloc, dealloc, handle_alloc_error}; +use core::{alloc::LayoutError, fmt}; + +use elpytios_abi::{PAGE_LAYOUT, PAGE_SIZE}; +use elpytios_bootinfo::paddr::PAddr; + +use crate::{ + statics::virt_to_phys, + vaddr::{VAddr, VFlags, VirtualMap, VirtualMapError}, +}; + +mod imp { + cfg_select! { + target_arch = "x86_64" => { + mod x86_64; + pub use x86_64::*; + } + _ => { + compile_error!("Unsupported architecture"); + } + } +} +pub use imp::schedule; + +mod process; +mod queue; +pub use process::*; +pub use queue::*; + +#[derive(Clone)] +pub enum TaskCreateError { + Layout(LayoutError), + VMap(VirtualMapError), +} + +impl fmt::Debug for TaskCreateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for TaskCreateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Layout(e) => write!(f, "Couldn't allocate memory for task memory: {e}"), + Self::VMap(e) => write!(f, "Couldn't virtual-map task memory: {e}"), + } + } +} + +impl From for TaskCreateError { + #[inline] + fn from(value: LayoutError) -> Self { + Self::Layout(value) + } +} + +impl From for TaskCreateError { + #[inline] + fn from(value: VirtualMapError) -> Self { + Self::VMap(value) + } +} + +pub struct Task { + user_stack: *mut u8, + kernel_stack: *mut u8, + virtual_map: VirtualMap, + virtual_map_phys: PAddr, + inner: imp::Task, +} + +impl Task { + pub const USER_STACK_PAGES: usize = 15; + pub const KERNEL_STACK_PAGES: usize = 1; + + pub unsafe fn new(mut addr_start: VAddr, entry: VAddr, virtual_map: VirtualMap, virtual_map_phys: PAddr) -> Result { + let mut next_addr = |count| { + let prev = addr_start; + addr_start = addr_start.byte_add(count * PAGE_SIZE); + prev + }; + + unsafe { + let user_stack_layout = PAGE_LAYOUT.repeat_packed(Self::USER_STACK_PAGES + 1)?; + let user_stack = alloc(user_stack_layout); + if user_stack.is_null() { + handle_alloc_error(user_stack_layout) + } + + let kernel_stack_layout = PAGE_LAYOUT.repeat_packed(Self::KERNEL_STACK_PAGES + 1)?; + let kernel_stack = alloc(kernel_stack_layout); + if kernel_stack.is_null() { + handle_alloc_error(kernel_stack_layout) + } + + let user_stack_lower = next_addr(Self::USER_STACK_PAGES); + virtual_map.map( + virt_to_phys(VAddr::new(user_stack.addr() + PAGE_SIZE)), + user_stack_lower.byte_add(PAGE_SIZE), + Self::USER_STACK_PAGES, + VFlags::USER_MODE | VFlags::WRITABLE, + )?; + + let inner = imp::Task::new( + entry, + user_stack.add((Self::USER_STACK_PAGES + 1) * PAGE_SIZE), + user_stack_lower.byte_add((Self::USER_STACK_PAGES + 1) * PAGE_SIZE), + kernel_stack.add((Self::KERNEL_STACK_PAGES + 1) * PAGE_SIZE), + ); + + Ok(Self { + user_stack, + kernel_stack, + virtual_map, + virtual_map_phys, + inner, + }) + } + } +} + +impl Drop for Task { + fn drop(&mut self) { + unsafe { + dealloc(self.user_stack, PAGE_LAYOUT.repeat_packed(Self::USER_STACK_PAGES + 1).unwrap_unchecked()); + dealloc( + self.kernel_stack, + PAGE_LAYOUT.repeat_packed(Self::KERNEL_STACK_PAGES + 1).unwrap_unchecked(), + ); + } + } +} diff --git a/kernel/src/task/process.rs b/kernel/src/task/process.rs new file mode 100644 index 0000000..fa384a1 --- /dev/null +++ b/kernel/src/task/process.rs @@ -0,0 +1,171 @@ +use alloc::{ + alloc::{alloc, handle_alloc_error}, + boxed::Box, +}; +use core::{ + alloc::{Layout, LayoutError}, + fmt, ptr, +}; + +use elpytios_abi::PAGE_SIZE; +use elpytios_elf::{ + Elf64, ElfError, ElfSegmentType, + sys::{ElfProgramFlags, ElfRela64, ElfRela64Type, ElfType}, +}; + +use crate::{ + LOWER_HALF_ADDRESSES, + statics::{get_virtual_map, virt_to_phys}, + task::{Task, TaskCreateError}, + vaddr::{VAddr, VFlags, VirtualMapError}, +}; + +#[derive(Clone)] +pub enum ProcessCreateError { + Elf(ElfError), + Layout(LayoutError), + VMap(VirtualMapError), + Task(TaskCreateError), + NonRelocatable, + InvalidAlignment(u64), +} + +impl fmt::Debug for ProcessCreateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for ProcessCreateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Elf(e) => write!(f, "Couldn't parse ELF for process creation: {e}"), + Self::Layout(e) => write!(f, "Couldn't allocate memory for program segment: {e}"), + Self::VMap(e) => write!(f, "Couldn't virtual-map program segment: {e}"), + Self::Task(e) => write!(f, "{e}"), + Self::NonRelocatable => write!(f, "ELF is non-relocatable; recompile the program with -fPIE"), + Self::InvalidAlignment(align) => write!(f, "ELF program segment alignment isn't {PAGE_SIZE} ({align})"), + } + } +} + +impl From for ProcessCreateError { + #[inline] + fn from(value: ElfError) -> Self { + Self::Elf(value) + } +} + +impl From for ProcessCreateError { + #[inline] + fn from(value: LayoutError) -> Self { + Self::Layout(value) + } +} + +impl From for ProcessCreateError { + #[inline] + fn from(value: VirtualMapError) -> Self { + Self::VMap(value) + } +} + +impl From for ProcessCreateError { + #[inline] + fn from(value: TaskCreateError) -> Self { + Self::Task(value) + } +} + +#[repr(C, align(4096))] +pub struct Process { + executable: [u8], +} + +impl Process { + pub fn from_elf(elf: Elf64) -> Result<(Box, Task), ProcessCreateError> { + if !matches!(elf.prologue().elf_type, ElfType::DYNAMIC) { + return Err(ProcessCreateError::NonRelocatable) + } + + let mut base = u64::MAX; + let mut top = 0; + for segment in elf.program_segments() { + let segment = segment?; + let ElfSegmentType::Load(..) = segment.segment_type else { continue }; + base = base.min(segment.virtual_address); + top = top.max(segment.virtual_address + segment.memory_size); + } + + let layout = Layout::from_size_align((top - base) as usize, PAGE_SIZE)?; + let this = unsafe { alloc(layout) }; + if this.is_null() { + handle_alloc_error(layout) + } + + let exec_addr = virt_to_phys(VAddr::new(this.addr())); + + let (virtual_map, virtual_map_phys) = unsafe { get_virtual_map().for_userspace() }; + for segment in elf.program_segments() { + let segment = segment?; + let ElfSegmentType::Load(slice) = segment.segment_type else { continue }; + + if segment.alignment != PAGE_SIZE as u64 { + return Err(ProcessCreateError::InvalidAlignment(segment.alignment)) + } + + unsafe { + let offset = (segment.virtual_address - base) as usize; + this.byte_add(offset).copy_from_nonoverlapping(slice.as_ptr(), slice.len()); + this.byte_add(offset + slice.len()) + .write_bytes(0, segment.memory_size as usize - slice.len()); + + virtual_map.map( + exec_addr.byte_add(offset), + LOWER_HALF_ADDRESSES.start.byte_add(offset), + (segment.memory_size as usize).div_ceil(PAGE_SIZE), + { + let mut flags = VFlags::USER_MODE; + if !segment.flags.contains(ElfProgramFlags::EXECUTABLE) { + flags |= VFlags::EXECUTE_DISABLE; + } + if segment.flags.contains(ElfProgramFlags::WRITABLE) { + flags |= VFlags::WRITABLE; + } + flags + }, + )?; + } + } + + for segment in elf.program_segments() { + let ElfSegmentType::Dynamic { offset, size, stride } = segment?.segment_type else { continue }; + for i in 0..size / stride { + unsafe { + let rela = this.cast::().byte_add(offset - base as usize).add(i).read_unaligned(); + match rela.info.kind { + ElfRela64Type::X86_64_NONE => {} + ElfRela64Type::X86_64_RELATIVE => { + let slide = LOWER_HALF_ADDRESSES.start.addr() as i64 - base.cast_signed(); + let patch_addr = this.add(rela.offset as usize - base as usize); + let value = slide + rela.addend; + patch_addr.cast::().write(value); + } + kind => panic!("Unsupported Elf64_Rela kind: {}", kind.0), + } + } + } + } + + let addr_start = LOWER_HALF_ADDRESSES + .start + .byte_add(((top - base) as usize).next_multiple_of(PAGE_SIZE) + PAGE_SIZE); + + unsafe { + let entry = LOWER_HALF_ADDRESSES.start.byte_add((elf.program_entry() - base) as usize); + let this = Box::from_raw(ptr::from_raw_parts_mut(this, layout.size())); + let task = Task::new(addr_start, entry, virtual_map, virtual_map_phys)?; + Ok((this, task)) + } + } +} diff --git a/kernel/src/task/queue.rs b/kernel/src/task/queue.rs new file mode 100644 index 0000000..51fae64 --- /dev/null +++ b/kernel/src/task/queue.rs @@ -0,0 +1,31 @@ +use alloc::collections::vec_deque::VecDeque; + +use crate::{spin_sync::SpinMutex, task::Task}; + +// TODO create a lock-free structure for this +pub struct TaskQueue { + inner: SpinMutex>, +} + +unsafe impl Sync for TaskQueue {} + +impl TaskQueue { + #[inline] + pub const fn new() -> Self { + Self { + inner: SpinMutex::new(VecDeque::new()), + } + } + + #[inline] + pub fn pop_front(&self) -> Option { + self.inner.lock().pop_front() + } + + #[inline] + pub fn push_back(&self, task: Task) { + self.inner.lock().push_back(task) + } +} + +pub static TASK_QUEUE: TaskQueue = TaskQueue::new(); diff --git a/kernel/src/timer/mod.rs b/kernel/src/timer/mod.rs new file mode 100644 index 0000000..2f4f7b6 --- /dev/null +++ b/kernel/src/timer/mod.rs @@ -0,0 +1,41 @@ +use core::time::Duration; + +cfg_select! { + target_arch = "x86_64" => { + mod x86_64; + use x86_64 as imp; + pub use imp::Timer as TimerX86_64; + } + _ => { + compile_error!("Unsupported architecture"); + } +} + +#[repr(transparent)] +pub struct Timer { + pub inner: imp::Timer, +} + +impl !Send for imp::Timer {} +impl !Sync for imp::Timer {} + +impl Timer { + #[inline] + pub fn new() -> Self { + Self { inner: imp::Timer::new() } + } + + #[inline] + pub fn busy_wait(&self, duration: Duration) { + self.inner.busy_wait(duration); + } + + /// Schedules a timer interrupt. + /// + /// # Notes + /// Passing the function to be called by the interrupt is platform-specific. + #[inline] + pub fn schedule(&self, duration: Duration) { + self.inner.schedule(duration); + } +} diff --git a/kernel/src/timer/x86_64.rs b/kernel/src/timer/x86_64.rs new file mode 100644 index 0000000..78ec58a --- /dev/null +++ b/kernel/src/timer/x86_64.rs @@ -0,0 +1,111 @@ +use core::{ + arch::x86_64::{__cpuid_count, _rdtsc}, + hint::spin_loop, + time::Duration, +}; + +use crate::arch::x86_64::{Msr, wrmsr}; + +pub enum Timer { + Tsc { clock_hz: u64 }, +} + +impl Timer { + pub(super) fn new() -> Self { + if let Some(clock_hz) = tsc_frequency() { + Self::Tsc { clock_hz } + } else { + unimplemented!("Timer implementation fallback (FADT, HPET, legacy PIT)") + } + } + + pub(super) fn busy_wait(&self, duration: Duration) { + match *self { + Self::Tsc { clock_hz } => { + let ticks = Self::to_ticks(duration, clock_hz); + let start = unsafe { _rdtsc() }; + + while unsafe { _rdtsc() }.wrapping_sub(start) < ticks { + spin_loop(); + } + } + } + } + + pub(super) fn schedule(&self, duration: Duration) { + match *self { + Self::Tsc { clock_hz } => { + let ticks = Self::to_ticks(duration, clock_hz); + unsafe { + wrmsr(Msr::Ia32TscDeadline, _rdtsc().wrapping_add(ticks)); + } + } + } + } + + fn to_ticks(duration: Duration, clock_hz: u64) -> u64 { + let secs = duration.as_secs() as u128; + let nanos = duration.subsec_nanos() as u128; + let hz = clock_hz as u128; + + let ticks_secs = secs * hz; + let ticks_nanos = (nanos * hz) / 1_000_000_000; + + u64::try_from(ticks_secs + ticks_nanos).unwrap_or(u64::MAX) + } +} + +fn tsc_frequency() -> Option { + // TSC is only used if TSC-deadline mode is supported + let leaf_deadline = __cpuid_count(0x1, 0x0); + if leaf_deadline.ecx & (1 << 24) == 0 { + return None + } + + let max_leaf = __cpuid_count(0x0, 0x0).eax; + + // Core crystal clock + if max_leaf >= 0x15 { + let leaf = __cpuid_count(0x15, 0x0); + let denom = leaf.eax; + let numer = leaf.ebx; + let hz = leaf.ecx; + + if hz != 0 + && denom != 0 + && numer != 0 + && let Ok(freq) = u64::try_from(hz as u128 * numer as u128 / denom as u128) + { + return Some(freq) + } + } + + // Processor frequency (fallback) + if max_leaf >= 0x16 { + let leaf = __cpuid_count(0x16, 0x0); + let mhz = leaf.eax as u64; + + if mhz != 0 + && let Ok(freq) = u64::try_from(mhz as u128 * 1_000_000) + { + return Some(freq) + } + } + + // Hypervisor timing info (fallback, QEMU/KVM) + if leaf_deadline.ecx & (1 << 31) != 0 { + let leaf_hv = __cpuid_count(0x4000_0000, 0x0); + if leaf_hv.eax >= 0x4000_0010 { + let leaf_time = __cpuid_count(0x4000_0010, 0x0); + let khz = leaf_time.eax; + + if khz != 0 + && let Ok(freq) = u64::try_from(khz as u128 * 1_000) + { + return Some(freq) + } + } + } + + None +} diff --git a/kernel/src/vaddr/mod.rs b/kernel/src/vaddr/mod.rs new file mode 100644 index 0000000..2b7315a --- /dev/null +++ b/kernel/src/vaddr/mod.rs @@ -0,0 +1,63 @@ +pub(crate) use imp::*; +pub use imp::{VAddr, VirtualMap, VirtualMapBuilder}; + +cfg_select! { + target_arch = "x86_64" => { + mod x86_64; + use x86_64 as imp; + } + _ => { + compile_error!("Unsupported architecture"); + } +} + +use core::fmt; + +use bitflags::bitflags; +use elpytios_bootinfo::paddr::PAddr; + +#[derive(Debug, Clone, Copy)] +#[repr(transparent)] +pub struct VFlags(usize); +bitflags! { + impl VFlags: usize { + const WRITABLE = 1 << 0; + const USER_MODE = 1 << 1; + const WRITE_THROUGH = 1 << 2; + const CACHE_DISABLED = 1 << 3; + const ACCESSED = 1 << 4; + + /// Don't flush translation lookaside buffers when switching virtual map tables + const GLOBAL = 1 << 5; + const EXECUTE_DISABLE = 1 << 6; + } +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub enum VirtualMapError { + PageTable, + AlreadyMapped { p_addr: PAddr, v_addr: VAddr, p_addr_existing: PAddr }, +} + +impl fmt::Debug for VirtualMapError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for VirtualMapError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Self::PageTable => writeln!(f, "Couldn't allocate a page table"), + Self::AlreadyMapped { + p_addr, + v_addr, + p_addr_existing, + } => write!( + f, + "Couldn't map {v_addr:p} to {p_addr:p}: the virtual address is already mapped to {p_addr_existing:p}" + ), + } + } +} diff --git a/bootinfo/src/vaddr/x86_64/mod.rs b/kernel/src/vaddr/x86_64/mod.rs similarity index 83% rename from bootinfo/src/vaddr/x86_64/mod.rs rename to kernel/src/vaddr/x86_64/mod.rs index 4b20f7b..ad7476a 100644 --- a/bootinfo/src/vaddr/x86_64/mod.rs +++ b/kernel/src/vaddr/x86_64/mod.rs @@ -1,10 +1,9 @@ mod phys; mod virt; -pub(crate) use phys::*; +use elpytios_bootinfo::PAGE_SIZE; +pub use phys::*; pub use virt::*; -use crate::PAGE_SIZE; - const _: () = assert!(size_of::() == size_of::()); const fn assert_size_align() { diff --git a/bootinfo/src/vaddr/x86_64/phys.rs b/kernel/src/vaddr/x86_64/phys.rs similarity index 82% rename from bootinfo/src/vaddr/x86_64/phys.rs rename to kernel/src/vaddr/x86_64/phys.rs index c73a09a..2a76df4 100644 --- a/bootinfo/src/vaddr/x86_64/phys.rs +++ b/kernel/src/vaddr/x86_64/phys.rs @@ -2,9 +2,10 @@ use core::{fmt, mem}; use bitflags::bitflags; use bytemuck::Zeroable; +use elpytios_bootinfo::{PAGE_SIZE, paddr::PAddr}; use super::assert_size_align; -use crate::{PAGE_SIZE, paddr::PAddr, vaddr::VFlags}; +use crate::vaddr::VFlags; const _: () = assert_size_align::(); const _: () = assert_size_align::(); @@ -48,7 +49,9 @@ impl From for Entry { if value.contains(VFlags::ACCESSED) { out |= Self::ACCESSED } - + if value.contains(VFlags::EXECUTE_DISABLE) { + out |= Self::EXECUTE_DISABLE; + } out } } @@ -84,13 +87,13 @@ bitflags! { #[derive(Debug, Zeroable)] #[repr(C, align(4096))] pub struct Pml4Table { - pub pdpt_entries: [NodeEntry; PAGE_SIZE / size_of::()], + pub pml4_to_pdpt: [NodeEntry; PAGE_SIZE / size_of::()], } #[derive(Debug, Zeroable)] #[repr(C, align(4096))] pub struct PdptTable { - pub pd_entries: [PdptEntry; PAGE_SIZE / size_of::()], + pub pdpt_to_pd: [PdptEntry; PAGE_SIZE / size_of::()], } #[derive(Debug, Copy, Clone, Zeroable)] @@ -102,15 +105,10 @@ impl PdptLeafEntry { PAddr::new(self.0 & Self::ADDRESS_MASK.0) } - /*#[inline] + #[inline] pub const fn new(entry: Entry, addr: PAddr) -> Self { Self((entry.0 | Entry::PRESENT.0) & !Self::ADDRESS_MASK.0 | addr.addr() & Self::ADDRESS_MASK.0) } - - #[inline] - pub const fn is_present(self) -> bool { - self.0 & Entry::PRESENT.0 != 0 - }*/ } bitflags! { impl PdptLeafEntry: usize { @@ -122,6 +120,17 @@ bitflags! { } } +impl From for PdptLeafEntry { + #[inline] + fn from(value: VFlags) -> Self { + let mut out = Self::empty(); + if value.contains(VFlags::GLOBAL) { + out |= Self::GLOBAL; + } + out + } +} + #[derive(Copy, Clone)] pub enum UnionEntry { Node(Node), @@ -134,12 +143,6 @@ impl UnionEntry { let Self::Node(node) = self else { panic!("Not a node!") }; node } - - /*#[inline] - pub fn force_leaf(self) -> Leaf { - let Self::Leaf(leaf) = self else { panic!("Not a node!") }; - leaf - }*/ } #[derive(Copy, Clone, Zeroable)] @@ -162,10 +165,10 @@ impl PdptEntry { Self { node } } - /*#[inline] + #[inline] pub const fn leaf(leaf: PdptLeafEntry) -> Self { - unsafe { mem::transmute::(mem::transmute::(Self { leaf }) | NODE_IS_LEAF) } - }*/ + unsafe { mem::transmute::(mem::transmute::(Self { leaf }) | UNION_IS_LEAF) } + } #[inline] pub const fn kind(self) -> UnionEntry { @@ -178,17 +181,6 @@ impl PdptEntry { } } - /*#[inline] - pub const fn kind_mut(&mut self) -> UnionEntry<&mut NodeEntry, &mut PdptLeafEntry> { - unsafe { - if mem::transmute::(*self) & UNION_IS_LEAF != 0 { - UnionEntry::Leaf(&mut self.leaf) - } else { - UnionEntry::Node(&mut self.node) - } - } - }*/ - #[inline] pub const fn is_present(self) -> bool { unsafe { mem::transmute::(self) & Entry::PRESENT.0 != 0 } @@ -198,7 +190,7 @@ impl PdptEntry { #[derive(Zeroable)] #[repr(C, align(4096))] pub struct PdTable { - pub pt_entries: [PdEntry; PAGE_SIZE / size_of::()], + pub pd_to_pt: [PdEntry; PAGE_SIZE / size_of::()], } #[derive(Debug, Copy, Clone, Zeroable)] @@ -210,15 +202,10 @@ impl PdLeafEntry { PAddr::new(self.0 & Self::ADDRESS_MASK.0) } - /*#[inline] + #[inline] pub const fn new(entry: Entry, addr: PAddr) -> Self { Self((entry.0 | Entry::PRESENT.0) & !Self::ADDRESS_MASK.0 | addr.addr() & Self::ADDRESS_MASK.0) } - - #[inline] - pub const fn is_present(self) -> bool { - unsafe { mem::transmute::(self) & Entry::PRESENT.0 != 0 } - }*/ } bitflags! { impl PdLeafEntry: usize { @@ -230,6 +217,17 @@ bitflags! { } } +impl From for PdLeafEntry { + #[inline] + fn from(value: VFlags) -> Self { + let mut out = Self::empty(); + if value.contains(VFlags::GLOBAL) { + out |= Self::GLOBAL; + } + out + } +} + #[derive(Copy, Clone, Zeroable)] #[repr(C)] pub union PdEntry { @@ -250,10 +248,10 @@ impl PdEntry { Self { node } } - /*#[inline] + #[inline] pub const fn leaf(leaf: PdLeafEntry) -> Self { - unsafe { mem::transmute::(mem::transmute::(Self { leaf }) | NODE_IS_LEAF) } - }*/ + unsafe { mem::transmute::(mem::transmute::(Self { leaf }) | UNION_IS_LEAF) } + } #[inline] pub const fn kind(self) -> UnionEntry { @@ -266,17 +264,6 @@ impl PdEntry { } } - /*#[inline] - pub const fn kind_mut(&mut self) -> UnionEntry<&mut NodeEntry, &mut PdLeafEntry> { - unsafe { - if mem::transmute::(*self) & NODE_IS_LEAF != 0 { - UnionEntry::Leaf(&mut self.leaf) - } else { - UnionEntry::Node(&mut self.node) - } - } - }*/ - #[inline] pub const fn is_present(self) -> bool { unsafe { mem::transmute::(self) & Entry::PRESENT.0 != 0 } @@ -305,7 +292,7 @@ impl PtEntry { #[inline] pub const fn is_present(self) -> bool { - unsafe { mem::transmute::(self) & Entry::PRESENT.0 != 0 } + self.0 & Entry::PRESENT.0 != 0 } } bitflags! { @@ -317,3 +304,14 @@ bitflags! { const PROTECTION_KEY = ((1 << 4) - 1) << 59; } } + +impl From for PtEntry { + #[inline] + fn from(value: VFlags) -> Self { + let mut out = Self::empty(); + if value.contains(VFlags::GLOBAL) { + out |= Self::GLOBAL; + } + out + } +} diff --git a/kernel/src/vaddr/x86_64/virt.rs b/kernel/src/vaddr/x86_64/virt.rs new file mode 100644 index 0000000..1a7dcf9 --- /dev/null +++ b/kernel/src/vaddr/x86_64/virt.rs @@ -0,0 +1,411 @@ +use core::{fmt, hint::assert_unchecked}; + +use bytemuck::Zeroable; +use elpytios_bootinfo::{PAGE_SIZE, paddr::PAddr}; + +use crate::{ + HIGHER_HALF_ADDRESSES, + statics::phys_to_virt, + vaddr::{ + Entry, NodeEntry, PdEntry, PdLeafEntry, PdTable, PdptEntry, PdptLeafEntry, PdptTable, Pml4Table, PtEntry, PtTable, UnionEntry, VFlags, + VirtualMapError, + }, +}; + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Zeroable)] +#[repr(transparent)] +pub struct VAddr(usize); +impl VAddr { + #[inline] + pub const fn new(virtual_address: usize) -> Self { + Self(virtual_address) + } + + #[inline] + pub const fn addr(self) -> usize { + self.0 + } + + #[inline] + pub const fn ptr(self) -> *const T { + self.0 as *const T + } + + #[inline] + pub const fn ptr_mut(self) -> *mut T { + self.0 as *mut T + } + + #[inline] + pub const fn byte_add(self, offset: usize) -> Self { + Self(self.0 + offset) + } + + #[inline] + pub(crate) const fn info(self) -> VAddrInfo { + VAddrInfo { + page_offset: self.0 & 0xfff, + pt_index: (self.0 >> 12) & 0x1ff, + pd_index: (self.0 >> 21) & 0x1ff, + pdpt_index: (self.0 >> 30) & 0x1ff, + pml4_index: (self.0 >> 39) & 0x1ff, + } + } +} + +impl From<*const T> for VAddr { + #[inline] + fn from(value: *const T) -> Self { + Self(value as usize) + } +} + +impl From<*mut T> for VAddr { + #[inline] + fn from(value: *mut T) -> Self { + Self(value as usize) + } +} + +#[derive(Debug, Clone, Copy, Zeroable)] +#[repr(C)] +pub(crate) struct VAddrInfo { + pub page_offset: usize, + pub pt_index: usize, + pub pd_index: usize, + pub pdpt_index: usize, + pub pml4_index: usize, +} + +impl fmt::Debug for VAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.info()) + } +} + +impl fmt::Pointer for VAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:#018p}", self.0 as *const ()) + } +} + +#[derive(Debug)] +#[repr(transparent)] +pub struct VirtualMapBuilder Option> { + map: VirtualMap>, +} + +impl Option> VirtualMapBuilder { + /// # Safety + /// - `page_table_new` must return a [`PAGE_SIZE`]-aligned physical address that is completely + /// free to be written to (nothing else "owns" it). + /// - `page_table_phys` must be one such pointer that satisfies to be a return value of + /// `page_table_new` + /// - `page_table_ptr` must convert physical addresses returned by `new_page_table` into a + /// pointer that points to a page table. + #[inline] + pub const unsafe fn new(page_table_phys: PAddr, page_table_new: T, page_table_ptr: unsafe fn(PAddr) -> *mut ()) -> Self { + Self { + map: VirtualMap { + mapper: LocalMapper { + page_table_phys, + page_table_new, + page_table_ptr, + }, + }, + } + } + + #[inline] + pub fn map(&mut self, p_addr: PAddr, v_addr: VAddr, page_count: usize, flags: VFlags) -> Result<(), VirtualMapError> { + unsafe { self.map.map(p_addr, v_addr, page_count, flags) } + } + + /// # Safety + /// Direct-map offset must have been set and usable by the time *any* methods in the returned + /// [`VirtualMap`] is called. + pub unsafe fn finish(self) -> VirtualMap { + VirtualMap { + mapper: unsafe { sealed::OffsetMapper::new(phys_to_virt(self.map.mapper.page_table_phys).ptr_mut()) }, + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct LocalMapper Option> { + page_table_phys: PAddr, + page_table_new: T, + page_table_ptr: unsafe fn(PAddr) -> *mut (), +} + +unsafe impl Option> sealed::VirtualMapper for LocalMapper { + #[inline] + fn new_page_table(&self) -> Option { + (self.page_table_new)().inspect(|&addr| unsafe { (self.page_table_ptr)(addr).cast::().write_bytes(0, PAGE_SIZE) }) + } + + #[inline] + fn pml4(&self) -> *mut Pml4Table { + unsafe { (self.page_table_ptr)(self.page_table_phys).cast() } + } + + #[inline] + unsafe fn pdpt(&self, pml4_index: usize) -> *mut PdptTable { + unsafe { + (self.page_table_ptr)((*self.pml4()).pml4_to_pdpt[pml4_index].child_addr()) + .cast::() + .as_mut_unchecked() + } + } + + #[inline] + unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> *mut PdTable { + unsafe { + (self.page_table_ptr)(match (*self.pdpt(pml4_index)).pdpt_to_pd[pdpt_index].kind() { + UnionEntry::Node(e) => e.child_addr(), + UnionEntry::Leaf(..) => unreachable!("PDPT entry is a huge page entry"), + }) + .cast::() + .as_mut_unchecked() + } + } + + #[inline] + unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> *mut PtTable { + unsafe { + (self.page_table_ptr)(match (*self.pd(pml4_index, pdpt_index)).pd_to_pt[pd_index].kind() { + UnionEntry::Node(e) => e.child_addr(), + UnionEntry::Leaf(..) => unreachable!("PD entry is a huge page entry"), + }) + .cast::() + .as_mut_unchecked() + } + } +} + +#[derive(Debug)] +#[repr(transparent)] +// Note: From user-facing API perspective, `VirtualMap` must have no trait bounds. +pub struct VirtualMap { + mapper: T, +} + +impl VirtualMap { + /// # Safety + /// - Both addresses must be page-aligned. + /// - There must never be concurrent (multithreaded) calls to this method that have the same + /// virtual page occupied by `v_addr`. + pub unsafe fn map(&self, mut p_addr: PAddr, mut v_addr: VAddr, mut page_count: usize, flags: VFlags) -> Result<(), VirtualMapError> { + unsafe { + assert_unchecked(p_addr.addr().is_multiple_of(PAGE_SIZE)); + assert_unchecked(v_addr.addr().is_multiple_of(PAGE_SIZE)); + } + + let mut node_entry = Entry::WRITABLE; + if flags.contains(VFlags::USER_MODE) { + node_entry |= Entry::USER_MODE; + } + + while page_count > 0 { + let VAddrInfo { + pt_index, + pd_index, + pdpt_index, + pml4_index, + .. + } = v_addr.info(); + + unsafe { + match &raw mut (*self.mapper.pml4()).pml4_to_pdpt[pml4_index] { + e if !(*e).is_present() => e.write(NodeEntry::new( + node_entry, + self.mapper.new_page_table().ok_or(VirtualMapError::PageTable)?, + )), + _ => {} + } + + match &raw mut (*self.mapper.pdpt(pml4_index)).pdpt_to_pd[pdpt_index] { + e if !(*e).is_present() => { + if pd_index == 0 && page_count >= 512 * 512 { + e.write(PdptEntry::leaf(PdptLeafEntry::new(flags.into(), p_addr) | flags.into())); + p_addr = p_addr.byte_add(512 * 512 * PAGE_SIZE); + v_addr = v_addr.byte_add(512 * 512 * PAGE_SIZE); + page_count -= 512 * 512; + continue + } else { + e.write(PdptEntry::node(NodeEntry::new( + node_entry, + self.mapper.new_page_table().ok_or(VirtualMapError::PageTable)?, + ))) + } + } + e if let UnionEntry::Leaf(e) = (*e).kind() => { + return Err(VirtualMapError::AlreadyMapped { + p_addr, + v_addr, + p_addr_existing: e.addr(), + }) + } + _ => {} + } + + match &raw mut (*self.mapper.pd(pml4_index, pdpt_index)).pd_to_pt[pd_index] { + e if !(*e).is_present() => { + if pt_index == 0 && page_count >= 512 { + e.write(PdEntry::leaf(PdLeafEntry::new(flags.into(), p_addr) | flags.into())); + p_addr = p_addr.byte_add(512 * PAGE_SIZE); + v_addr = v_addr.byte_add(512 * PAGE_SIZE); + page_count -= 512; + continue + } else { + e.write(PdEntry::node(NodeEntry::new( + node_entry, + self.mapper.new_page_table().ok_or(VirtualMapError::PageTable)?, + ))) + } + } + e if let UnionEntry::Leaf(e) = (*e).kind() => { + return Err(VirtualMapError::AlreadyMapped { + p_addr, + v_addr, + p_addr_existing: e.addr(), + }) + } + _ => {} + } + + match &raw mut (*self.mapper.pt(pml4_index, pdpt_index, pd_index)).phys_pages[pt_index] { + e if (*e).is_present() => { + return Err(VirtualMapError::AlreadyMapped { + p_addr, + v_addr, + p_addr_existing: (*e).addr(), + }) + } + e => { + e.write(PtEntry::new(flags.into(), p_addr) | flags.into()); + p_addr = p_addr.byte_add(PAGE_SIZE); + v_addr = v_addr.byte_add(PAGE_SIZE); + page_count -= 1; + } + } + } + } + + Ok(()) + } +} + +impl VirtualMap { + /// # Safety + /// - [`map`](Self::map) must not be called for higher-half addresses on the returned mapper + /// - [`map`](Self::map) must not be called for lower-half addresses on the `self` mapper. + pub unsafe fn for_userspace(&self) -> (Self, PAddr) { + use sealed::VirtualMapper; + + let pml4_phys = self + .mapper + .new_page_table() + .expect("Couldn't allocate a new page for userspace virtual map"); + let pml4 = phys_to_virt(pml4_phys).ptr_mut::(); + + unsafe { + let kernel_pml4 = self.mapper.pml4(); + for i in HIGHER_HALF_ADDRESSES.start.info().pml4_index..=HIGHER_HALF_ADDRESSES.end.info().pml4_index { + (*pml4).pml4_to_pdpt[i] = (*kernel_pml4).pml4_to_pdpt[i]; + } + + ( + Self { + mapper: sealed::OffsetMapper::new(pml4), + }, + pml4_phys, + ) + } + } +} + +mod sealed { + use core::hint::unreachable_unchecked; + + use super::*; + use crate::statics::get_phys_alloc; + + #[allow(unused_variables, reason = "Available for implementors, not defaults")] + pub unsafe trait VirtualMapper { + fn new_page_table(&self) -> Option; + + fn pml4(&self) -> *mut Pml4Table; + + /// # Safety + /// - `pml4_index` must be within `0..512` (exclusive). + unsafe fn pdpt(&self, pml4_index: usize) -> *mut PdptTable; + + /// # Safety: + /// - [`Self::pdpt()`] to the given indices must return a node entry, not leaf. + /// - `pml4_index` and `pdpt_index` must be within `0..512` (exclusive). + unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> *mut PdTable; + + /// # Safety: + /// - [`Self::pd()`] to the given indices must return a node entry, not leaf. + /// - `pml4_index`, `pdpt_index`, and `pd_index` must be within `0..512` (exclusive). + unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> *mut PtTable; + } + + #[derive(Debug)] + #[repr(transparent)] + pub struct OffsetMapper { + pml4: *mut Pml4Table, + } + + impl OffsetMapper { + #[inline] + pub const unsafe fn new(pml4: *mut Pml4Table) -> Self { + Self { pml4 } + } + } + + unsafe impl VirtualMapper for OffsetMapper { + #[inline] + fn new_page_table(&self) -> Option { + get_phys_alloc() + .lock() + .alloc(0) + .ok() + .inspect(|&addr| unsafe { phys_to_virt(addr).ptr_mut::().write_bytes(0, PAGE_SIZE) }) + } + + #[inline] + fn pml4(&self) -> *mut Pml4Table { + self.pml4 + } + + #[inline] + unsafe fn pdpt(&self, pml4_index: usize) -> *mut PdptTable { + unsafe { phys_to_virt((*self.pml4()).pml4_to_pdpt.get_unchecked_mut(pml4_index).child_addr()).ptr_mut() } + } + + #[inline] + unsafe fn pd(&self, pml4_index: usize, pdpt_index: usize) -> *mut PdTable { + unsafe { + phys_to_virt(match (*self.pdpt(pml4_index)).pdpt_to_pd.get_unchecked_mut(pdpt_index).kind() { + UnionEntry::Leaf(..) => unreachable_unchecked(), + UnionEntry::Node(e) => e.child_addr(), + }) + .ptr_mut() + } + } + + #[inline] + unsafe fn pt(&self, pml4_index: usize, pdpt_index: usize, pd_index: usize) -> *mut PtTable { + unsafe { + phys_to_virt(match (*self.pd(pml4_index, pdpt_index)).pd_to_pt.get_unchecked_mut(pd_index).kind() { + UnionEntry::Leaf(..) => unreachable_unchecked(), + UnionEntry::Node(e) => e.child_addr(), + }) + .ptr_mut() + } + } + } +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml index c5fc9b2..a7bfd04 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly" -components = ["rust-src"] +channel = "nightly-2026-07-19" +components = ["rust-analyzer", "rust-src", "rustfmt"] targets = ["x86_64-unknown-uefi", "x86_64-unknown-none"] diff --git a/rustc-sysroot b/rustc-sysroot index 226a387..eb65dde 100755 --- a/rustc-sysroot +++ b/rustc-sysroot @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -import re -import os import subprocess import sys from pathlib import Path @@ -43,4 +41,4 @@ if __name__ == "__main__": print(err) exit(1) except: - exit(1) \ No newline at end of file + exit(1) diff --git a/std/sys-src/alloc/mod.rs b/std/sys-src/alloc/mod.rs index bd7de6c..6f88a43 100644 --- a/std/sys-src/alloc/mod.rs +++ b/std/sys-src/alloc/mod.rs @@ -1,56 +1,42 @@ -use crate::alloc::{GlobalAlloc, Layout, System}; - -// The minimum alignment guaranteed by the architecture. This value is used to -// add fast paths for low alignment values. -#[allow(dead_code)] -const MIN_ALIGN: usize = if cfg!(any( - all(target_arch = "riscv32", any(target_os = "espidf", target_os = "zkvm")), - all(target_arch = "xtensa", target_os = "espidf"), -)) { - // The allocator on the esp-idf and zkvm platforms guarantees 4 byte alignment. - 4 -} else if cfg!(any( - target_arch = "x86", - target_arch = "arm", - target_arch = "m68k", - target_arch = "csky", - target_arch = "loongarch32", - target_arch = "mips", - target_arch = "mips32r6", - target_arch = "powerpc", - target_arch = "powerpc64", - target_arch = "sparc", - target_arch = "wasm32", - target_arch = "hexagon", - target_arch = "riscv32", - target_arch = "xtensa", -)) { - 8 -} else if cfg!(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "arm64ec", - target_arch = "loongarch64", - target_arch = "mips64", - target_arch = "mips64r6", - target_arch = "s390x", - target_arch = "sparc64", - target_arch = "riscv64", - target_arch = "wasm64", -)) { - 16 -} else { - panic!("add a value for MIN_ALIGN") +use elpytios_abi::{FileHandle, Syscall}; +use elpytios_alloc::{HeapAllocator, PageAllocator}; + +use crate::{ + alloc::{GlobalAlloc, Layout}, + ptr::NonNull, }; -#[stable(feature = "alloc_system_type", since = "1.28.0")] -#[expect(unused)] -unsafe impl GlobalAlloc for System { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - unimplemented!() - } +#[global_allocator] +static IMPL: HeapAllocator = HeapAllocator::new(StdPageAllocator); - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - unimplemented!() +struct StdPageAllocator; +unsafe impl PageAllocator for StdPageAllocator { + #[inline] + fn alloc(&self, order: u32) -> Option> { + unsafe { NonNull::new(Syscall::mem_map(FileHandle::NONE, 0, 1 << order, 0)) } } + + // TODO unmap syscall + #[inline] + unsafe fn dealloc(&self, _ptr: NonNull, _order: u32) {} +} + +#[inline] +pub unsafe fn alloc(layout: Layout) -> *mut u8 { + unsafe { IMPL.alloc(layout) } +} + +#[inline] +pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { + unsafe { IMPL.dealloc(ptr, layout) } +} + +#[inline] +pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 { + unsafe { IMPL.alloc_zeroed(layout) } +} + +#[inline] +pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + unsafe { IMPL.realloc(ptr, layout, new_size) } } diff --git a/std/sys-src/fs/common.rs b/std/sys-src/fs/common.rs index c115da1..246adbc 100644 --- a/std/sys-src/fs/common.rs +++ b/std/sys-src/fs/common.rs @@ -1,68 +1,12 @@ use crate::{ - fmt, fs, - io::{self, Error, ErrorKind}, + fmt, io, path::{Path, PathBuf}, sys::{ IntoInner, fs::{File, FileAttr, OpenOptions}, - helpers::ignore_notfound, }, }; -pub(crate) const NOT_FILE_ERROR: Error = io::const_error!( - ErrorKind::InvalidInput, - "the source path is neither a regular file nor a symlink to a regular file", -); - -pub fn copy(from: &Path, to: &Path) -> io::Result { - let mut reader = fs::File::open(from)?; - let metadata = reader.metadata()?; - - if !metadata.is_file() { - return Err(NOT_FILE_ERROR); - } - - let mut writer = fs::File::create(to)?; - let perm = metadata.permissions(); - - let ret = io::copy(&mut reader, &mut writer)?; - writer.set_permissions(perm)?; - Ok(ret) -} - -pub fn remove_dir_all(path: &Path) -> io::Result<()> { - let filetype = fs::symlink_metadata(path)?.file_type(); - if filetype.is_symlink() { fs::remove_file(path) } else { remove_dir_all_recursive(path) } -} - -fn remove_dir_all_recursive(path: &Path) -> io::Result<()> { - for child in fs::read_dir(path)? { - let result: io::Result<()> = try { - let child = child?; - if child.file_type()?.is_dir() { - remove_dir_all_recursive(&child.path())?; - } else { - fs::remove_file(&child.path())?; - } - }; - // ignore internal NotFound errors to prevent race conditions - if let Err(err) = &result - && err.kind() != io::ErrorKind::NotFound - { - return result; - } - } - ignore_notfound(fs::remove_dir(path)) -} - -pub fn exists(path: &Path) -> io::Result { - match fs::metadata(path) { - Ok(_) => Ok(true), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(error), - } -} - pub struct Dir { path: PathBuf, } @@ -79,6 +23,14 @@ impl Dir { pub fn metadata(&self) -> io::Result { self.path.metadata().map(|m| m.into_inner()) } + + pub fn remove_file(&self, path: &Path) -> io::Result<()> { + crate::fs::remove_file(self.path.join(path)) + } + + pub fn rename(&self, from: &Path, to_dir: &Self, to: &Path) -> io::Result<()> { + crate::fs::rename(self.path.join(from), to_dir.path.join(to)) + } } impl fmt::Debug for Dir { diff --git a/std/sys-src/io/mod.rs b/std/sys-src/io/mod.rs index e14b937..f24c5c3 100644 --- a/std/sys-src/io/mod.rs +++ b/std/sys-src/io/mod.rs @@ -11,7 +11,3 @@ mod kernel_copy; pub use error::{decode_error_kind, errno, error_string, is_interrupted}; pub use is_terminal::is_terminal; pub use kernel_copy::{CopyState, kernel_copy}; - -// Bare metal platforms usually have very small amounts of RAM -// (in the order of hundreds of KB) -pub const DEFAULT_BUF_SIZE: usize = 8 * 1024; diff --git a/std/sys-src/mod.rs b/std/sys-src/mod.rs index c369d01..e7f945c 100644 --- a/std/sys-src/mod.rs +++ b/std/sys-src/mod.rs @@ -1,9 +1,9 @@ -mod alloc; mod configure_builtins; mod helpers; -mod pal; mod personality; +mod startup; +pub mod alloc; pub mod args; pub mod backtrace; pub mod cmath; @@ -26,7 +26,7 @@ pub mod thread; pub mod thread_local; pub mod time; -pub use pal::*; +pub use startup::*; /// A trait for viewing representations from std types. #[cfg_attr(not(target_os = "linux"), allow(unused))] diff --git a/std/sys-src/os_str/mod.rs b/std/sys-src/os_str/mod.rs index a3ddfba..88cd603 100644 --- a/std/sys-src/os_str/mod.rs +++ b/std/sys-src/os_str/mod.rs @@ -279,12 +279,6 @@ impl Slice { self.inner.clone_into(&mut buf.inner) } - #[inline] - pub fn into_box(&self) -> Box { - let boxed: Box = self.inner.into(); - unsafe { mem::transmute(boxed) } - } - #[inline] pub fn empty_box() -> Box { let boxed: Box = Default::default(); diff --git a/std/sys-src/pal.rs b/std/sys-src/pal.rs deleted file mode 100644 index ece6ebc..0000000 --- a/std/sys-src/pal.rs +++ /dev/null @@ -1,11 +0,0 @@ -// SAFETY: must be called only once during runtime initialization. -// NOTE: this is not guaranteed to run, for example when Rust code is called externally. -pub unsafe fn init(_argc: isize, _argv: *const *const u8, _sigpipe: u8) {} - -// SAFETY: must be called only once during runtime cleanup. -// NOTE: this is not guaranteed to run, for example when the program aborts. -pub unsafe fn cleanup() {} - -pub fn abort_internal() -> ! { - core::intrinsics::abort(); -} diff --git a/std/sys-src/personality/mod.rs b/std/sys-src/personality/mod.rs index fece058..325c47b 100644 --- a/std/sys-src/personality/mod.rs +++ b/std/sys-src/personality/mod.rs @@ -14,5 +14,5 @@ mod dwarf; #[lang = "eh_personality"] fn rust_eh_personality() { - crate::intrinsics::abort() + loop {} } diff --git a/std/sys-src/startup.rs b/std/sys-src/startup.rs new file mode 100644 index 0000000..b0ff4b7 --- /dev/null +++ b/std/sys-src/startup.rs @@ -0,0 +1,29 @@ +use crate::ffi::c_int; + +/// SAFETY: must be called only once during runtime initialization. +pub unsafe fn init(_argc: isize, _argv: *const *const u8, _sigpipe: u8) {} + +/// SAFETY: must be called only once during runtime cleanup. +pub unsafe fn cleanup() {} + +pub fn abort_internal() -> ! { + loop {} +} + +// Compiler-generated shim +unsafe extern "C" { + safe fn main(argc: c_int, argv: *const *const u8) -> c_int; +} + +cfg_select! { + target_arch = "x86_64" => { + #[unsafe(no_mangle)] + pub unsafe extern "sysv64" fn _start(argc: c_int, argv: *const *const u8) -> ! { + let _exit_code = main(argc, argv); + loop {} + } + } + _ => { + compile_error!("Unsupported architecture"); + } +} diff --git a/std/sys-src/thread/mod.rs b/std/sys-src/thread/mod.rs index 011b25b..16e4ada 100644 --- a/std/sys-src/thread/mod.rs +++ b/std/sys-src/thread/mod.rs @@ -1,3 +1,5 @@ +use elpytios_abi::Syscall; + use crate::{ ffi::CStr, io, @@ -35,8 +37,11 @@ pub fn current_os_id() -> Option { None } +#[inline] pub fn yield_now() { - // do nothing + unsafe { + Syscall::yield_now(); + } } pub fn set_name(_name: &CStr) { diff --git a/target-specs/x86_64-unknown-elpytios.json b/target-specs/x86_64-unknown-elpytios.json index 4f209ed..6da49f6 100644 --- a/target-specs/x86_64-unknown-elpytios.json +++ b/target-specs/x86_64-unknown-elpytios.json @@ -3,7 +3,7 @@ "cpu": "x86-64", "crt-static-respected": true, "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", - "llvm-target": "x86_64-unknown-none", + "llvm-target": "x86_64-unknown-elpytios", "max-atomic-width": 64, "has-thread-local": false, "metadata": { @@ -13,16 +13,22 @@ "tier": 3 }, "os": "elpytios", + "executables": true, + "emit-debug-gdb-scripts": false, + "relocation-model": "pic", + "position-independent-executables": true, "target-pointer-width": 64, "linker-flavor": "ld.lld", "linker": "rust-lld", "panic-strategy": "abort", "pre-link-args": { - "gnu-cc": [ - "-m64" - ], - "gnu-lld-cc": [ - "-m64" + "ld.lld": [ + "-z", + "max-page-size=4096", + "-z", + "common-page-size=4096", + "-z", + "separate-loadable-segments" ] } } \ No newline at end of file diff --git a/user/shell/Cargo.toml b/user/shell/Cargo.toml new file mode 100644 index 0000000..90a2ed5 --- /dev/null +++ b/user/shell/Cargo.toml @@ -0,0 +1,12 @@ +cargo-features = ['per-package-target'] + +[package] +name = "elpytios-shell" +version = "0.1.0" +edition = "2024" +rust-version = "1.98" + +forced-target = "target-specs/x86_64-unknown-elpytios.json" + +[dependencies] +elpytios-abi = { path = "../../abi" } diff --git a/user/shell/src/main.rs b/user/shell/src/main.rs new file mode 100644 index 0000000..7ad5586 --- /dev/null +++ b/user/shell/src/main.rs @@ -0,0 +1,5 @@ +fn main() { + loop { + std::thread::yield_now(); + } +} diff --git a/x b/x index 0371129..385f9e4 100755 --- a/x +++ b/x @@ -21,6 +21,7 @@ from pathlib import Path root = Path(__file__).resolve().parent elpytios_abi_root = root / "abi" +elpytios_alloc_root = root / "alloc" elpytios_std_root = root / "std" library_dst_root = elpytios_std_root / "rust-src" @@ -186,6 +187,10 @@ def fetch_std(_args): "path": str(elpytios_abi_root), "features": ["sysroot-dep"], } + packages["std"].manifest["dependencies"]["elpytios-alloc"] = { + "path": str(elpytios_alloc_root), + "features": ["sysroot-dep"], + } for package in packages.values(): package.file.write_text(package.manifest.as_string(), encoding="utf-8") @@ -283,15 +288,7 @@ def create_file_qemu(_args): runner_esp.mkdir(parents=True, exist_ok=True) -def run_qemu(args): - if not runner_fs.exists(): - create_file_qemu(args) - - accel = "tcg" - match sys.platform: - case "win32": accel = "whpx" - case "linux": accel = "kvm" - +def build_boot(args): profile = "bootloader_debug" if args.debug else "bootloader" subprocess.run( [ @@ -321,6 +318,18 @@ def run_qemu(args): check=True ) +def run_qemu(args): + profile = "bootloader_debug" if args.debug else "bootloader" + if not runner_fs.exists(): + create_file_qemu(args) + + build_boot(args) + + accel = "tcg" + match sys.platform: + case "win32": accel = "whpx" + case "linux": accel = "kvm" + runner_boot_dir.mkdir(parents=True, exist_ok=True) shutil.copy(root / "target" / "x86_64-unknown-uefi" / profile / "elpytios-bootloader.efi", runner_boot_file) @@ -328,6 +337,8 @@ def run_qemu(args): [ f"qemu-system-{platform.machine().replace("AMD64", "x86_64")}", "-accel", accel, + "-cpu", "host,migratable=off", + # "-smp", "cores=4,threads=2,sockets=1", "-drive", f"if=pflash,format=raw,readonly=on,file={runner_ovmf / "OVMF_CODE.4m.fd"}", "-drive", f"if=pflash,format=raw,readonly=on,file={runner_ovmf / "OVMF_VARS.4m.fd"}", "-drive", f"format=raw,file=fat:rw:{runner_esp}", @@ -336,11 +347,15 @@ def run_qemu(args): "-m", "4G", "-device", "virtio-vga", "-vga", "virtio", - "-monitor", "stdio", + # Using `COM3` is the cleanest for the kernel, for some reason. + "-serial", "file:platform.log", + "-serial", "null", + "-serial", "stdio", + "-d", "int,cpu_reset,guest_errors", "-no-reboot", "-no-shutdown", "-D", "qemu_except.log", *(["-s"] if args.debug else []), ], stdout=None, - stderr=None, + stderr=open("qemu_stderr.log", "w"), check=True ) @@ -370,6 +385,14 @@ def main(): std_sub.add_parser("build").set_defaults(func=build_std) std_sub.add_parser("clean").set_defaults(func=clean_std) + # `x boot` + boot = sub.add_parser("boot") + boot_sub = boot.add_subparsers(dest="boot_cmd") + + boot_build = boot_sub.add_parser("build") + boot_build.set_defaults(func=build_boot) + boot_build.add_argument("--debug", action="store_true") + # `x qemu` qemu = sub.add_parser("qemu") qemu.set_defaults(func=run_qemu) # Default to `x qemu run`