From 0892b096ee7c2d026b6698f6679f6159c9a9e8b8 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Wed, 29 Nov 2023 07:46:18 -0800 Subject: [PATCH] On 32-bit x86, get the vsyscall address from the AUX vector On 32-bit x86, get the vsyscall address from the AT_SYSINFO AUX vector entry, rather than looking it up in the vDSO. This avoids the need to link in all the vDSO code if it isn't otherwise needed. And, it's simpler, avoiding the need for the `rustix_int_0x80` function. --- src/backend/linux_raw/arch/mod.rs | 6 +- src/backend/linux_raw/arch/x86.rs | 2 +- src/backend/linux_raw/mod.rs | 6 +- src/backend/linux_raw/param/auxv.rs | 104 +++++++++--- src/backend/linux_raw/param/init.rs | 12 ++ src/backend/linux_raw/param/libc_auxv.rs | 32 ++++ src/backend/linux_raw/vdso_wrappers.rs | 199 ----------------------- src/backend/linux_raw/x86_vsyscall.rs | 103 ++++++++++++ 8 files changed, 237 insertions(+), 227 deletions(-) create mode 100644 src/backend/linux_raw/x86_vsyscall.rs diff --git a/src/backend/linux_raw/arch/mod.rs b/src/backend/linux_raw/arch/mod.rs index ac9e25fa7..12c82be18 100644 --- a/src/backend/linux_raw/arch/mod.rs +++ b/src/backend/linux_raw/arch/mod.rs @@ -51,11 +51,11 @@ pub(in crate::backend) mod asm; ))] pub(in crate::backend) use self::asm as choose; -// On 32-bit x86, use vDSO wrappers for all syscalls. We could use the -// architecture syscall instruction (`int 0x80`), but the vDSO kernel_vsyscall +// On 32-bit x86, use the kernel_vsyscall mechanism for syscalls. We could use +// the architecture syscall instruction (`int 0x80`), but the kernel_vsyscall // mechanism is much faster. #[cfg(target_arch = "x86")] -pub(in crate::backend) use super::vdso_wrappers::x86_via_vdso as choose; +pub(in crate::backend) use super::x86_vsyscall as choose; // This would be the code for always using `int 0x80` on 32-bit x86. //#[cfg(target_arch = "x86")] diff --git a/src/backend/linux_raw/arch/x86.rs b/src/backend/linux_raw/arch/x86.rs index e789181cc..2429659e5 100644 --- a/src/backend/linux_raw/arch/x86.rs +++ b/src/backend/linux_raw/arch/x86.rs @@ -15,7 +15,7 @@ use crate::backend::reg::{ ArgReg, FromAsm, RetReg, SyscallNumber, ToAsm, A0, A1, A2, A3, A4, A5, R0, }; -use crate::backend::vdso_wrappers::SyscallType; +use crate::backend::x86_vsyscall::SyscallType; use core::arch::asm; #[inline] diff --git a/src/backend/linux_raw/mod.rs b/src/backend/linux_raw/mod.rs index 0d4e5332d..335507f91 100644 --- a/src/backend/linux_raw/mod.rs +++ b/src/backend/linux_raw/mod.rs @@ -18,10 +18,12 @@ mod arch; mod conv; mod reg; -#[cfg(any(feature = "time", feature = "process", target_arch = "x86"))] +#[cfg(any(feature = "time", feature = "process"))] mod vdso; -#[cfg(any(feature = "time", feature = "process", target_arch = "x86"))] +#[cfg(any(feature = "time", feature = "process"))] mod vdso_wrappers; +#[cfg(target_arch = "x86")] +mod x86_vsyscall; #[cfg(feature = "event")] pub(crate) mod event; diff --git a/src/backend/linux_raw/param/auxv.rs b/src/backend/linux_raw/param/auxv.rs index b9ac27eec..888ad8a22 100644 --- a/src/backend/linux_raw/param/auxv.rs +++ b/src/backend/linux_raw/param/auxv.rs @@ -20,6 +20,8 @@ use core::sync::atomic::AtomicU8; use core::sync::atomic::Ordering::Relaxed; use core::sync::atomic::{AtomicPtr, AtomicUsize}; use linux_raw_sys::elf::*; +#[cfg(target_arch = "x86")] +use linux_raw_sys::general::AT_SYSINFO; use linux_raw_sys::general::{ AT_BASE, AT_CLKTCK, AT_EXECFN, AT_HWCAP, AT_HWCAP2, AT_NULL, AT_PAGESZ, AT_SYSINFO_EHDR, }; @@ -34,8 +36,12 @@ pub(crate) fn page_size() -> usize { let mut page_size = PAGE_SIZE.load(Relaxed); if page_size == 0 { - init_auxv(); - page_size = PAGE_SIZE.load(Relaxed); + #[cold] + fn compute_page_size() -> usize { + init_auxv(); + PAGE_SIZE.load(Relaxed) + } + page_size = compute_page_size(); } page_size @@ -47,8 +53,12 @@ pub(crate) fn clock_ticks_per_second() -> u64 { let mut ticks = CLOCK_TICKS_PER_SECOND.load(Relaxed); if ticks == 0 { - init_auxv(); - ticks = CLOCK_TICKS_PER_SECOND.load(Relaxed); + #[cold] + fn compute_clock_ticks_per_second() -> usize { + init_auxv(); + CLOCK_TICKS_PER_SECOND.load(Relaxed) + } + ticks = compute_clock_ticks_per_second(); } ticks as u64 @@ -61,9 +71,12 @@ pub(crate) fn linux_hwcap() -> (usize, usize) { let mut hwcap2 = HWCAP2.load(Relaxed); if hwcap == 0 || hwcap2 == 0 { - init_auxv(); - hwcap = HWCAP.load(Relaxed); - hwcap2 = HWCAP2.load(Relaxed); + #[cold] + fn compute_linux_hwcap() -> (usize, usize) { + init_auxv(); + (HWCAP.load(Relaxed), HWCAP2.load(Relaxed)) + } + (hwcap, hwcap2) = compute_linux_hwcap(); } (hwcap, hwcap2) @@ -75,8 +88,12 @@ pub(crate) fn linux_execfn() -> &'static CStr { let mut execfn = EXECFN.load(Relaxed); if execfn.is_null() { - init_auxv(); - execfn = EXECFN.load(Relaxed); + #[cold] + fn compute_linux_execfn() -> *mut c::c_char { + init_auxv(); + EXECFN.load(Relaxed) + } + execfn = compute_linux_execfn(); } // SAFETY: We assume the `AT_EXECFN` value provided by the kernel is a @@ -91,8 +108,12 @@ pub(crate) fn linux_secure() -> bool { // 0 means not initialized yet. if secure == 0 { - init_auxv(); - secure = SECURE.load(Relaxed); + #[cold] + fn compute_linux_secure() -> u8 { + init_auxv(); + SECURE.load(Relaxed) + } + secure = compute_linux_secure(); } // 0 means not present. Libc `getauxval(AT_SECURE)` would return 0. @@ -108,11 +129,13 @@ pub(crate) fn exe_phdrs() -> (*const c::c_void, usize, usize) { let mut phent = PHENT.load(Relaxed); let mut phnum = PHNUM.load(Relaxed); - if phdr.is_null() || phnum == 0 { - init_auxv(); - phdr = PHDR.load(Relaxed); - phent = PHENT.load(Relaxed); - phnum = PHNUM.load(Relaxed); + if phdr.is_null() || phent == 0 || phnum == 0 { + #[cold] + fn compute_exe_phdrs() -> (*mut Elf_Phdr, usize, usize) { + init_auxv(); + (PHDR.load(Relaxed), PHENT.load(Relaxed), PHNUM.load(Relaxed)) + } + (phdr, phent, phnum) = compute_exe_phdrs(); } (phdr.cast(), phent, phnum) @@ -125,8 +148,12 @@ pub(in super::super) fn sysinfo_ehdr() -> *const Elf_Ehdr { let mut ehdr = SYSINFO_EHDR.load(Relaxed); if ehdr.is_null() { - init_auxv(); - ehdr = SYSINFO_EHDR.load(Relaxed); + #[cold] + fn compute_sysinfo_ehdr() -> *mut Elf_Ehdr { + init_auxv(); + SYSINFO_EHDR.load(Relaxed) + } + ehdr = compute_sysinfo_ehdr(); } ehdr @@ -138,8 +165,12 @@ pub(crate) fn entry() -> usize { let mut entry = ENTRY.load(Relaxed); if entry == 0 { - init_auxv(); - entry = ENTRY.load(Relaxed); + #[cold] + fn compute_entry() -> usize { + init_auxv(); + ENTRY.load(Relaxed) + } + entry = compute_entry(); } entry @@ -151,13 +182,34 @@ pub(crate) fn random() -> *const [u8; 16] { let mut random = RANDOM.load(Relaxed); if random.is_null() { - init_auxv(); - random = RANDOM.load(Relaxed); + #[cold] + fn compute_random() -> *mut [u8; 16] { + init_auxv(); + RANDOM.load(Relaxed) + } + random = compute_random(); } random } +#[cfg(target_arch = "x86")] +#[inline] +pub(crate) fn vsyscall() -> *const c::c_void { + let mut vsyscall = VSYSCALL.load(Relaxed); + + if vsyscall.is_null() { + #[cold] + fn compute_vsyscall() -> *const c::c_void { + init_auxv(); + VSYSCALL.load(Relaxed) + } + vsyscall = compute_vsyscall(); + } + + vsyscall +} + static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0); static CLOCK_TICKS_PER_SECOND: AtomicUsize = AtomicUsize::new(0); static HWCAP: AtomicUsize = AtomicUsize::new(0); @@ -176,6 +228,8 @@ static PHNUM: AtomicUsize = AtomicUsize::new(0); static ENTRY: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "runtime")] static RANDOM: AtomicPtr<[u8; 16]> = AtomicPtr::new(null_mut()); +#[cfg(feature = "x86")] +static VSYSCALL: AtomicPtr = AtomicPtr::new(null_mut()); #[cfg(feature = "alloc")] fn pr_get_auxv() -> crate::io::Result> { @@ -315,6 +369,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator) -> Opti let mut egid = None; #[cfg(feature = "runtime")] let mut random = null_mut(); + #[cfg(target_arch = "x86")] + let mut vsyscall = null_mut(); for Elf_auxv_t { a_type, a_val } in aux_iter { match a_type as _ { @@ -353,6 +409,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator) -> Opti AT_ENTRY => entry = a_val as usize, #[cfg(feature = "runtime")] AT_RANDOM => random = check_raw_pointer::<[u8; 16]>(a_val as *mut _)?.as_ptr(), + #[cfg(target_arch = "x86")] + AT_SYSINFO => vsyscall = a_val.cast(), AT_NULL => break, _ => (), @@ -388,6 +446,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator) -> Opti ENTRY.store(entry, Relaxed); #[cfg(feature = "runtime")] RANDOM.store(random, Relaxed); + #[cfg(target_arch = "x86")] + VSYSCALL.store(vsyscall, Relaxed); Some(()) } diff --git a/src/backend/linux_raw/param/init.rs b/src/backend/linux_raw/param/init.rs index fe29e9c04..d92968880 100644 --- a/src/backend/linux_raw/param/init.rs +++ b/src/backend/linux_raw/param/init.rs @@ -14,6 +14,8 @@ use core::ptr::{null_mut, read, NonNull}; use core::sync::atomic::AtomicBool; use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use linux_raw_sys::elf::*; +#[cfg(target_arch = "x86")] +use linux_raw_sys::general::AT_SYSINFO; use linux_raw_sys::general::{ AT_CLKTCK, AT_EXECFN, AT_HWCAP, AT_HWCAP2, AT_NULL, AT_PAGESZ, AT_SYSINFO_EHDR, }; @@ -90,6 +92,12 @@ pub(crate) fn random() -> *const [u8; 16] { unsafe { RANDOM.load(Ordering::Relaxed) } } +#[cfg(target_arch = "x86")] +#[inline] +pub(crate) fn vsyscall() -> *const c_void { + unsafe { VSYSCALL.load(Ordering::Relaxed) } +} + static mut PAGE_SIZE: AtomicUsize = AtomicUsize::new(0); static mut CLOCK_TICKS_PER_SECOND: AtomicUsize = AtomicUsize::new(0); static mut HWCAP: AtomicUsize = AtomicUsize::new(0); @@ -111,6 +119,8 @@ static mut PHNUM: AtomicUsize = AtomicUsize::new(0); static mut ENTRY: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "runtime")] static mut RANDOM: AtomicPtr<[u8; 16]> = AtomicPtr::new(NonNull::dangling().as_ptr()); +#[cfg(target_arch = "x86")] +static mut VSYSCALL: AtomicPtr = AtomicPtr::new(NonNull::dangling().as_ptr()); /// When "use-explicitly-provided-auxv" is enabled, we export a function to be /// called during initialization, and passed a pointer to the original @@ -162,6 +172,8 @@ unsafe fn init_from_auxp(mut auxp: *const Elf_auxv_t) { AT_ENTRY => ENTRY.store(a_val as usize, Ordering::Relaxed), #[cfg(feature = "runtime")] AT_RANDOM => RANDOM.store(a_val.cast::<[u8; 16]>(), Ordering::Relaxed), + #[cfg(feature = "x86")] + AT_SYSINFO => VSYSCALL.store(a_val.cast::(), Ordering::Relaxed), AT_NULL => break, _ => (), diff --git a/src/backend/linux_raw/param/libc_auxv.rs b/src/backend/linux_raw/param/libc_auxv.rs index cfdd7a543..3e72d1e8d 100644 --- a/src/backend/linux_raw/param/libc_auxv.rs +++ b/src/backend/linux_raw/param/libc_auxv.rs @@ -11,6 +11,11 @@ use crate::ffi::CStr; #[cfg(not(feature = "runtime"))] use core::ptr::null; use linux_raw_sys::elf::*; +#[cfg(target_arch = "x86")] +use { + core::ffi::c_void, core::ptr::null_mut, core::sync::atomic::AtomicPtr, + core::sync::atomic::Ordering::Relaxed, +}; // `getauxval` wasn't supported in glibc until 2.16. Also this lets us use // `*mut` as the return type to preserve strict provenance. @@ -38,6 +43,8 @@ const AT_RANDOM: c::c_ulong = 25; const AT_HWCAP2: c::c_ulong = 26; const AT_SECURE: c::c_ulong = 23; const AT_EXECFN: c::c_ulong = 31; +#[cfg(target_arch = "x86")] +const AT_SYSINFO: c::c_ulong = 32; const AT_SYSINFO_EHDR: c::c_ulong = 33; // Declare `sysconf` ourselves so that we don't depend on all of libc just for @@ -72,6 +79,9 @@ fn test_abi() { const_assert_eq!(self::AT_ENTRY, ::libc::AT_ENTRY); #[cfg(feature = "runtime")] const_assert_eq!(self::AT_RANDOM, ::libc::AT_RANDOM); + // TODO: Upstream x86's `AT_SYSINFO` to libc. + #[cfg(target_arch = "x86")] + const_assert_eq!(self::AT_SYSINFO, ::linux_raw_sys::general::AT_SYSINFO); } #[cfg(feature = "param")] @@ -173,3 +183,25 @@ pub(crate) fn entry() -> usize { pub(crate) fn random() -> *const [u8; 16] { unsafe { getauxval(AT_RANDOM) as *const [u8; 16] } } + +#[cfg(target_arch = "x86")] +#[inline] +pub(crate) fn vsyscall() -> *const c_void { + // We call this for every system call, so memoize the value. + static VSYSCALL: AtomicPtr = AtomicPtr::new(null_mut()); + + let mut vsyscall = VSYSCALL.load(Relaxed); + + if vsyscall.is_null() { + #[cold] + fn compute_vsyscall() -> *mut c_void { + let vsyscall = unsafe { getauxval(AT_SYSINFO) } as *mut c_void; + VSYSCALL.store(vsyscall, Relaxed); + vsyscall + } + + vsyscall = compute_vsyscall(); + } + + vsyscall +} diff --git a/src/backend/linux_raw/vdso_wrappers.rs b/src/backend/linux_raw/vdso_wrappers.rs index 441738f8d..8256b21ae 100644 --- a/src/backend/linux_raw/vdso_wrappers.rs +++ b/src/backend/linux_raw/vdso_wrappers.rs @@ -9,11 +9,7 @@ //! functions. #![allow(unsafe_code)] -#[cfg(target_arch = "x86")] -use super::reg::{ArgReg, RetReg, SyscallNumber, A0, A1, A2, A3, A4, A5, R0}; use super::vdso; -#[cfg(target_arch = "x86")] -use core::arch::global_asm; #[cfg(feature = "process")] #[cfg(any( target_arch = "x86_64", @@ -135,130 +131,6 @@ pub(crate) fn sched_getcpu() -> usize { } } -#[cfg(target_arch = "x86")] -pub(super) mod x86_via_vdso { - use super::{transmute, ArgReg, Relaxed, RetReg, SyscallNumber, A0, A1, A2, A3, A4, A5, R0}; - use crate::backend::arch::asm; - - #[inline] - pub(in crate::backend) unsafe fn syscall0(nr: SyscallNumber<'_>) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall0(callee, nr) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall1<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall1(callee, nr, a0) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall1_noreturn<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - ) -> ! { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall1_noreturn(callee, nr, a0) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall2<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall2(callee, nr, a0, a1) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall3<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall3(callee, nr, a0, a1, a2) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall4<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - a3: ArgReg<'a, A3>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall4(callee, nr, a0, a1, a2, a3) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall5<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - a3: ArgReg<'a, A3>, - a4: ArgReg<'a, A4>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall5(callee, nr, a0, a1, a2, a3, a4) - } - - #[inline] - pub(in crate::backend) unsafe fn syscall6<'a>( - nr: SyscallNumber<'a>, - a0: ArgReg<'a, A0>, - a1: ArgReg<'a, A1>, - a2: ArgReg<'a, A2>, - a3: ArgReg<'a, A3>, - a4: ArgReg<'a, A4>, - a5: ArgReg<'a, A5>, - ) -> RetReg { - let callee = match transmute(super::SYSCALL.load(Relaxed)) { - Some(callee) => callee, - None => super::init_syscall(), - }; - asm::indirect_syscall6(callee, nr, a0, a1, a2, a3, a4, a5) - } - - // With the indirect call, it isn't meaningful to do a separate - // `_readonly` optimization. - #[allow(unused_imports)] - pub(in crate::backend) use { - syscall0 as syscall0_readonly, syscall1 as syscall1_readonly, - syscall2 as syscall2_readonly, syscall3 as syscall3_readonly, - syscall4 as syscall4_readonly, syscall5 as syscall5_readonly, - syscall6 as syscall6_readonly, - }; -} - #[cfg(feature = "time")] type ClockGettimeType = unsafe extern "C" fn(c::c_int, *mut Timespec) -> c::c_int; @@ -271,12 +143,6 @@ type ClockGettimeType = unsafe extern "C" fn(c::c_int, *mut Timespec) -> c::c_in ))] type GetcpuType = unsafe extern "C" fn(*mut u32, *mut u32, *mut c_void) -> c::c_int; -/// The underlying syscall functions are only called from asm, using the -/// special syscall calling convention to pass arguments and return values, -/// which the signature here doesn't reflect. -#[cfg(target_arch = "x86")] -pub(super) type SyscallType = unsafe extern "C" fn(); - /// Initialize `CLOCK_GETTIME` and return its value. #[cfg(feature = "time")] #[cold] @@ -303,16 +169,6 @@ fn init_getcpu() -> GetcpuType { unsafe { transmute(GETCPU.load(Relaxed)) } } -/// Initialize `SYSCALL` and return its value. -#[cfg(target_arch = "x86")] -#[cold] -fn init_syscall() -> SyscallType { - init(); - // SAFETY: Load the function address from static storage that we just - // initialized. - unsafe { transmute(SYSCALL.load(Relaxed)) } -} - /// `AtomicPtr` can't hold a `fn` pointer, so we use a `*` pointer to this /// placeholder type, and cast it as needed. struct Function; @@ -326,8 +182,6 @@ static mut CLOCK_GETTIME: AtomicPtr = AtomicPtr::new(null_mut()); target_arch = "powerpc64" ))] static mut GETCPU: AtomicPtr = AtomicPtr::new(null_mut()); -#[cfg(target_arch = "x86")] -static mut SYSCALL: AtomicPtr = AtomicPtr::new(null_mut()); #[cfg(feature = "time")] unsafe extern "C" fn rustix_clock_gettime_via_syscall( @@ -405,34 +259,6 @@ unsafe extern "C" fn rustix_getcpu_via_syscall( } } -#[cfg(target_arch = "x86")] -extern "C" { - /// A symbol pointing to an `int 0x80` instruction. This “function” is only - /// called from assembly, and only with the x86 syscall calling convention, - /// so its signature here is not its true signature. - /// - /// This extern block and the `global_asm!` below can be replaced with - /// `#[naked]` if it's stabilized. - fn rustix_int_0x80(); -} - -#[cfg(target_arch = "x86")] -global_asm!( - r#" - .section .text.rustix_int_0x80,"ax",@progbits - .p2align 4 - .weak rustix_int_0x80 - .hidden rustix_int_0x80 - .type rustix_int_0x80, @function -rustix_int_0x80: - .cfi_startproc - int 0x80 - ret - .cfi_endproc - .size rustix_int_0x80, .-rustix_int_0x80 -"# -); - fn minimal_init() { // SAFETY: Store default function addresses in static storage so that if we // end up making any system calls while we read the vDSO, they'll work. If @@ -468,18 +294,6 @@ fn minimal_init() { ) .ok(); } - - #[cfg(target_arch = "x86")] - { - SYSCALL - .compare_exchange( - null_mut(), - rustix_int_0x80 as *mut Function, - Relaxed, - Relaxed, - ) - .ok(); - } } } @@ -591,18 +405,5 @@ fn init() { } } } - - // On x86, also look up the vsyscall entry point. - #[cfg(target_arch = "x86")] - { - let ptr = vdso.sym(cstr!("LINUX_2.5"), cstr!("__kernel_vsyscall")); - assert!(!ptr.is_null()); - - // SAFETY: As above, store the computed function addresses in - // static storage. - unsafe { - SYSCALL.store(ptr.cast(), Relaxed); - } - } } } diff --git a/src/backend/linux_raw/x86_vsyscall.rs b/src/backend/linux_raw/x86_vsyscall.rs new file mode 100644 index 000000000..b013fdaa8 --- /dev/null +++ b/src/backend/linux_raw/x86_vsyscall.rs @@ -0,0 +1,103 @@ +//! Implement syscalls using the x86 vsyscall mechanism. +//! +//! # Safety +//! +//! Similar to syscalls.rs, this file performs raw system calls, and sometimes +//! passes them uninitialized memory buffers. +#![allow(unsafe_code)] + +use super::reg::{ArgReg, RetReg, SyscallNumber, A0, A1, A2, A3, A4, A5, R0}; +use crate::backend::arch::asm; +use core::mem::transmute; + +#[inline] +pub(super) unsafe fn syscall0(nr: SyscallNumber<'_>) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall0(callee, nr) +} + +#[inline] +pub(super) unsafe fn syscall1<'a>(nr: SyscallNumber<'a>, a0: ArgReg<'a, A0>) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall1(callee, nr, a0) +} + +#[inline] +pub(super) unsafe fn syscall1_noreturn<'a>(nr: SyscallNumber<'a>, a0: ArgReg<'a, A0>) -> ! { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall1_noreturn(callee, nr, a0) +} + +#[inline] +pub(super) unsafe fn syscall2<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall2(callee, nr, a0, a1) +} + +#[inline] +pub(super) unsafe fn syscall3<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall3(callee, nr, a0, a1, a2) +} + +#[inline] +pub(super) unsafe fn syscall4<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, + a3: ArgReg<'a, A3>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall4(callee, nr, a0, a1, a2, a3) +} + +#[inline] +pub(super) unsafe fn syscall5<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, + a3: ArgReg<'a, A3>, + a4: ArgReg<'a, A4>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall5(callee, nr, a0, a1, a2, a3, a4) +} + +#[inline] +pub(super) unsafe fn syscall6<'a>( + nr: SyscallNumber<'a>, + a0: ArgReg<'a, A0>, + a1: ArgReg<'a, A1>, + a2: ArgReg<'a, A2>, + a3: ArgReg<'a, A3>, + a4: ArgReg<'a, A4>, + a5: ArgReg<'a, A5>, +) -> RetReg { + let callee = transmute(super::param::auxv::vsyscall()); + asm::indirect_syscall6(callee, nr, a0, a1, a2, a3, a4, a5) +} + +// With the indirect call, it isn't meaningful to do a separate +// `_readonly` optimization. +#[allow(unused_imports)] +pub(super) use { + syscall0 as syscall0_readonly, syscall1 as syscall1_readonly, syscall2 as syscall2_readonly, + syscall3 as syscall3_readonly, syscall4 as syscall4_readonly, syscall5 as syscall5_readonly, + syscall6 as syscall6_readonly, +}; + +/// The underlying syscall functions are only called from asm, using the +/// special syscall calling convention to pass arguments and return values, +/// which the signature here doesn't reflect. +pub(super) type SyscallType = unsafe extern "C" fn();