On 32-bit x86, get the vsyscall address from the AUX vector

README.md

@@ -160,6 +160,14 @@ oldest Linux version supported by:
 The specifics of this policy may change in the future, but we intend it to
 always reflect “very old” Linux versions.
 
+## Minimum glibc Version
+
+On glibc platforms, rustix requires at least glibc 2.17. This is at most the
+oldest glibc version supported by:
+ - [any current Rust target]
+The specifics of this policy may change in the future, but we intend it to
+always reflect “very old” glibc versions.
+
 [MSRV]: #minimum-supported-rust-version-msrv
 [Rust 1.63]: https://blog.rust-lang.org/2022/08/11/Rust-1.63.0.html
 [any current Rust target]: https://doc.rust-lang.org/nightly/rustc/platform-support.html

src/backend/libc/param/auxv.rs

@@ -5,12 +5,9 @@ use crate::backend::c;
 ))]
 use crate::ffi::CStr;
 
-// `getauxval` wasn't supported in glibc until 2.16.
-#[cfg(any(
- all(target_os = "android", target_pointer_width = "64"),
- target_os = "linux",
-))]
-weak!(fn getauxval(c::c_ulong) -> *mut c::c_void);
+extern "C" {
+ fn getauxval(type_: c::c_ulong) -> *mut c::c_void;
+}
 
 #[inline]
 pub(crate) fn page_size() -> usize {
@@ -29,14 +26,10 @@ pub(crate) fn clock_ticks_per_second() -> u64 {
 ))]
 #[inline]
 pub(crate) fn linux_hwcap() -> (usize, usize) {
- if let Some(libc_getauxval) = getauxval.get() {
- unsafe {
- let hwcap = libc_getauxval(c::AT_HWCAP) as usize;
- let hwcap2 = libc_getauxval(c::AT_HWCAP2) as usize;
- (hwcap, hwcap2)
- }
- } else {
- (0, 0)
+ unsafe {
+ let hwcap = getauxval(c::AT_HWCAP) as usize;
+ let hwcap2 = getauxval(c::AT_HWCAP2) as usize;
+ (hwcap, hwcap2)
  }
 }
 
@@ -61,9 +54,5 @@ pub(crate) fn linux_minsigstksz() -> usize {
 ))]
 #[inline]
 pub(crate) fn linux_execfn() -> &'static CStr {
- if let Some(libc_getauxval) = getauxval.get() {
- unsafe { CStr::from_ptr(libc_getauxval(c::AT_EXECFN).cast()) }
- } else {
- cstr!("")
- }
+ unsafe { CStr::from_ptr(getauxval(c::AT_EXECFN).cast()) }
 }

src/backend/linux_raw/arch/mod.rs

@@ -51,11 +51,11 @@ pub(in crate::backend) mod asm;
 ))]
 pub(in crate::backend) use self::asm as choose;
 
-// On 32-bit x86, use vDSO wrappers for all syscalls. We could use the
-// architecture syscall instruction (`int 0x80`), but the vDSO kernel_vsyscall
+// On 32-bit x86, use the kernel_vsyscall mechanism for syscalls. We could use
+// the architecture syscall instruction (`int 0x80`), but the kernel_vsyscall
 // mechanism is much faster.
 #[cfg(target_arch = "x86")]
-pub(in crate::backend) use super::vdso_wrappers::x86_via_vdso as choose;
+pub(in crate::backend) use super::x86_vsyscall as choose;
 
 // This would be the code for always using `int 0x80` on 32-bit x86.
 //#[cfg(target_arch = "x86")]

src/backend/linux_raw/arch/x86.rs

@@ -15,7 +15,7 @@
 use crate::backend::reg::{
  ArgReg, FromAsm, RetReg, SyscallNumber, ToAsm, A0, A1, A2, A3, A4, A5, R0,
 };
-use crate::backend::vdso_wrappers::SyscallType;
+use crate::backend::x86_vsyscall::SyscallType;
 use core::arch::asm;
 
 #[inline]

src/backend/linux_raw/mod.rs

@@ -18,10 +18,12 @@
 mod arch;
 mod conv;
 mod reg;
-#[cfg(any(feature = "time", feature = "process", target_arch = "x86"))]
+#[cfg(any(feature = "time", feature = "process"))]
 mod vdso;
-#[cfg(any(feature = "time", feature = "process", target_arch = "x86"))]
+#[cfg(any(feature = "time", feature = "process"))]
 mod vdso_wrappers;
+#[cfg(target_arch = "x86")]
+mod x86_vsyscall;
 
 #[cfg(feature = "event")]
 pub(crate) mod event;

src/backend/linux_raw/param/auxv.rs

@@ -21,6 +21,8 @@ use core::sync::atomic::AtomicU8;
 use core::sync::atomic::Ordering::Relaxed;
 use core::sync::atomic::{AtomicPtr, AtomicUsize};
 use linux_raw_sys::elf::*;
+#[cfg(target_arch = "x86")]
+use linux_raw_sys::general::AT_SYSINFO;
 use linux_raw_sys::general::{
  AT_BASE, AT_CLKTCK, AT_EXECFN, AT_HWCAP, AT_HWCAP2, AT_MINSIGSTKSZ, AT_NULL, AT_PAGESZ,
  AT_SYSINFO_EHDR,
@@ -38,8 +40,12 @@ pub(crate) fn page_size() -> usize {
  let mut page_size = PAGE_SIZE.load(Relaxed);
 
  if page_size == 0 {
- init_auxv();
- page_size = PAGE_SIZE.load(Relaxed);
+ #[cold]
+ fn compute_page_size() -> usize {
+ init_auxv();
+ PAGE_SIZE.load(Relaxed)
+ }
+ page_size = compute_page_size();
  }
 
  page_size
@@ -51,8 +57,12 @@ pub(crate) fn clock_ticks_per_second() -> u64 {
  let mut ticks = CLOCK_TICKS_PER_SECOND.load(Relaxed);
 
  if ticks == 0 {
- init_auxv();
- ticks = CLOCK_TICKS_PER_SECOND.load(Relaxed);
+ #[cold]
+ fn compute_clock_ticks_per_second() -> usize {
+ init_auxv();
+ CLOCK_TICKS_PER_SECOND.load(Relaxed)
+ }
+ ticks = compute_clock_ticks_per_second();
  }
 
  ticks as u64
@@ -65,9 +75,12 @@ pub(crate) fn linux_hwcap() -> (usize, usize) {
  let mut hwcap2 = HWCAP2.load(Relaxed);
 
  if hwcap == 0 || hwcap2 == 0 {
- init_auxv();
- hwcap = HWCAP.load(Relaxed);
- hwcap2 = HWCAP2.load(Relaxed);
+ #[cold]
+ fn compute_linux_hwcap() -> (usize, usize) {
+ init_auxv();
+ (HWCAP.load(Relaxed), HWCAP2.load(Relaxed))
+ }
+ (hwcap, hwcap2) = compute_linux_hwcap();
  }
 
  (hwcap, hwcap2)
@@ -92,8 +105,12 @@ pub(crate) fn linux_execfn() -> &'static CStr {
  let mut execfn = EXECFN.load(Relaxed);
 
  if execfn.is_null() {
- init_auxv();
- execfn = EXECFN.load(Relaxed);
+ #[cold]
+ fn compute_linux_execfn() -> *mut c::c_char {
+ init_auxv();
+ EXECFN.load(Relaxed)
+ }
+ execfn = compute_linux_execfn();
  }
 
  // SAFETY: We assume the `AT_EXECFN` value provided by the kernel is a
@@ -108,8 +125,12 @@ pub(crate) fn linux_secure() -> bool {
 
  // 0 means not initialized yet.
  if secure == 0 {
- init_auxv();
- secure = SECURE.load(Relaxed);
+ #[cold]
+ fn compute_linux_secure() -> u8 {
+ init_auxv();
+ SECURE.load(Relaxed)
+ }
+ secure = compute_linux_secure();
  }
 
  // 0 means not present. Libc `getauxval(AT_SECURE)` would return 0.
@@ -125,11 +146,13 @@ pub(crate) fn exe_phdrs() -> (*const c::c_void, usize, usize) {
  let mut phent = PHENT.load(Relaxed);
  let mut phnum = PHNUM.load(Relaxed);
 
- if phdr.is_null() || phnum == 0 {
- init_auxv();
- phdr = PHDR.load(Relaxed);
- phent = PHENT.load(Relaxed);
- phnum = PHNUM.load(Relaxed);
+ if phdr.is_null() || phent == 0 || phnum == 0 {
+ #[cold]
+ fn compute_exe_phdrs() -> (*mut Elf_Phdr, usize, usize) {
+ init_auxv();
+ (PHDR.load(Relaxed), PHENT.load(Relaxed), PHNUM.load(Relaxed))
+ }
+ (phdr, phent, phnum) = compute_exe_phdrs();
  }
 
  (phdr.cast(), phent, phnum)
@@ -145,12 +168,16 @@ pub(in super::super) fn sysinfo_ehdr() -> *const Elf_Ehdr {
  let mut ehdr = SYSINFO_EHDR.load(Relaxed);
 
  if ehdr.is_null() {
- // Use `maybe_init_auxv` to to read the aux vectors if it can, but do
- // nothing if it can't. If it can't, then we'll get a null pointer
- // here, which our callers are prepared to deal with.
- maybe_init_auxv();
+ #[cold]
+ fn compute_sysinfo_ehdr() -> *mut Elf_Ehdr {
+ // Use `maybe_init_auxv` to to read the aux vectors if it can, but do
+ // nothing if it can't. If it can't, then we'll get a null pointer
+ // here, which our callers are prepared to deal with.
+ maybe_init_auxv();
+ SYSINFO_EHDR.load(Relaxed)
+ }
 
- ehdr = SYSINFO_EHDR.load(Relaxed);
+ ehdr = compute_sysinfo_ehdr();
  }
 
  ehdr
@@ -162,8 +189,12 @@ pub(crate) fn entry() -> usize {
  let mut entry = ENTRY.load(Relaxed);
 
  if entry == 0 {
- init_auxv();
- entry = ENTRY.load(Relaxed);
+ #[cold]
+ fn compute_entry() -> usize {
+ init_auxv();
+ ENTRY.load(Relaxed)
+ }
+ entry = compute_entry();
  }
 
  entry
@@ -175,13 +206,34 @@ pub(crate) fn random() -> *const [u8; 16] {
  let mut random = RANDOM.load(Relaxed);
 
  if random.is_null() {
- init_auxv();
- random = RANDOM.load(Relaxed);
+ #[cold]
+ fn compute_random() -> *mut [u8; 16] {
+ init_auxv();
+ RANDOM.load(Relaxed)
+ }
+ random = compute_random();
  }
 
  random
 }
 
+#[cfg(target_arch = "x86")]
+#[inline]
+pub(crate) fn vsyscall() -> *const c::c_void {
+ let mut vsyscall = VSYSCALL.load(Relaxed);
+
+ if vsyscall.is_null() {
+ #[cold]
+ fn compute_vsyscall() -> *mut c::c_void {
+ init_auxv();
+ VSYSCALL.load(Relaxed)
+ }
+ vsyscall = compute_vsyscall();
+ }
+
+ vsyscall
+}
+
 static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
 static CLOCK_TICKS_PER_SECOND: AtomicUsize = AtomicUsize::new(0);
 static HWCAP: AtomicUsize = AtomicUsize::new(0);
@@ -201,6 +253,8 @@ static PHNUM: AtomicUsize = AtomicUsize::new(0);
 static ENTRY: AtomicUsize = AtomicUsize::new(0);
 #[cfg(feature = "runtime")]
 static RANDOM: AtomicPtr<[u8; 16]> = AtomicPtr::new(null_mut());
+#[cfg(target_arch = "x86")]
+static VSYSCALL: AtomicPtr<c::c_void> = AtomicPtr::new(null_mut());
 
 const PR_GET_AUXV: c::c_int = 0x4155_5856;
 
@@ -389,6 +443,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator<Item = Elf_auxv_t>) -> Opti
  let mut egid = None;
  #[cfg(feature = "runtime")]
  let mut random = null_mut();
+ #[cfg(target_arch = "x86")]
+ let mut vsyscall = null_mut();
 
  for Elf_auxv_t { a_type, a_val } in aux_iter {
  match a_type as _ {
@@ -428,6 +484,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator<Item = Elf_auxv_t>) -> Opti
  AT_ENTRY => entry = a_val as usize,
  #[cfg(feature = "runtime")]
  AT_RANDOM => random = check_raw_pointer::<[u8; 16]>(a_val as *mut _)?.as_ptr(),
+ #[cfg(target_arch = "x86")]
+ AT_SYSINFO => vsyscall = a_val.cast(),
 
  AT_NULL => break,
  _ => (),
@@ -464,6 +522,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator<Item = Elf_auxv_t>) -> Opti
  ENTRY.store(entry, Relaxed);
  #[cfg(feature = "runtime")]
  RANDOM.store(random, Relaxed);
+ #[cfg(target_arch = "x86")]
+ VSYSCALL.store(vsyscall, Relaxed);
 
  Some(())
 }

src/backend/linux_raw/param/init.rs

@@ -14,6 +14,8 @@ use core::ptr::{null_mut, read, NonNull};
 use core::sync::atomic::AtomicBool;
 use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering};
 use linux_raw_sys::elf::*;
+#[cfg(target_arch = "x86")]
+use linux_raw_sys::general::AT_SYSINFO;
 use linux_raw_sys::general::{
  AT_CLKTCK, AT_EXECFN, AT_HWCAP, AT_HWCAP2, AT_MINSIGSTKSZ, AT_NULL, AT_PAGESZ, AT_SYSINFO_EHDR,
 };
@@ -96,6 +98,12 @@ pub(crate) fn random() -> *const [u8; 16] {
  unsafe { RANDOM.load(Ordering::Relaxed) }
 }
 
+#[cfg(target_arch = "x86")]
+#[inline]
+pub(crate) fn vsyscall() -> *const c_void {
+ unsafe { VSYSCALL.load(Ordering::Relaxed) }
+}
+
 static mut PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
 static mut CLOCK_TICKS_PER_SECOND: AtomicUsize = AtomicUsize::new(0);
 static mut HWCAP: AtomicUsize = AtomicUsize::new(0);
@@ -118,6 +126,8 @@ static mut PHNUM: AtomicUsize = AtomicUsize::new(0);
 static mut ENTRY: AtomicUsize = AtomicUsize::new(0);
 #[cfg(feature = "runtime")]
 static mut RANDOM: AtomicPtr<[u8; 16]> = AtomicPtr::new(NonNull::dangling().as_ptr());
+#[cfg(target_arch = "x86")]
+static mut VSYSCALL: AtomicPtr<c_void> = AtomicPtr::new(NonNull::dangling().as_ptr());
 
 /// When "use-explicitly-provided-auxv" is enabled, we export a function to be
 /// called during initialization, and passed a pointer to the original
@@ -170,6 +180,8 @@ unsafe fn init_from_auxp(mut auxp: *const Elf_auxv_t) {
  AT_ENTRY => ENTRY.store(a_val as usize, Ordering::Relaxed),
  #[cfg(feature = "runtime")]
  AT_RANDOM => RANDOM.store(a_val.cast::<[u8; 16]>(), Ordering::Relaxed),
+ #[cfg(feature = "x86")]
+ AT_SYSINFO => VSYSCALL.store(a_val.cast::<c_void>(), Ordering::Relaxed),
 
  AT_NULL => break,
  _ => (),