From 3966cf790e774a2834922e747edd9720f1f2288c Mon Sep 17 00:00:00 2001 From: Bob Haarman Date: Sat, 6 May 2023 23:57:20 +0000 Subject: [PATCH] support 64-bit file offsets on 32-bit glibc/Linux Adds large file support (64-bit file positions) to the existing Nix API when used with glibc on 32-bit Linux. On many platforms that support 64-bit file positions, this support is present in the standard I/O functions. On 32-bit Linux, there have historically been two APIs: the standard API, which limits file size to 2**31-1 bytes, and a separate API that suffixes function names with "64" and supports 64-bit file sizes. Other platforms do not provide this API. As a result, using the *64 API will make programs non-portable, whereas using the standard API will result in programs that cannot handle large files on 32-bit Linux, even when the system is actually capable of handling such files. To support large files with a consistent API across platforms, this change makes Nix functions call the 64-bit capable equivalents on Linux when using glibc. Other C libraries may not provide the *64 API, so we continue to call the standard functions on those. Broadly, the change consists of 5 parts: 1. Uses of libc::off_t have are replaced by i64. This provides a consistent API across platforms and allows 64-bit offsets to be used even when libc::off_t is 32-bit. This is similar to std::fs, which uses u64 for file positions. Nix uses i64 because off_t is a signed type. Into is used where appropriate so that existing code that uses libc::off_t will continue to work without changes. 2. A largefile_fn macro that is used to select the large-file-capable version of a function. E.g. largefile_fn![pwrite] is equivalent to libc::pwrite64 on glibc/Linux and plain libc::pwrite everywhere else. 3. A new require_largefile macro that is used to skip tests that require libc::off_t to be larger than 32 bits. 4. Changes to fallocate, ftruncate, lseek, mmap, open, openat, posix_fadvise, posix_fallocate, pread, preadv, pwrite, pwritev, sendfile, and truncate, making them support large files. 5. A set of test_*_largefile tests to verify that each of the previously mentioned functions now works with files whose size requires more than 32 bits to represent. A few functions are still limited to 32-bit file sizes after this change. This includes the aio* functions, the *stat* functions, and mkstemp(). Adding large file support to those requires a bit more work than simply calling a different function and is therefore left for later. --- src/fcntl.rs | 64 +++++++++++++++++++++++------------ src/macros.rs | 44 ++++++++++++++++++++++++ src/sys/mman.rs | 7 ++-- src/sys/sendfile.rs | 30 ++++++++--------- src/sys/uio.rs | 44 +++++++++++++++--------- src/unistd.rs | 36 +++++++++++++++----- test/common/mod.rs | 19 +++++++++++ test/sys/test_mman.rs | 43 ++++++++++++++++++++++- test/sys/test_sockopt.rs | 1 + test/sys/test_uio.rs | 46 +++++++++++++++++++------ test/test.rs | 2 -- test/test_fcntl.rs | 73 ++++++++++++++++++++++++++++++++++++++-- test/test_sendfile.rs | 25 ++++++++++++-- test/test_unistd.rs | 59 +++++++++++++++++++++++++++++--- 14 files changed, 409 insertions(+), 84 deletions(-) diff --git a/src/fcntl.rs b/src/fcntl.rs index a5dca8a539..cb89e84d06 100644 --- a/src/fcntl.rs +++ b/src/fcntl.rs @@ -2,7 +2,7 @@ use crate::errno::Errno; #[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] use core::slice; -use libc::{c_int, c_uint, size_t, ssize_t}; +use libc::{self, c_int, c_uint, size_t, ssize_t}; #[cfg(any( target_os = "netbsd", apple_targets, @@ -163,8 +163,12 @@ libc_bitflags!( all(target_os = "linux", not(target_env = "musl"), not(target_env = "ohos")), target_os = "redox"))] O_FSYNC; - /// Allow files whose sizes can't be represented in an `off_t` to be opened. + /// On 32-bit Linux, O_LARGEFILE allows the use of file + /// offsets greater than 32 bits. Nix accepts the flag for + /// compatibility, but always opens files in large file mode + /// even if it isn't specified. #[cfg(linux_android)] + #[cfg_attr(docsrs, doc(cfg(all())))] O_LARGEFILE; /// Do not update the file last access time during `read(2)`s. #[cfg(linux_android)] @@ -249,7 +253,7 @@ pub fn open( use std::os::fd::FromRawFd; let fd = path.with_nix_path(|cstr| unsafe { - libc::open(cstr.as_ptr(), oflag.bits(), mode.bits() as c_uint) + largefile_fn![open](cstr.as_ptr(), oflag.bits(), mode.bits() as c_uint) })?; Errno::result(fd)?; @@ -280,7 +284,11 @@ pub fn openat( use std::os::fd::FromRawFd; let fd = path.with_nix_path(|cstr| unsafe { - libc::openat(dirfd.as_fd().as_raw_fd(), cstr.as_ptr(), oflag.bits(), mode.bits() as c_uint) + largefile_fn![openat]( + dirfd.as_fd().as_raw_fd(), + cstr.as_ptr(), + oflag.bits(), + mode.bits() as c_uint) })?; Errno::result(fd)?; @@ -1303,15 +1311,18 @@ feature! { /// file referred to by fd. #[cfg(target_os = "linux")] #[cfg(feature = "fs")] -pub fn fallocate( +pub fn fallocate>( fd: Fd, mode: FallocateFlags, - offset: libc::off_t, - len: libc::off_t, + offset: Off, + len: Off, ) -> Result<()> { use std::os::fd::AsRawFd; - let res = unsafe { libc::fallocate(fd.as_fd().as_raw_fd(), mode.bits(), offset, len) }; + let res = unsafe { + largefile_fn![fallocate]( + fd.as_fd().as_raw_fd(), mode.bits(), offset.into(), len.into()) + }; Errno::result(res).map(drop) } @@ -1319,7 +1330,7 @@ pub fn fallocate( /// the file offset, and the second is the length of the region. #[cfg(any(target_os = "freebsd"))] #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct SpacectlRange(pub libc::off_t, pub libc::off_t); +pub struct SpacectlRange(pub off_t, pub off_t); #[cfg(any(target_os = "freebsd"))] impl SpacectlRange { @@ -1334,13 +1345,13 @@ impl SpacectlRange { /// Remaining length of the range #[inline] - pub fn len(&self) -> libc::off_t { + pub fn len(&self) -> off_t { self.1 } /// Next file offset to operate on #[inline] - pub fn offset(&self) -> libc::off_t { + pub fn offset(&self) -> off_t { self.0 } } @@ -1439,8 +1450,8 @@ pub fn fspacectl(fd: Fd, range: SpacectlRange) -> Result< #[inline] // Delays codegen, preventing linker errors with dylibs and --no-allow-shlib-undefined pub fn fspacectl_all( fd: Fd, - offset: libc::off_t, - len: libc::off_t, + offset: off_t, + len: off_t, ) -> Result<()> { use std::os::fd::AsRawFd; @@ -1505,15 +1516,21 @@ mod posix_fadvise { /// /// # See Also /// * [`posix_fadvise`](https://pubs.opengroup.org/onlinepubs/9699919799/functions/posix_fadvise.html) - pub fn posix_fadvise( + pub fn posix_fadvise>( fd: Fd, - offset: libc::off_t, - len: libc::off_t, + offset: Off, + len: Off, advice: PosixFadviseAdvice, ) -> Result<()> { use std::os::fd::AsRawFd; - let res = unsafe { libc::posix_fadvise(fd.as_fd().as_raw_fd(), offset, len, advice as libc::c_int) }; + let res = unsafe { + largefile_fn![posix_fadvise]( + fd.as_fd().as_raw_fd(), + offset.into(), + len.into(), + advice as libc::c_int) + }; if res == 0 { Ok(()) @@ -1535,14 +1552,19 @@ mod posix_fadvise { target_os = "fuchsia", target_os = "wasi", ))] -pub fn posix_fallocate( +pub fn posix_fallocate>( fd: Fd, - offset: libc::off_t, - len: libc::off_t, + offset: Off, + len: Off, ) -> Result<()> { use std::os::fd::AsRawFd; - let res = unsafe { libc::posix_fallocate(fd.as_fd().as_raw_fd(), offset, len) }; + let res = unsafe { + largefile_fn![posix_fallocate]( + fd.as_fd().as_raw_fd(), + offset.into(), + len.into()) + }; match Errno::result(res) { Err(err) => Err(err), Ok(0) => Ok(()), diff --git a/src/macros.rs b/src/macros.rs index 3010a1a053..4cfe365df8 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -1,3 +1,5 @@ +use cfg_if::cfg_if; + // Thanks to Tokio for this macro macro_rules! feature { ( @@ -329,3 +331,45 @@ macro_rules! libc_enum { } }; } + +cfg_if! { + if #[cfg(all(target_os = "linux", target_env = "gnu"))] { + /// Function variant that supports large file positions. + /// + /// On some platforms, the standard I/O functions support a limited + /// range of file positions, and there is an alternate set of + /// functions that support larger file positions. This macro takes + /// the identifier of a standard I/O function and returns the + /// identifier of the corresponding I/O function with large file + /// support. + #[allow(unused_macro_rules)] + macro_rules! largefile_fn { + [fallocate] => (libc::fallocate64); + [ftruncate] => (libc::ftruncate64); + [lseek] => (libc::lseek64); + [mmap] => (libc::mmap64); + [open] => (libc::open64); + [openat] => (libc::openat64); + [posix_fadvise] => (libc::posix_fadvise64); + [posix_fallocate] => (libc::posix_fallocate64); + [pread] => (libc::pread64); + [preadv] => (libc::preadv64); + [pwrite] => (libc::pwrite64); + [pwritev] => (libc::pwritev64); + [sendfile] => (libc::sendfile64); + [truncate] => (libc::truncate64); + } + } else { + /// Function variant that supports large file positions. + /// + /// On some platforms, the standard I/O functions support a limited + /// range of file positions, and there is an alternate set of + /// functions that support larger file positions. This macro takes + /// the identifier of a standard I/O function and returns the + /// identifier of the corresponding I/O function with large file + /// support. + macro_rules! largefile_fn { + [$id:ident] => (libc::$id); + } + } +} diff --git a/src/sys/mman.rs b/src/sys/mman.rs index a64f14f588..90d8da766d 100644 --- a/src/sys/mman.rs +++ b/src/sys/mman.rs @@ -7,7 +7,7 @@ use crate::Result; #[cfg(not(target_os = "android"))] #[cfg(feature = "fs")] use crate::{fcntl::OFlag, sys::stat::Mode}; -use libc::{self, c_int, c_void, off_t, size_t}; +use libc::{self, c_int, c_void, size_t}; use std::ptr::NonNull; use std::{ num::NonZeroUsize, @@ -398,13 +398,14 @@ pub unsafe fn mmap( prot: ProtFlags, flags: MapFlags, f: F, - offset: off_t, + offset: i64, ) -> Result> { let ptr = addr.map_or(std::ptr::null_mut(), |a| a.get() as *mut c_void); let fd = f.as_fd().as_raw_fd(); let ret = unsafe { - libc::mmap(ptr, length.into(), prot.bits(), flags.bits(), fd, offset) + largefile_fn![mmap]( + ptr, length.into(), prot.bits(), flags.bits(), fd, offset) }; if ret == libc::MAP_FAILED { diff --git a/src/sys/sendfile.rs b/src/sys/sendfile.rs index d7452edd7c..52c9045932 100644 --- a/src/sys/sendfile.rs +++ b/src/sys/sendfile.rs @@ -4,7 +4,7 @@ use cfg_if::cfg_if; use std::os::unix::io::{AsFd, AsRawFd}; use std::ptr; -use libc::{self, off_t}; +use libc; use crate::errno::Errno; use crate::Result; @@ -26,14 +26,14 @@ use crate::Result; pub fn sendfile( out_fd: F1, in_fd: F2, - offset: Option<&mut off_t>, + offset: Option<&mut i64>, count: usize, ) -> Result { let offset = offset .map(|offset| offset as *mut _) .unwrap_or(ptr::null_mut()); let ret = unsafe { - libc::sendfile( + largefile_fn![sendfile]( out_fd.as_fd().as_raw_fd(), in_fd.as_fd().as_raw_fd(), offset, @@ -208,19 +208,19 @@ cfg_if! { pub fn sendfile( in_fd: F1, out_sock: F2, - offset: off_t, + offset: i64, count: Option, headers: Option<&[&[u8]]>, trailers: Option<&[&[u8]]>, flags: SfFlags, readahead: u16 - ) -> (Result<()>, off_t) { + ) -> (Result<()>, i64) { // Readahead goes in upper 16 bits // Flags goes in lower 16 bits // see `man 2 sendfile` let ra32 = u32::from(readahead); let flags: u32 = (ra32 << 16) | (flags.bits() as u32); - let mut bytes_sent: off_t = 0; + let mut bytes_sent: i64 = 0; let hdtr = headers.or(trailers).map(|_| SendfileHeaderTrailer::new(headers, trailers)); let hdtr_ptr = hdtr.as_ref().map_or(ptr::null(), |s| &s.raw as *const libc::sf_hdtr); let return_code = unsafe { @@ -229,7 +229,7 @@ cfg_if! { offset, count.unwrap_or(0), hdtr_ptr as *mut libc::sf_hdtr, - &mut bytes_sent as *mut off_t, + &mut bytes_sent as *mut i64, flags as c_int) }; (Errno::result(return_code).and(Ok(())), bytes_sent) @@ -258,12 +258,12 @@ cfg_if! { pub fn sendfile( in_fd: F1, out_sock: F2, - offset: off_t, + offset: i64, count: Option, headers: Option<&[&[u8]]>, trailers: Option<&[&[u8]]>, - ) -> (Result<()>, off_t) { - let mut bytes_sent: off_t = 0; + ) -> (Result<()>, i64) { + let mut bytes_sent: i64 = 0; let hdtr = headers.or(trailers).map(|_| SendfileHeaderTrailer::new(headers, trailers)); let hdtr_ptr = hdtr.as_ref().map_or(ptr::null(), |s| &s.raw as *const libc::sf_hdtr); let return_code = unsafe { @@ -272,7 +272,7 @@ cfg_if! { offset, count.unwrap_or(0), hdtr_ptr as *mut libc::sf_hdtr, - &mut bytes_sent as *mut off_t, + &mut bytes_sent as *mut i64, 0) }; (Errno::result(return_code).and(Ok(())), bytes_sent) @@ -304,11 +304,11 @@ cfg_if! { pub fn sendfile( in_fd: F1, out_sock: F2, - offset: off_t, - count: Option, + offset: i64, + count: Option, headers: Option<&[&[u8]]>, trailers: Option<&[&[u8]]> - ) -> (Result<()>, off_t) { + ) -> (Result<()>, i64) { let mut len = count.unwrap_or(0); let hdtr = headers.or(trailers).map(|_| SendfileHeaderTrailer::new(headers, trailers)); let hdtr_ptr = hdtr.as_ref().map_or(ptr::null(), |s| &s.raw as *const libc::sf_hdtr); @@ -316,7 +316,7 @@ cfg_if! { libc::sendfile(in_fd.as_fd().as_raw_fd(), out_sock.as_fd().as_raw_fd(), offset, - &mut len as *mut off_t, + &mut len as *mut i64, hdtr_ptr as *mut libc::sf_hdtr, 0) }; diff --git a/src/sys/uio.rs b/src/sys/uio.rs index cdf380dd11..b3c2b95742 100644 --- a/src/sys/uio.rs +++ b/src/sys/uio.rs @@ -2,7 +2,7 @@ use crate::errno::Errno; use crate::Result; -use libc::{self, c_int, off_t, size_t}; +use libc::{self, c_int, size_t}; use std::io::{IoSlice, IoSliceMut}; use std::os::unix::io::{AsFd, AsRawFd}; @@ -54,17 +54,20 @@ pub fn readv(fd: Fd, iov: &mut [IoSliceMut<'_>]) -> Result { /// /// See also: [`writev`](fn.writev.html) and [`pwrite`](fn.pwrite.html) #[cfg(not(any(target_os = "redox", target_os = "haiku", target_os = "solaris")))] -pub fn pwritev( +pub fn pwritev>( fd: Fd, iov: &[IoSlice<'_>], - offset: off_t, + offset: Off, ) -> Result { + let offset = offset.into(); + // uclibc uses 64-bit offsets for pwritev even if it otherwise uses + // 32-bit file offsets. #[cfg(target_env = "uclibc")] - let offset = offset as libc::off64_t; // uclibc doesn't use off_t + let offset = offset as libc::off64_t; // SAFETY: same as in writev() let res = unsafe { - libc::pwritev( + largefile_fn![pwritev]( fd.as_fd().as_raw_fd(), iov.as_ptr().cast(), iov.len() as c_int, @@ -86,17 +89,20 @@ pub fn pwritev( // Clippy doesn't know that we need to pass iov mutably only because the // mutation happens after converting iov to a pointer #[allow(clippy::needless_pass_by_ref_mut)] -pub fn preadv( +pub fn preadv>( fd: Fd, iov: &mut [IoSliceMut<'_>], - offset: off_t, + offset: Off, ) -> Result { + let offset = offset.into(); + // uclibc uses 64-bit offsets for preadv even if it otherwise uses + // 32-bit file offsets. #[cfg(target_env = "uclibc")] - let offset = offset as libc::off64_t; // uclibc doesn't use off_t + let offset = offset as libc::off64_t; // SAFETY: same as in readv() let res = unsafe { - libc::preadv( + largefile_fn![preadv]( fd.as_fd().as_raw_fd(), iov.as_ptr().cast(), iov.len() as c_int, @@ -111,13 +117,17 @@ pub fn preadv( /// /// See also [pwrite(2)](https://pubs.opengroup.org/onlinepubs/9699919799/functions/pwrite.html) // TODO: move to unistd -pub fn pwrite(fd: Fd, buf: &[u8], offset: off_t) -> Result { +pub fn pwrite>( + fd: Fd, + buf: &[u8], + offset: Off, +) -> Result { let res = unsafe { - libc::pwrite( + largefile_fn![pwrite]( fd.as_fd().as_raw_fd(), buf.as_ptr().cast(), buf.len() as size_t, - offset, + offset.into(), ) }; @@ -128,13 +138,17 @@ pub fn pwrite(fd: Fd, buf: &[u8], offset: off_t) -> Result { /// /// See also [pread(2)](https://pubs.opengroup.org/onlinepubs/9699919799/functions/pread.html) // TODO: move to unistd -pub fn pread(fd: Fd, buf: &mut [u8], offset: off_t) -> Result { +pub fn pread>( + fd: Fd, + buf: &mut [u8], + offset: Off, +) -> Result { let res = unsafe { - libc::pread( + largefile_fn![pread]( fd.as_fd().as_raw_fd(), buf.as_mut_ptr().cast(), buf.len() as size_t, - offset, + offset.into(), ) }; diff --git a/src/unistd.rs b/src/unistd.rs index 382333af3e..49c0a5fa3a 100644 --- a/src/unistd.rs +++ b/src/unistd.rs @@ -23,7 +23,9 @@ use crate::sys::stat::FileFlag; use crate::{Error, NixPath, Result}; #[cfg(not(target_os = "redox"))] use cfg_if::cfg_if; -use libc::{c_char, c_int, c_long, c_uint, gid_t, off_t, pid_t, size_t, uid_t}; +use libc::{ + self, c_char, c_int, c_long, c_uint, gid_t, pid_t, size_t, uid_t, +}; use std::convert::Infallible; #[cfg(not(target_os = "redox"))] use std::ffi::CString; @@ -1447,12 +1449,20 @@ pub enum Whence { /// Move the read/write file offset. /// /// See also [lseek(2)](https://pubs.opengroup.org/onlinepubs/9699919799/functions/lseek.html) -pub fn lseek(fd: Fd, offset: off_t, whence: Whence) -> Result { +pub fn lseek>( + fd: Fd, + offset: Off, + whence: Whence, +) -> Result { use std::os::fd::AsRawFd; - let res = unsafe { libc::lseek(fd.as_fd().as_raw_fd(), offset, whence as i32) }; + let res = unsafe { + largefile_fn![lseek](fd.as_fd().as_raw_fd(), offset.into(), whence as i32) + }; - Errno::result(res).map(|r| r as off_t) + // r may or may not already be i64; suppress unnecessary_cast diagnostic. + #[allow(clippy::unnecessary_cast)] + Errno::result(res).map(|r| r as i64) } /// Move the read/write file offset. @@ -1533,9 +1543,14 @@ pub fn pipe2(flags: OFlag) -> Result<(std::os::fd::OwnedFd, std::os::fd::OwnedFd /// See also /// [truncate(2)](https://pubs.opengroup.org/onlinepubs/9699919799/functions/truncate.html) #[cfg(not(any(target_os = "redox", target_os = "fuchsia")))] -pub fn truncate(path: &P, len: off_t) -> Result<()> { +pub fn truncate>( + path: &P, + len: Off, +) -> Result<()> { let res = path - .with_nix_path(|cstr| unsafe { libc::truncate(cstr.as_ptr(), len) })?; + .with_nix_path(|cstr| unsafe { + largefile_fn![truncate](cstr.as_ptr(), len.into()) + })?; Errno::result(res).map(drop) } @@ -1544,10 +1559,15 @@ pub fn truncate(path: &P, len: off_t) -> Result<()> { /// /// See also /// [ftruncate(2)](https://pubs.opengroup.org/onlinepubs/9699919799/functions/ftruncate.html) -pub fn ftruncate(fd: Fd, len: off_t) -> Result<()> { +pub fn ftruncate>( + fd: Fd, + len: Off, +) -> Result<()> { use std::os::fd::AsRawFd; - Errno::result(unsafe { libc::ftruncate(fd.as_fd().as_raw_fd(), len) }).map(drop) + Errno::result(unsafe { + largefile_fn![ftruncate](fd.as_fd().as_raw_fd(), len.into()) + }).map(drop) } /// Determines if the file descriptor refers to a valid terminal type device. diff --git a/test/common/mod.rs b/test/common/mod.rs index ab0e746367..933025df1e 100644 --- a/test/common/mod.rs +++ b/test/common/mod.rs @@ -32,6 +32,25 @@ cfg_if! { } } +/// Skip the test if we cannot handle offsets larger than 32 bits. +/// +/// This is generally the case if libc::off_t::MAX is 1 << 32 or less. +/// However, glibc on Linux provides variants of the standard I/O +/// functions that do support 64-bit offsets (e.g. lseek64 instead of lseek). +#[macro_export] +macro_rules! require_largefile { + ($name:expr) => { + #[cfg(not(all(target_os = "linux", target_env = "gnu")))] + if (libc::off_t::MAX >> 31) <= 1 { + $crate::skip!( + "{} requires file offsets \ + larger than 32 bits. Skipping test.", + $name + ); + } + }; +} + /// Skip the test if we don't have the ability to mount file systems. #[cfg(target_os = "freebsd")] #[macro_export] diff --git a/test/sys/test_mman.rs b/test/sys/test_mman.rs index 3689f642be..f80da43bf7 100644 --- a/test/sys/test_mman.rs +++ b/test/sys/test_mman.rs @@ -1,7 +1,11 @@ #![allow(clippy::redundant_slicing)] -use nix::sys::mman::{mmap_anonymous, MapFlags, ProtFlags}; +use nix::sys::mman::{mmap, mmap_anonymous, MapFlags, ProtFlags}; +use nix::unistd::{lseek, sysconf, write, SysconfVar, Whence}; use std::num::NonZeroUsize; +use tempfile::tempfile; + +use crate::{require_largefile, skip}; #[test] fn test_mmap_anonymous() { @@ -20,6 +24,43 @@ fn test_mmap_anonymous() { } } +#[test] +fn test_mmap_largefile() { + require_largefile!("test_mmap_largefile"); + + // Calculate an offset that requires more than 32 bits to represent + // and is a multiple of the page size. + #[allow(clippy::unnecessary_cast)] // Some platforms need the cast. + let pagesize = match sysconf(SysconfVar::PAGE_SIZE) { + Ok(Some(x)) => x, + _ => { + skip!("test_mmap_largefile: Could not determine page size. Skipping test."); + } + } as i64; + let pages_in_32bits = 1_0000_0000i64 / pagesize; + // Silence Clippy warning - offset's type varies by platform. + #[allow(clippy::useless_conversion)] + let offset = ((pages_in_32bits + 2) * pagesize).try_into().unwrap(); + // Create a file, seek to offset, and write some bytes. + let file = tempfile().unwrap(); + lseek(&file, offset, Whence::SeekSet).unwrap(); + write(&file, b"xyzzy").unwrap(); + // Check that we can map it and see the bytes. + let slice = unsafe { + let res = mmap( + None, + NonZeroUsize::new(5).unwrap(), + ProtFlags::PROT_READ, + MapFlags::MAP_PRIVATE, + &file, + offset, + ); + assert!(res.is_ok()); + std::slice::from_raw_parts(res.unwrap().as_ptr() as *const u8, 5) + }; + assert_eq!(slice, b"xyzzy"); +} + #[test] #[cfg(any(target_os = "linux", target_os = "netbsd"))] fn test_mremap_grow() { diff --git a/test/sys/test_sockopt.rs b/test/sys/test_sockopt.rs index fd055ef3dd..41919df4f8 100644 --- a/test/sys/test_sockopt.rs +++ b/test/sys/test_sockopt.rs @@ -1,5 +1,6 @@ #[cfg(linux_android)] use crate::*; +use cfg_if::cfg_if; use nix::sys::socket::{ getsockopt, setsockopt, socket, sockopt, AddressFamily, SockFlag, SockProtocol, SockType, diff --git a/test/sys/test_uio.rs b/test/sys/test_uio.rs index f7d648403e..3ca10084f6 100644 --- a/test/sys/test_uio.rs +++ b/test/sys/test_uio.rs @@ -1,13 +1,18 @@ +#[cfg(not(target_os = "redox"))] +use crate::require_largefile; +#[cfg(not(target_os = "redox"))] use nix::sys::uio::*; use nix::unistd::*; use rand::distributions::Alphanumeric; use rand::{thread_rng, Rng}; use std::fs::OpenOptions; -use std::io::IoSlice; +use std::io::{IoSlice, Write}; +#[cfg(not(target_os = "redox"))] +use std::os::unix::fs::FileExt; use std::{cmp, iter}; #[cfg(not(target_os = "redox"))] -use std::io::IoSliceMut; +use std::io::{IoSliceMut, Read, Seek, SeekFrom}; use tempfile::tempdir; #[cfg(not(target_os = "redox"))] @@ -103,8 +108,6 @@ fn test_readv() { #[test] #[cfg(not(target_os = "redox"))] fn test_pwrite() { - use std::io::Read; - let mut file = tempfile().unwrap(); let buf = [1u8; 8]; assert_eq!(Ok(8), pwrite(&file, &buf, 8)); @@ -116,9 +119,23 @@ fn test_pwrite() { } #[test] -fn test_pread() { - use std::io::Write; +#[cfg(not(target_os = "redox"))] +fn test_pwrite_largefile() { + require_largefile!("test_pwrite_largefile"); + + let mut file = tempfile().unwrap(); + let buf = [255u8; 1]; + let pos = 0x1_0000_0002i64; + assert_eq!(pwrite(&file, &buf, pos), Ok(1)); + assert_eq!(file.metadata().unwrap().len(), 0x1_0000_0003); + file.seek(SeekFrom::End(-1)).unwrap(); + let mut file_content = Vec::new(); + file.read_to_end(&mut file_content).unwrap(); + assert_eq!(file_content, [255u8; 1]); +} +#[test] +fn test_pread() { let tempdir = tempdir().unwrap(); let path = tempdir.path().join("pread_test_file"); @@ -144,9 +161,20 @@ fn test_pread() { target_os = "haiku", target_os = "solaris" )))] -fn test_pwritev() { - use std::io::Read; +fn test_pread_largefile() { + require_largefile!("test_pread_largefile"); + + let file = tempfile().unwrap(); + file.write_all_at(b"The text", 0x1_0000_0005).unwrap(); + let mut buf = [0u8; 4]; + let pos = 0x1_0000_0009i64; + assert_eq!(pread(&file, &mut buf, pos), Ok(4)); + assert_eq!(&buf, b"text"); +} +#[test] +#[cfg(not(any(target_os = "redox", target_os = "haiku")))] +fn test_pwritev() { let to_write: Vec = (0..128).collect(); let expected: Vec = [vec![0; 100], to_write.clone()].concat(); @@ -184,8 +212,6 @@ fn test_pwritev() { target_os = "solaris" )))] fn test_preadv() { - use std::io::Write; - let to_write: Vec = (0..200).collect(); let expected: Vec = (100..200).collect(); diff --git a/test/test.rs b/test/test.rs index 3427b2c3db..c234613333 100644 --- a/test/test.rs +++ b/test/test.rs @@ -1,5 +1,3 @@ -#[macro_use] -extern crate cfg_if; #[cfg_attr(not(any(target_os = "redox", target_os = "haiku")), macro_use)] extern crate nix; diff --git a/test/test_fcntl.rs b/test/test_fcntl.rs index d49f49670f..26c70c094b 100644 --- a/test/test_fcntl.rs +++ b/test/test_fcntl.rs @@ -1,4 +1,6 @@ #[cfg(not(target_os = "redox"))] +use crate::require_largefile; +#[cfg(not(target_os = "redox"))] use nix::errno::*; #[cfg(not(target_os = "redox"))] use nix::fcntl::{open, readlink, OFlag}; @@ -21,7 +23,7 @@ use nix::fcntl::{renameat2, RenameFlags}; #[cfg(not(target_os = "redox"))] use nix::sys::stat::Mode; #[cfg(not(target_os = "redox"))] -use nix::unistd::read; +use nix::unistd::{close, lseek, read, write, Whence}; #[cfg(not(target_os = "redox"))] use std::fs::File; #[cfg(not(target_os = "redox"))] @@ -29,7 +31,29 @@ use std::io::prelude::*; #[cfg(not(target_os = "redox"))] use std::os::unix::fs; #[cfg(not(target_os = "redox"))] -use tempfile::NamedTempFile; +use tempfile::{self, tempdir, NamedTempFile}; + +#[test] +#[cfg(not(target_os = "redox"))] +fn test_open_largefile() { + // Checks that files opened with open can grow beyond 32-bit file size. + // On some platforms, this requires opening the file with an alternative + // open() function or a flag. + require_largefile!("test_open_largefile"); + + let tempdir = tempdir().unwrap(); + let path = tempdir.path().join("file"); + File::create(&path).unwrap(); + let fd = open(&path, OFlag::O_WRONLY, Mode::empty()).unwrap(); + // Seek to a position that requires more than 32 bits to represent. + let pos = (0x1_0000_0004u64).try_into().unwrap(); + let result = lseek(&fd, pos, Whence::SeekSet); + assert_eq!(result, Ok(pos)); + // Write a byte there, close the file, and check its size. + write(&fd, b"x").unwrap(); + unsafe { close(fd).unwrap(); } + assert_eq!(std::fs::metadata(&path).unwrap().len(), 0x1_0000_0005); +} #[test] #[cfg(not(target_os = "redox"))] @@ -112,6 +136,29 @@ fn test_openat2_forbidden() { assert_eq!(res.unwrap_err(), Errno::EXDEV); } +#[test] +#[cfg(not(target_os = "redox"))] +// QEMU does not handle openat well enough to satisfy this test +// https://gitlab.com/qemu-project/qemu/-/issues/829 +#[cfg_attr(qemu, ignore)] +fn test_openat_largefile() { + require_largefile!("test_openat_largefile"); + + let tempdir = tempdir().unwrap(); + let dirfd = open(tempdir.path(), OFlag::empty(), Mode::empty()).unwrap(); + let path = tempdir.path().join("file"); + File::create(&path).unwrap(); + let fd = openat(dirfd, "file", OFlag::O_WRONLY, Mode::empty()).unwrap(); + // Seek to a position that requires more than 32 bits to represent. + let pos = (0x1_0000_0004u64).try_into().unwrap(); + let result = lseek(&fd, pos, Whence::SeekSet); + assert_eq!(result, Ok(pos)); + // Write a byte there, close the file, and check its size. + write(&fd, b"x").unwrap(); + unsafe { close(fd).unwrap(); } + assert_eq!(std::fs::metadata(&path).unwrap().len(), 0x1_0000_0005); +} + #[test] #[cfg(not(target_os = "redox"))] fn test_renameat() { @@ -497,8 +544,11 @@ mod test_posix_fadvise { use nix::errno::Errno; use nix::fcntl::*; use nix::unistd::pipe; + use std::io::{Seek, Write}; use tempfile::NamedTempFile; + use crate::require_largefile; + #[test] fn test_success() { let tmp = NamedTempFile::new().unwrap(); @@ -513,6 +563,23 @@ mod test_posix_fadvise { posix_fadvise(&rd, 0, 100, PosixFadviseAdvice::POSIX_FADV_WILLNEED); assert_eq!(res, Err(Errno::ESPIPE)); } + + #[test] + fn test_posix_fadvise_largefile() { + require_largefile!("test_posix_fadvise_largefile"); + + let mut tmp = tempfile::tempfile().unwrap(); + tmp.seek(std::io::SeekFrom::Start(0xfffffffc)).unwrap(); + tmp.write_all(b"forty-two").unwrap(); + let pos = 0x1_0000_0004i64; + assert!(posix_fadvise( + &tmp, + 0, + pos, + PosixFadviseAdvice::POSIX_FADV_DONTNEED, + ) + .is_ok()); + } } #[cfg(any( @@ -534,7 +601,7 @@ mod test_posix_fallocate { fn success() { const LEN: usize = 100; let mut tmp = NamedTempFile::new().unwrap(); - let res = posix_fallocate(&tmp, 0, LEN as libc::off_t); + let res = posix_fallocate(&tmp, 0, LEN as i64); match res { Ok(_) => { let mut data = [1u8; LEN]; diff --git a/test/test_sendfile.rs b/test/test_sendfile.rs index 3ca8092fc6..73146fec1f 100644 --- a/test/test_sendfile.rs +++ b/test/test_sendfile.rs @@ -1,6 +1,9 @@ +#[cfg(any(target_os = "android", target_os = "linux"))] +use crate::require_largefile; +use cfg_if::cfg_if; use std::io::prelude::*; +#[cfg(any(target_os = "android", target_os = "linux"))] -use libc::off_t; use nix::sys::sendfile::*; use tempfile::tempfile; @@ -21,7 +24,7 @@ fn test_sendfile_linux() { tmp.write_all(CONTENTS).unwrap(); let (rd, wr) = pipe().unwrap(); - let mut offset: off_t = 5; + let mut offset: i64 = 5; let res = sendfile(&wr, &tmp, Some(&mut offset), 2).unwrap(); assert_eq!(2, res); @@ -32,6 +35,24 @@ fn test_sendfile_linux() { assert_eq!(7, offset); } +#[cfg(any(target_os = "android", target_os = "linux"))] +#[test] +fn test_sendfile_largefile() { + require_largefile!("test_sendfile_largefile"); + + let mut offset = 0x100000000i64; + let start: u64 = (offset - 4).try_into().unwrap(); + let mut src = tempfile().unwrap(); + let mut dst = tempfile().unwrap(); + src.seek(std::io::SeekFrom::Start(start)).unwrap(); + src.write_all(b"The example text").unwrap(); + assert_eq!(sendfile(&dst, &src, Some(&mut offset), 12), Ok(12)); + dst.rewind().unwrap(); + let mut buf = [0u8; 12]; + assert_eq!(read(&dst, &mut buf), Ok(12)); + assert_eq!(b"example text", &buf); +} + #[cfg(target_os = "linux")] #[test] fn test_sendfile64_linux() { diff --git a/test/test_unistd.rs b/test/test_unistd.rs index 5cb019bd95..438a6b730c 100644 --- a/test/test_unistd.rs +++ b/test/test_unistd.rs @@ -1,4 +1,5 @@ -use libc::{_exit, mode_t, off_t}; +use cfg_if::cfg_if; +use libc::{_exit, mode_t}; use nix::errno::Errno; #[cfg(not(any(target_os = "redox", target_os = "haiku")))] use nix::fcntl::readlink; @@ -25,7 +26,7 @@ use std::ffi::CString; #[cfg(not(target_os = "redox"))] use std::fs::DirBuilder; use std::fs::{self, File}; -use std::io::Write; +use std::io::{Seek, Write}; #[cfg(not(any( target_os = "fuchsia", target_os = "redox", @@ -582,14 +583,39 @@ fn test_lseek() { let mut tmp = tempfile().unwrap(); tmp.write_all(CONTENTS).unwrap(); - let offset: off_t = 5; - lseek(&tmp, offset, Whence::SeekSet).unwrap(); + lseek(&tmp, 5, Whence::SeekSet).unwrap(); let mut buf = [0u8; 7]; crate::read_exact(&tmp, &mut buf); assert_eq!(b"f123456", &buf); } +#[test] +fn test_lseek_largefile() { + require_largefile!("test_lseek_largefile"); + + const CONTENTS: &[u8] = b"The example text"; + let mut tmp = tempfile().unwrap(); + let start = 0xffff_fffci64; // 4 bytes before exceeding 32-bit capacity + let offset = start + 4; + + // The version of seek in std::io supports 64-bit offsets, so we + // use that to write contents to the file. Many platforms and filesystems + // support files with holes, which allows them to avoid writing the ~4GiB + // before the content we are writing. However, it is possible for the seek + // or write to fail (e.g. on platforms that actually do not support large + // files), in which case we provide a descriptive message to indicate that + // we were unable to test the part we're actually interested in. + tmp.seek(std::io::SeekFrom::Start(start as u64)) + .expect("Cannot test lseek with large offsets: std::io seek failed"); + tmp.write_all(CONTENTS) + .expect("Cannot test lseek with large offsets: write_all failed"); + assert_eq!(lseek(&tmp, offset, Whence::SeekSet), Ok(offset)); + let mut buf = [0u8; 12]; + crate::read_exact(&tmp, &mut buf); + assert_eq!(b"example text", &buf); +} + #[cfg(linux_android)] #[test] fn test_lseek64() { @@ -782,6 +808,20 @@ fn test_truncate() { assert_eq!(4, metadata.len()); } +#[test] +#[cfg(not(any(target_os = "redox", target_os = "fuchsia")))] +fn test_truncate_largefile() { + require_largefile!("test_truncate_largefile"); + + let tempdir = tempdir().unwrap(); + let path = tempdir.path().join("file"); + File::create(&path).unwrap(); + let length = 0x1_0000_0008i64; + truncate(&path, length).unwrap(); + let metadata = fs::metadata(&path).unwrap(); + assert_eq!(0x1_0000_0008, metadata.len()); +} + #[test] fn test_ftruncate() { let tempdir = tempdir().unwrap(); @@ -798,6 +838,17 @@ fn test_ftruncate() { assert_eq!(2, metadata.len()); } +#[test] +fn test_ftruncate_largefile() { + require_largefile!("test_ftruncate_largefile"); + + let tmp = tempfile().unwrap(); + let length = 0x1_0000_000ci64; + ftruncate(&tmp, length).unwrap(); + let metadata = tmp.metadata().unwrap(); + assert_eq!(0x1_0000_000c, metadata.len()); +} + // Used in `test_alarm`. #[cfg(not(target_os = "redox"))] static mut ALARM_CALLED: bool = false;