Auto merge of #83682 - bjorn3:mmap_wrapper, r=cjgillot

Add an Mmap wrapper to rustc_data_structures

This wrapper implements StableAddress and falls back to directly reading the file on wasm32.

Taken from #83640, which I will close due to the perf regression.
This commit is contained in:
bors 2021-04-03 13:23:42 +00:00
commit 97717a5618
12 changed files with 63 additions and 50 deletions

View File

@ -3812,7 +3812,6 @@ dependencies = [
"itertools 0.9.0",
"jobserver",
"libc",
"memmap2",
"pathdiff",
"rustc_apfloat",
"rustc_ast",
@ -3847,6 +3846,7 @@ dependencies = [
"jobserver",
"libc",
"measureme",
"memmap2",
"parking_lot",
"rustc-hash",
"rustc-rayon",
@ -4146,7 +4146,6 @@ name = "rustc_metadata"
version = "0.0.0"
dependencies = [
"libc",
"memmap2",
"rustc_ast",
"rustc_attr",
"rustc_data_structures",

View File

@ -240,15 +240,6 @@ dependencies = [
"libc",
]
[[package]]
name = "memmap2"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04e3e85b970d650e2ae6d70592474087051c11c54da7f7b4949725c5735fbcc6"
dependencies = [
"libc",
]
[[package]]
name = "object"
version = "0.23.0"
@ -319,7 +310,6 @@ dependencies = [
"gimli",
"indexmap",
"libloading",
"memmap2",
"object",
"smallvec",
"target-lexicon",

View File

@ -22,7 +22,6 @@ ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg
indexmap = "1.0.2"
libloading = { version = "0.6.0", optional = true }
smallvec = "1.6.1"
memmap2 = "0.2.1"
# Uncomment to use local checkout of cranelift
#[patch."https://github.com/bytecodealliance/wasmtime/"]

View File

@ -1,11 +1,11 @@
//! Reading and writing of the rustc metadata for rlibs and dylibs
use std::fs::File;
use std::ops::Deref;
use std::path::Path;
use rustc_codegen_ssa::METADATA_FILENAME;
use rustc_data_structures::owning_ref::{OwningRef, StableAddress};
use rustc_data_structures::memmap::Mmap;
use rustc_data_structures::owning_ref::OwningRef;
use rustc_data_structures::rustc_erase_owner;
use rustc_data_structures::sync::MetadataRef;
use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader};
@ -17,26 +17,13 @@
pub(crate) struct CraneliftMetadataLoader;
struct StableMmap(memmap2::Mmap);
impl Deref for StableMmap {
type Target = [u8];
fn deref(&self) -> &[u8] {
&*self.0
}
}
unsafe impl StableAddress for StableMmap {}
fn load_metadata_with(
path: &Path,
f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
) -> Result<MetadataRef, String> {
let file = File::open(path).map_err(|e| format!("{:?}", e))?;
let data = unsafe { memmap2::MmapOptions::new().map_copy_read_only(&file) }
.map_err(|e| format!("{:?}", e))?;
let metadata = OwningRef::new(StableMmap(data)).try_map(f)?;
let data = unsafe { Mmap::map(file) }.map_err(|e| format!("{:?}", e))?;
let metadata = OwningRef::new(data).try_map(f)?;
return Ok(rustc_erase_owner!(metadata.map_owner_box()));
}

View File

@ -11,7 +11,6 @@ test = false
bitflags = "1.2.1"
cc = "1.0.1"
itertools = "0.9"
memmap2 = "0.2.1"
tracing = "0.1"
libc = "0.2.50"
jobserver = "0.1.11"

View File

@ -2,6 +2,7 @@
use crate::traits::*;
use crate::ModuleCodegen;
use rustc_data_structures::memmap::Mmap;
use rustc_errors::FatalError;
use std::ffi::CString;
@ -93,7 +94,7 @@ pub fn cost(&self) -> u64 {
pub enum SerializedModule<M: ModuleBufferMethods> {
Local(M),
FromRlib(Vec<u8>),
FromUncompressedFile(memmap2::Mmap),
FromUncompressedFile(Mmap),
}
impl<M: ModuleBufferMethods> SerializedModule<M> {

View File

@ -10,6 +10,7 @@
use crate::traits::*;
use jobserver::{Acquired, Client};
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::memmap::Mmap;
use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_data_structures::profiling::TimingGuard;
use rustc_data_structures::profiling::VerboseTimingGuard;
@ -1958,7 +1959,7 @@ pub fn submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>(
.unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e));
let mmap = unsafe {
memmap2::Mmap::map(&file).unwrap_or_else(|e| {
Mmap::map(file).unwrap_or_else(|e| {
panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
})
};

View File

@ -36,3 +36,6 @@ features = ["nightly"]
[target.'cfg(windows)'.dependencies]
winapi = { version = "0.3", features = ["fileapi", "psapi"] }
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
memmap2 = "0.2.1"

View File

@ -84,6 +84,7 @@ macro_rules! unlikely {
pub mod stable_map;
pub mod svh;
pub use ena::snapshot_vec;
pub mod memmap;
pub mod sorted_map;
pub mod stable_set;
#[macro_use]

View File

@ -0,0 +1,47 @@
use std::fs::File;
use std::io;
use std::ops::Deref;
use crate::owning_ref::StableAddress;
/// A trivial wrapper for [`memmap2::Mmap`] that implements [`StableAddress`].
#[cfg(not(target_arch = "wasm32"))]
pub struct Mmap(memmap2::Mmap);
#[cfg(target_arch = "wasm32")]
pub struct Mmap(Vec<u8>);
#[cfg(not(target_arch = "wasm32"))]
impl Mmap {
#[inline]
pub unsafe fn map(file: File) -> io::Result<Self> {
memmap2::Mmap::map(&file).map(Mmap)
}
}
#[cfg(target_arch = "wasm32")]
impl Mmap {
#[inline]
pub unsafe fn map(mut file: File) -> io::Result<Self> {
use std::io::Read;
let mut data = Vec::new();
file.read_to_end(&mut data)?;
Ok(Mmap(data))
}
}
impl Deref for Mmap {
type Target = [u8];
#[inline]
fn deref(&self) -> &[u8] {
&*self.0
}
}
// SAFETY: On architectures other than WASM, mmap is used as backing storage. The address of this
// memory map is stable. On WASM, `Vec<u8>` is used as backing storage. The `Mmap` type doesn't
// export any function that can cause the `Vec` to be re-allocated. As such the address of the
// bytes inside this `Vec` is stable.
unsafe impl StableAddress for Mmap {}

View File

@ -11,7 +11,6 @@ doctest = false
libc = "0.2"
snap = "1"
tracing = "0.1"
memmap2 = "0.2.1"
smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
rustc_middle = { path = "../rustc_middle" }
rustc_attr = { path = "../rustc_attr" }

View File

@ -216,6 +216,7 @@
use crate::rmeta::{rustc_version, MetadataBlob, METADATA_HEADER};
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_data_structures::memmap::Mmap;
use rustc_data_structures::owning_ref::OwningRef;
use rustc_data_structures::svh::Svh;
use rustc_data_structures::sync::MetadataRef;
@ -232,7 +233,6 @@
use snap::read::FrameDecoder;
use std::io::{Read, Result as IoResult, Write};
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::{cmp, fmt, fs};
use tracing::{debug, info, warn};
@ -727,19 +727,6 @@ fn find_commandline_library(&mut self) -> Result<Option<Library>, CrateError> {
}
}
/// A trivial wrapper for `Mmap` that implements `StableDeref`.
struct StableDerefMmap(memmap2::Mmap);
impl Deref for StableDerefMmap {
type Target = [u8];
fn deref(&self) -> &[u8] {
self.0.deref()
}
}
unsafe impl stable_deref_trait::StableDeref for StableDerefMmap {}
fn get_metadata_section(
target: &Target,
flavor: CrateFlavor,
@ -779,11 +766,11 @@ fn get_metadata_section(
// mmap the file, because only a small fraction of it is read.
let file = std::fs::File::open(filename)
.map_err(|_| format!("failed to open rmeta metadata: '{}'", filename.display()))?;
let mmap = unsafe { memmap2::Mmap::map(&file) };
let mmap = unsafe { Mmap::map(file) };
let mmap = mmap
.map_err(|_| format!("failed to mmap rmeta metadata: '{}'", filename.display()))?;
rustc_erase_owner!(OwningRef::new(StableDerefMmap(mmap)).map_owner_box())
rustc_erase_owner!(OwningRef::new(mmap).map_owner_box())
}
};
let blob = MetadataBlob::new(raw_bytes);