Use mmap for metadata loading

This can have a significant improvement on compilation times. In
addition it reduces the memory consumption.

Fixes #927

Cargo.lock

@@ -240,6 +240,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "memmap2"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04e3e85b970d650e2ae6d70592474087051c11c54da7f7b4949725c5735fbcc6"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "object"
 version = "0.23.0"
@@ -310,6 +319,7 @@ dependencies = [
  "gimli",
  "indexmap",
  "libloading",
+ "memmap2",
  "object",
  "smallvec",
  "target-lexicon",

Cargo.toml

@@ -16,12 +16,13 @@ cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch
 cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 target-lexicon = "0.11.0"
 gimli = { version = "0.23.0", default-features = false, features = ["write"]}
-object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] }
+object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
 ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
 indexmap = "1.0.2"
 libloading = { version = "0.6.0", optional = true }
 smallvec = "1.6.1"
+memmap2 = "0.2.1"
 
 # Uncomment to use local checkout of cranelift
 #[patch."https://github.com/bytecodealliance/wasmtime/"]

src/metadata.rs

@@ -1,11 +1,11 @@
 //! Reading and writing of the rustc metadata for rlibs and dylibs
 
-use std::convert::TryFrom;
 use std::fs::File;
+use std::ops::Deref;
 use std::path::Path;
 
 use rustc_codegen_ssa::METADATA_FILENAME;
-use rustc_data_structures::owning_ref::OwningRef;
+use rustc_data_structures::owning_ref::{OwningRef, StableAddress};
 use rustc_data_structures::rustc_erase_owner;
 use rustc_data_structures::sync::MetadataRef;
 use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader};
@@ -17,38 +17,56 @@ use crate::backend::WriteMetadata;
 
 pub(crate) struct CraneliftMetadataLoader;
 
+struct StableMmap(memmap2::Mmap);
+
+impl Deref for StableMmap {
+    type Target = [u8];
+
+    fn deref(&self) -> &[u8] {
+        &*self.0
+    }
+}
+
+unsafe impl StableAddress for StableMmap {}
+
+fn load_metadata_with(
+    path: &Path,
+    f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
+) -> Result<MetadataRef, String> {
+    let file = File::open(path).map_err(|e| format!("{:?}", e))?;
+    let data = unsafe { memmap2::MmapOptions::new().map_copy_read_only(&file) }
+        .map_err(|e| format!("{:?}", e))?;
+    let metadata = OwningRef::new(StableMmap(data)).try_map(f)?;
+    return Ok(rustc_erase_owner!(metadata.map_owner_box()));
+}
+
 impl MetadataLoader for CraneliftMetadataLoader {
     fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
-        let mut archive = ar::Archive::new(File::open(path).map_err(|e| format!("{:?}", e))?);
+        load_metadata_with(path, |data| {
-        // Iterate over all entries in the archive:
+            let archive = object::read::archive::ArchiveFile::parse(&*data)
-        while let Some(entry_result) = archive.next_entry() {
+                .map_err(|e| format!("{:?}", e))?;
-            let mut entry = entry_result.map_err(|e| format!("{:?}", e))?;
-            if entry.header().identifier() == METADATA_FILENAME.as_bytes() {
-                let mut buf = Vec::with_capacity(
-                    usize::try_from(entry.header().size())
-                        .expect("Rlib metadata file too big to load into memory."),
-                );
-                ::std::io::copy(&mut entry, &mut buf).map_err(|e| format!("{:?}", e))?;
-                let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-                return Ok(rustc_erase_owner!(buf.map_owner_box()));
-            }
-        }
 
-        Err("couldn't find metadata entry".to_string())
+                for entry_result in archive.members() {
+                let entry = entry_result.map_err(|e| format!("{:?}", e))?;
+                if entry.name() == METADATA_FILENAME.as_bytes() {
+                    return Ok(entry.data());
+                }
+            }
+
+            Err("couldn't find metadata entry".to_string())
+        })
     }
 
     fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
         use object::{Object, ObjectSection};
-        let file = std::fs::read(path).map_err(|e| format!("read:{:?}", e))?;
+
-        let file = object::File::parse(&file).map_err(|e| format!("parse: {:?}", e))?;
+        load_metadata_with(path, |data| {
-        let buf = file
+            let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?;
-            .section_by_name(".rustc")
+            file.section_by_name(".rustc")
-            .ok_or("no .rustc section")?
+                .ok_or("no .rustc section")?
-            .data()
+                .data()
-            .map_err(|e| format!("failed to read .rustc section: {:?}", e))?
+                .map_err(|e| format!("failed to read .rustc section: {:?}", e))
-            .to_owned();
+        })
-        let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-        Ok(rustc_erase_owner!(buf.map_owner_box()))
     }
 }