mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #96544 - m-ysk:feature/issue-96358, r=cjgillot

Browse Source 
Stop keeping metadata in memory before writing it to disk

Fixes #96358

I created this PR according with the instruction given in the issue except for the following points:

- While the issue says "Write metadata into the temporary file in `encode_and_write_metadata` even if `!need_metadata_file`", I could not do that. That is because though I tried to do that and run `x.py test`, I got a lot of test failures as follows.

<details>
<summary>List of failed tests</summary>
<pre>
<code>
failures:
    [ui] src/test/ui/json-multiple.rs
    [ui] src/test/ui/json-options.rs
    [ui] src/test/ui/rmeta/rmeta-rpass.rs
    [ui] src/test/ui/save-analysis/emit-notifications.rs
    [ui] src/test/ui/svh/changing-crates.rs
    [ui] src/test/ui/svh/svh-change-lit.rs
    [ui] src/test/ui/svh/svh-change-significant-cfg.rs
    [ui] src/test/ui/svh/svh-change-trait-bound.rs
    [ui] src/test/ui/svh/svh-change-type-arg.rs
    [ui] src/test/ui/svh/svh-change-type-ret.rs
    [ui] src/test/ui/svh/svh-change-type-static.rs
    [ui] src/test/ui/svh/svh-use-trait.rs

test result: FAILED. 12915 passed; 12 failed; 100 ignored; 0 measured; 0 filtered out; finished in 71.41s

Some tests failed in compiletest suite=ui mode=ui host=x86_64-unknown-linux-gnu target=x86_64-unknown-linux-gnu
Build completed unsuccessfully in 0:01:58
</code>
</pre>
</details>

- I could not resolve the extra tasks about `create_rmeta_file` and `create_compressed_metadata_file` for my lack of ability.
This commit is contained in:
bors 2022-07-14 21:50:14 +00:00
commit 1ba1fec234
16 changed files with 300 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock
View File

@ -3969,7 +3969,6 @@ dependencies = [
"rustc_ty_utils",
"rustc_typeck",
"smallvec",
"tempfile",
"tracing",
"winapi",
]
@ -4080,6 +4079,7 @@ dependencies = [
"rustc_type_ir",
"smallvec",
"snap",
"tempfile",
"tracing",
]

22
compiler/rustc_codegen_ssa/src/back/link.rs
View File

@ -6,6 +6,7 @@ use rustc_data_structures::temp_dir::MaybeTempDir;
use rustc_errors::{ErrorGuaranteed, Handler};
use rustc_fs_util::fix_windows_verbatim_for_gcc;
use rustc_hir::def_id::CrateNum;
use rustc_metadata::fs::{emit_metadata, METADATA_FILENAME};
use rustc_middle::middle::dependency_format::Linkage;
use rustc_middle::middle::exported_symbols::SymbolExportKind;
use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Strip};
@ -28,10 +29,7 @@ use super::command::Command;
use super::linker::{self, Linker};
use super::metadata::{create_rmeta_file, MetadataPosition};
use super::rpath::{self, RPathConfig};
use crate::{
looks_like_rust_object_file, CodegenResults, CompiledModule, CrateInfo, NativeLib,
METADATA_FILENAME,
};
use crate::{looks_like_rust_object_file, CodegenResults, CompiledModule, CrateInfo, NativeLib};
use cc::windows_registry;
use regex::Regex;
@ -241,22 +239,6 @@ pub fn each_linked_rlib(
Ok(())
}
/// We use a temp directory here to avoid races between concurrent rustc processes,
/// such as builds in the same directory using the same filename for metadata while
/// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a
/// directory being searched for `extern crate` (observing an incomplete file).
/// The returned path is the temporary file containing the complete metadata.
pub fn emit_metadata(sess: &Session, metadata: &[u8], tmpdir: &MaybeTempDir) -> PathBuf {
let out_filename = tmpdir.as_ref().join(METADATA_FILENAME);
let result = fs::write(&out_filename, metadata);
if let Err(e) = result {
sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
}
out_filename
}
/// Create an 'rlib'.
///
/// An rlib in its current incarnation is essentially a renamed .a file. The rlib primarily contains

3
compiler/rustc_codegen_ssa/src/back/metadata.rs
View File

@ -16,14 +16,13 @@ use rustc_data_structures::memmap::Mmap;
use rustc_data_structures::owning_ref::OwningRef;
use rustc_data_structures::rustc_erase_owner;
use rustc_data_structures::sync::MetadataRef;
use rustc_metadata::fs::METADATA_FILENAME;
use rustc_metadata::EncodedMetadata;
use rustc_session::cstore::MetadataLoader;
use rustc_session::Session;
use rustc_target::abi::Endian;
use rustc_target::spec::{RelocModel, Target};
use crate::METADATA_FILENAME;
/// The default metadata loader. This is used by cg_llvm and cg_clif.
///
/// # Metadata location

3
compiler/rustc_codegen_ssa/src/lib.rs
View File

@ -64,9 +64,6 @@ pub struct ModuleCodegen<M> {
pub kind: ModuleKind,
}
// FIXME(eddyb) maybe include the crate name in this?
pub const METADATA_FILENAME: &str = "lib.rmeta";
impl<M> ModuleCodegen<M> {
pub fn into_compiled_module(
self,

63
compiler/rustc_data_structures/src/memmap.rs
View File

@ -1,6 +1,6 @@
use std::fs::File;
use std::io;
use std::ops::Deref;
use std::ops::{Deref, DerefMut};
use crate::owning_ref::StableAddress;
@ -45,3 +45,64 @@ impl Deref for Mmap {
// export any function that can cause the `Vec` to be re-allocated. As such the address of the
// bytes inside this `Vec` is stable.
unsafe impl StableAddress for Mmap {}
#[cfg(not(target_arch = "wasm32"))]
pub struct MmapMut(memmap2::MmapMut);
#[cfg(target_arch = "wasm32")]
pub struct MmapMut(Vec<u8>);
#[cfg(not(target_arch = "wasm32"))]
impl MmapMut {
#[inline]
pub fn map_anon(len: usize) -> io::Result<Self> {
let mmap = memmap2::MmapMut::map_anon(len)?;
Ok(MmapMut(mmap))
}
#[inline]
pub fn flush(&mut self) -> io::Result<()> {
self.0.flush()
}
#[inline]
pub fn make_read_only(self) -> std::io::Result<Mmap> {
let mmap = self.0.make_read_only()?;
Ok(Mmap(mmap))
}
}
#[cfg(target_arch = "wasm32")]
impl MmapMut {
#[inline]
pub fn map_anon(len: usize) -> io::Result<Self> {
let data = Vec::with_capacity(len);
Ok(MmapMut(data))
}
#[inline]
pub fn flush(&mut self) -> io::Result<()> {
Ok(())
}
#[inline]
pub fn make_read_only(self) -> std::io::Result<Mmap> {
Ok(Mmap(self.0))
}
}
impl Deref for MmapMut {
type Target = [u8];
#[inline]
fn deref(&self) -> &[u8] {
&*self.0
}
}
impl DerefMut for MmapMut {
#[inline]
fn deref_mut(&mut self) -> &mut [u8] {
&mut *self.0
}
}

1
compiler/rustc_interface/Cargo.toml
View File

@ -46,7 +46,6 @@ rustc_query_impl = { path = "../rustc_query_impl" }
rustc_resolve = { path = "../rustc_resolve" }
rustc_trait_selection = { path = "../rustc_trait_selection" }
rustc_ty_utils = { path = "../rustc_ty_utils" }
tempfile = "3.2"
[target.'cfg(unix)'.dependencies]
libc = "0.2"

74
compiler/rustc_interface/src/passes.rs
View File

@ -5,18 +5,15 @@ use crate::util;
use ast::CRATE_NODE_ID;
use rustc_ast::{self as ast, visit};
use rustc_borrowck as mir_borrowck;
use rustc_codegen_ssa::back::link::emit_metadata;
use rustc_codegen_ssa::traits::CodegenBackend;
use rustc_data_structures::parallel;
use rustc_data_structures::sync::{Lrc, OnceCell, WorkerLocal};
use rustc_data_structures::temp_dir::MaybeTempDir;
use rustc_errors::{Applicability, ErrorGuaranteed, MultiSpan, PResult};
use rustc_expand::base::{ExtCtxt, LintStoreExpand, ResolverExpand};
use rustc_hir::def_id::{StableCrateId, LOCAL_CRATE};
use rustc_hir::def_id::StableCrateId;
use rustc_hir::definitions::Definitions;
use rustc_lint::{BufferedEarlyLint, EarlyCheckNode, LintStore};
use rustc_metadata::creader::CStore;
use rustc_metadata::{encode_metadata, EncodedMetadata};
use rustc_middle::arena::Arena;
use rustc_middle::dep_graph::DepGraph;
use rustc_middle::ty::query::{ExternProviders, Providers};
@ -29,14 +26,13 @@ use rustc_query_impl::{OnDiskCache, Queries as TcxQueries};
use rustc_resolve::{Resolver, ResolverArenas};
use rustc_session::config::{CrateType, Input, OutputFilenames, OutputType};
use rustc_session::cstore::{CrateStoreDyn, MetadataLoader, MetadataLoaderDyn};
use rustc_session::output::{filename_for_input, filename_for_metadata};
use rustc_session::output::filename_for_input;
use rustc_session::search_paths::PathKind;
use rustc_session::{Limit, Session};
use rustc_span::symbol::{sym, Symbol};
use rustc_span::FileName;
use rustc_trait_selection::traits;
use rustc_typeck as typeck;
use tempfile::Builder as TempFileBuilder;
use tracing::{info, warn};
use std::any::Any;
@ -993,69 +989,6 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> {
Ok(())
}
fn encode_and_write_metadata(
tcx: TyCtxt<'_>,
outputs: &OutputFilenames,
) -> (EncodedMetadata, bool) {
#[derive(PartialEq, Eq, PartialOrd, Ord)]
enum MetadataKind {
None,
Uncompressed,
Compressed,
}
let metadata_kind = tcx
.sess
.crate_types()
.iter()
.map(|ty| match *ty {
CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => MetadataKind::None,
CrateType::Rlib => MetadataKind::Uncompressed,
CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed,
})
.max()
.unwrap_or(MetadataKind::None);
let metadata = match metadata_kind {
MetadataKind::None => EncodedMetadata::new(),
MetadataKind::Uncompressed | MetadataKind::Compressed => encode_metadata(tcx),
};
let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata");
let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata);
if need_metadata_file {
let crate_name = tcx.crate_name(LOCAL_CRATE);
let out_filename = filename_for_metadata(tcx.sess, crate_name.as_str(), outputs);
// To avoid races with another rustc process scanning the output directory,
// we need to write the file somewhere else and atomically move it to its
// final destination, with an `fs::rename` call. In order for the rename to
// always succeed, the temporary file needs to be on the same filesystem,
// which is why we create it inside the output directory specifically.
let metadata_tmpdir = TempFileBuilder::new()
.prefix("rmeta")
.tempdir_in(out_filename.parent().unwrap())
.unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err)));
let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps);
let metadata_filename = emit_metadata(tcx.sess, metadata.raw_data(), &metadata_tmpdir);
if let Err(e) = util::non_durable_rename(&metadata_filename, &out_filename) {
tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
}
if tcx.sess.opts.json_artifact_notifications {
tcx.sess
.parse_sess
.span_diagnostic
.emit_artifact_notification(&out_filename, "metadata");
}
}
let need_metadata_module = metadata_kind == MetadataKind::Compressed;
(metadata, need_metadata_module)
}
/// Runs the codegen backend, after which the AST and analysis can
/// be discarded.
pub fn start_codegen<'tcx>(
@ -1065,7 +998,8 @@ pub fn start_codegen<'tcx>(
) -> Box<dyn Any> {
info!("Pre-codegen\n{:?}", tcx.debug_stats());
let (metadata, need_metadata_module) = encode_and_write_metadata(tcx, outputs);
let (metadata, need_metadata_module) =
rustc_metadata::fs::encode_and_write_metadata(tcx, outputs);
let codegen = tcx.sess.time("codegen_crate", move || {
codegen_backend.codegen_crate(tcx, metadata, need_metadata_module)

18
compiler/rustc_interface/src/util.rs
View File

@ -650,24 +650,6 @@ pub fn build_output_filenames(
}
}
#[cfg(not(target_os = "linux"))]
pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> {
std::fs::rename(src, dst)
}
/// This function attempts to bypass the auto_da_alloc heuristic implemented by some filesystems
/// such as btrfs and ext4. When renaming over a file that already exists then they will "helpfully"
/// write back the source file before committing the rename in case a developer forgot some of
/// the fsyncs in the open/write/fsync(file)/rename/fsync(dir) dance for atomic file updates.
///
/// To avoid triggering this heuristic we delete the destination first, if it exists.
/// The cost of an extra syscall is much lower than getting descheduled for the sync IO.
#[cfg(target_os = "linux")]
pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> {
let _ = std::fs::remove_file(dst);
std::fs::rename(src, dst)
}
/// Returns a version string such as "1.46.0 (04488afe3 2020-08-24)"
pub fn version_str() -> Option<&'static str> {
option_env!("CFG_VERSION")

1
compiler/rustc_metadata/Cargo.toml
View File

@ -12,6 +12,7 @@ odht = { version = "0.3.1", features = ["nightly"] }
snap = "1"
tracing = "0.1"
smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
tempfile = "3.2"
rustc_middle = { path = "../rustc_middle" }
rustc_attr = { path = "../rustc_attr" }
rustc_data_structures = { path = "../rustc_data_structures" }

137
compiler/rustc_metadata/src/fs.rs Normal file
View File

@ -0,0 +1,137 @@
use crate::{encode_metadata, EncodedMetadata};
use rustc_data_structures::temp_dir::MaybeTempDir;
use rustc_hir::def_id::LOCAL_CRATE;
use rustc_middle::ty::TyCtxt;
use rustc_session::config::{CrateType, OutputFilenames, OutputType};
use rustc_session::output::filename_for_metadata;
use rustc_session::Session;
use tempfile::Builder as TempFileBuilder;
use std::fs;
use std::path::{Path, PathBuf};
// FIXME(eddyb) maybe include the crate name in this?
pub const METADATA_FILENAME: &str = "lib.rmeta";
/// We use a temp directory here to avoid races between concurrent rustc processes,
/// such as builds in the same directory using the same filename for metadata while
/// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a
/// directory being searched for `extern crate` (observing an incomplete file).
/// The returned path is the temporary file containing the complete metadata.
pub fn emit_metadata(sess: &Session, metadata: &[u8], tmpdir: &MaybeTempDir) -> PathBuf {
let out_filename = tmpdir.as_ref().join(METADATA_FILENAME);
let result = fs::write(&out_filename, metadata);
if let Err(e) = result {
sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
}
out_filename
}
pub fn encode_and_write_metadata(
tcx: TyCtxt<'_>,
outputs: &OutputFilenames,
) -> (EncodedMetadata, bool) {
#[derive(PartialEq, Eq, PartialOrd, Ord)]
enum MetadataKind {
None,
Uncompressed,
Compressed,
}
let metadata_kind = tcx
.sess
.crate_types()
.iter()
.map(|ty| match *ty {
CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => MetadataKind::None,
CrateType::Rlib => MetadataKind::Uncompressed,
CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed,
})
.max()
.unwrap_or(MetadataKind::None);
let crate_name = tcx.crate_name(LOCAL_CRATE);
let out_filename = filename_for_metadata(tcx.sess, crate_name.as_str(), outputs);
// To avoid races with another rustc process scanning the output directory,
// we need to write the file somewhere else and atomically move it to its
// final destination, with an `fs::rename` call. In order for the rename to
// always succeed, the temporary file needs to be on the same filesystem,
// which is why we create it inside the output directory specifically.
let metadata_tmpdir = TempFileBuilder::new()
.prefix("rmeta")
.tempdir_in(out_filename.parent().unwrap_or_else(|| Path::new("")))
.unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err)));
let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps);
let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME);
// Always create a file at `metadata_filename`, even if we have nothing to write to it.
// This simplifies the creation of the output `out_filename` when requested.
match metadata_kind {
MetadataKind::None => {
std::fs::File::create(&metadata_filename).unwrap_or_else(|e| {
tcx.sess.fatal(&format!(
"failed to create the file {}: {}",
metadata_filename.display(),
e
))
});
}
MetadataKind::Uncompressed | MetadataKind::Compressed => {
encode_metadata(tcx, &metadata_filename);
}
};
let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata");
// If the user requests metadata as output, rename `metadata_filename`
// to the expected output `out_filename`. The match above should ensure
// this file always exists.
let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata);
let (metadata_filename, metadata_tmpdir) = if need_metadata_file {
if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) {
tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
}
if tcx.sess.opts.json_artifact_notifications {
tcx.sess
.parse_sess
.span_diagnostic
.emit_artifact_notification(&out_filename, "metadata");
}
(out_filename, None)
} else {
(metadata_filename, Some(metadata_tmpdir))
};
// Load metadata back to memory: codegen may need to include it in object files.
let metadata =
EncodedMetadata::from_path(metadata_filename, metadata_tmpdir).unwrap_or_else(|e| {
tcx.sess.fatal(&format!("failed to create encoded metadata from file: {}", e))
});
let need_metadata_module = metadata_kind == MetadataKind::Compressed;
(metadata, need_metadata_module)
}
#[cfg(not(target_os = "linux"))]
pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> {
std::fs::rename(src, dst)
}
/// This function attempts to bypass the auto_da_alloc heuristic implemented by some filesystems
/// such as btrfs and ext4. When renaming over a file that already exists then they will "helpfully"
/// write back the source file before committing the rename in case a developer forgot some of
/// the fsyncs in the open/write/fsync(file)/rename/fsync(dir) dance for atomic file updates.
///
/// To avoid triggering this heuristic we delete the destination first, if it exists.
/// The cost of an extra syscall is much lower than getting descheduled for the sync IO.
#[cfg(target_os = "linux")]
pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> {
let _ = std::fs::remove_file(dst);
std::fs::rename(src, dst)
}

2
compiler/rustc_metadata/src/lib.rs
View File

@ -34,6 +34,8 @@ mod native_libs;
mod rmeta;
pub mod creader;
pub mod fs;
pub mod locator;
pub use fs::{emit_metadata, METADATA_FILENAME};
pub use rmeta::{encode_metadata, EncodedMetadata, METADATA_HEADER};

103
compiler/rustc_metadata/src/rmeta/encoder.rs
View File

@ -4,8 +4,10 @@ use crate::rmeta::*;
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
use rustc_data_structures::memmap::{Mmap, MmapMut};
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::sync::{join, par_iter, Lrc, ParallelIterator};
use rustc_data_structures::temp_dir::MaybeTempDir;
use rustc_hir as hir;
use rustc_hir::def::DefKind;
use rustc_hir::def_id::{
@ -27,8 +29,7 @@ use rustc_middle::ty::codec::TyEncoder;
use rustc_middle::ty::fast_reject::{self, SimplifiedType, TreatParams};
use rustc_middle::ty::query::Providers;
use rustc_middle::ty::{self, SymbolName, Ty, TyCtxt};
use rustc_serialize::opaque::MemEncoder;
use rustc_serialize::{Encodable, Encoder};
use rustc_serialize::{opaque, Decodable, Decoder, Encodable, Encoder};
use rustc_session::config::CrateType;
use rustc_session::cstore::{ForeignModule, LinkagePreference, NativeLib};
use rustc_span::hygiene::{ExpnIndex, HygieneEncodeContext, MacroKind};
@ -39,12 +40,14 @@ use rustc_span::{
use rustc_target::abi::VariantIdx;
use std::borrow::Borrow;
use std::hash::Hash;
use std::io::{Read, Seek, Write};
use std::iter;
use std::num::NonZeroUsize;
use std::path::{Path, PathBuf};
use tracing::{debug, trace};
pub(super) struct EncodeContext<'a, 'tcx> {
opaque: MemEncoder,
opaque: opaque::FileEncoder,
tcx: TyCtxt<'tcx>,
feat: &'tcx rustc_feature::Features,
@ -729,12 +732,19 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
assert_eq!(total_bytes, computed_total_bytes);
if tcx.sess.meta_stats() {
self.opaque.flush();
// Rewind and re-read all the metadata to count the zero bytes we wrote.
let pos_before_rewind = self.opaque.file().stream_position().unwrap();
let mut zero_bytes = 0;
for e in self.opaque.data.iter() {
if *e == 0 {
self.opaque.file().rewind().unwrap();
let file = std::io::BufReader::new(self.opaque.file());
for e in file.bytes() {
if e.unwrap() == 0 {
zero_bytes += 1;
}
}
assert_eq!(self.opaque.file().stream_position().unwrap(), pos_before_rewind);
let perc = |bytes| (bytes * 100) as f64 / total_bytes as f64;
let p = |label, bytes| {
@ -2133,24 +2143,58 @@ fn prefetch_mir(tcx: TyCtxt<'_>) {
// will allow us to slice the metadata to the precise length that we just
// generated regardless of trailing bytes that end up in it.
#[derive(Encodable, Decodable)]
pub struct EncodedMetadata {
raw_data: Vec<u8>,
// The declaration order matters because `mmap` should be dropped before `_temp_dir`.
mmap: Option<Mmap>,
// We need to carry MaybeTempDir to avoid deleting the temporary
// directory while accessing the Mmap.
_temp_dir: Option<MaybeTempDir>,
}
impl EncodedMetadata {
#[inline]
pub fn new() -> EncodedMetadata {
EncodedMetadata { raw_data: Vec::new() }
pub fn from_path(path: PathBuf, temp_dir: Option<MaybeTempDir>) -> std::io::Result<Self> {
let file = std::fs::File::open(&path)?;
let file_metadata = file.metadata()?;
if file_metadata.len() == 0 {
return Ok(Self { mmap: None, _temp_dir: None });
}
let mmap = unsafe { Some(Mmap::map(file)?) };
Ok(Self { mmap, _temp_dir: temp_dir })
}
#[inline]
pub fn raw_data(&self) -> &[u8] {
&self.raw_data
self.mmap.as_ref().map(|mmap| mmap.as_ref()).unwrap_or_default()
}
}
pub fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata {
impl<S: Encoder> Encodable<S> for EncodedMetadata {
fn encode(&self, s: &mut S) {
let slice = self.raw_data();
slice.encode(s)
}
}
impl<D: Decoder> Decodable<D> for EncodedMetadata {
fn decode(d: &mut D) -> Self {
let len = d.read_usize();
let mmap = if len > 0 {
let mut mmap = MmapMut::map_anon(len).unwrap();
for _ in 0..len {
(&mut mmap[..]).write(&[d.read_u8()]).unwrap();
}
mmap.flush().unwrap();
Some(mmap.make_read_only().unwrap())
} else {
None
};
Self { mmap, _temp_dir: None }
}
}
pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path) {
let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata");
// Since encoding metadata is not in a query, and nothing is cached,
@ -2158,7 +2202,7 @@ pub fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata {
tcx.dep_graph.assert_ignored();
join(
|| encode_metadata_impl(tcx),
|| encode_metadata_impl(tcx, path),
|| {
if tcx.sess.threads() == 1 {
return;
@ -2168,12 +2212,12 @@ pub fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata {
// It can be removed if it turns out to cause trouble or be detrimental to performance.
join(|| prefetch_mir(tcx), || tcx.exported_symbols(LOCAL_CRATE));
},
)
.0
);
}
fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata {
let mut encoder = MemEncoder::new();
fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) {
let mut encoder = opaque::FileEncoder::new(path)
.unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to create file encoder: {}", err)));
encoder.emit_raw_bytes(METADATA_HEADER);
// Will be filled with the root position after encoding everything.
@ -2208,20 +2252,29 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata {
// culminating in the `CrateRoot` which points to all of it.
let root = ecx.encode_crate_root();
let mut result = ecx.opaque.finish();
ecx.opaque.flush();
let mut file = ecx.opaque.file();
// We will return to this position after writing the root position.
let pos_before_seek = file.stream_position().unwrap();
// Encode the root position.
let header = METADATA_HEADER.len();
file.seek(std::io::SeekFrom::Start(header as u64))
.unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to seek the file: {}", err)));
let pos = root.position.get();
result[header + 0] = (pos >> 24) as u8;
result[header + 1] = (pos >> 16) as u8;
result[header + 2] = (pos >> 8) as u8;
result[header + 3] = (pos >> 0) as u8;
file.write_all(&[(pos >> 24) as u8, (pos >> 16) as u8, (pos >> 8) as u8, (pos >> 0) as u8])
.unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to write to the file: {}", err)));
// Return to the position where we are before writing the root position.
file.seek(std::io::SeekFrom::Start(pos_before_seek)).unwrap();
// Record metadata size for self-profiling
tcx.prof.artifact_size("crate_metadata", "crate_metadata", result.len() as u64);
EncodedMetadata { raw_data: result }
tcx.prof.artifact_size(
"crate_metadata",
"crate_metadata",
file.metadata().unwrap().len() as u64,
);
}
pub fn provide(providers: &mut Providers) {
@ -2242,5 +2295,5 @@ pub fn provide(providers: &mut Providers) {
},
..*providers
};
}
}

4
compiler/rustc_metadata/src/rmeta/mod.rs
View File

@ -21,7 +21,7 @@ use rustc_middle::ty::fast_reject::SimplifiedType;
use rustc_middle::ty::query::Providers;
use rustc_middle::ty::{self, ReprOptions, Ty};
use rustc_middle::ty::{GeneratorDiagnosticData, ParameterizedOverTcx, TyCtxt};
use rustc_serialize::opaque::MemEncoder;
use rustc_serialize::opaque::FileEncoder;
use rustc_session::config::SymbolManglingVersion;
use rustc_session::cstore::{CrateDepKind, ForeignModule, LinkagePreference, NativeLib};
use rustc_span::edition::Edition;
@ -322,7 +322,7 @@ macro_rules! define_tables {
}
impl TableBuilders {
fn encode(&self, buf: &mut MemEncoder) -> LazyTables {
fn encode(&self, buf: &mut FileEncoder) -> LazyTables {
LazyTables {
$($name: self.$name.encode(buf)),+
}

6
compiler/rustc_metadata/src/rmeta/table.rs
View File

@ -4,8 +4,8 @@ use rustc_data_structures::fingerprint::Fingerprint;
use rustc_hir::def::{CtorKind, CtorOf};
use rustc_index::vec::Idx;
use rustc_middle::ty::ParameterizedOverTcx;
use rustc_serialize::opaque::MemEncoder;
use rustc_serialize::Encoder;
use rustc_serialize::opaque::FileEncoder;
use rustc_serialize::Encoder as _;
use rustc_span::hygiene::MacroKind;
use std::convert::TryInto;
use std::marker::PhantomData;
@ -281,7 +281,7 @@ where
Some(value).write_to_bytes(&mut self.blocks[i]);
}
pub(crate) fn encode<const N: usize>(&self, buf: &mut MemEncoder) -> LazyTable<I, T>
pub(crate) fn encode<const N: usize>(&self, buf: &mut FileEncoder) -> LazyTable<I, T>
where
Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>,
{

4
compiler/rustc_serialize/src/opaque.rs
View File

@ -297,6 +297,10 @@ impl FileEncoder {
}
}
pub fn file(&self) -> &File {
&self.file
}
#[inline]
fn capacity(&self) -> usize {
self.buf.len()

4
src/test/run-make-fulldeps/issue-26092/Makefile
View File

@ -1,4 +1,6 @@
-include ../tools.mk
# This test ensures that rustc does not panic with `-o ""` option.
all:
$(RUSTC) -o "" blank.rs 2>&1 | $(CGREP) -i 'No such file or directory'
$(RUSTC) -o "" blank.rs 2>&1 | $(CGREP) -i 'panic' && exit 1 || exit 0