mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #98098 - bjorn3:archive_refactor, r=michaelwoerister

Browse Source 
Remove the source archive functionality of ArchiveWriter

We now build archives through strictly additive means rather than taking an existing archive and potentially substracting parts. This is simpler and makes it easier to swap out the archive writer in https://github.com/rust-lang/rust/pull/97485.
This commit is contained in:
bors 2022-06-21 16:24:56 +00:00
commit dc80ca78b6
5 changed files with 61 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
compiler/rustc_codegen_cranelift/src/archive.rs
View File

@ -30,25 +30,7 @@ pub(crate) struct ArArchiveBuilder<'a> {
} }
impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
fn new(sess: &'a Session, output: &Path, input: Option<&Path>) -> Self { fn new(sess: &'a Session, output: &Path) -> Self {
let (src_archives, entries) = if let Some(input) = input {
let read_cache = ReadCache::new(File::open(input).unwrap());
let archive = ArchiveFile::parse(&read_cache).unwrap();
let mut entries = Vec::new();
for entry in archive.members() {
let entry = entry.unwrap();
entries.push((
entry.name().to_vec(),
ArchiveEntry::FromArchive { archive_index: 0, file_range: entry.file_range() },
));
}
(vec![read_cache.into_inner()], entries)
} else {
(vec![], Vec::new())
};
ArArchiveBuilder { ArArchiveBuilder {
sess, sess,
dst: output.to_path_buf(), dst: output.to_path_buf(),
@ -56,24 +38,11 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
// FIXME fix builtin ranlib on macOS // FIXME fix builtin ranlib on macOS
no_builtin_ranlib: sess.target.is_like_osx, no_builtin_ranlib: sess.target.is_like_osx,
src_archives, src_archives: vec![],
entries, entries: vec![],
} }
} }
fn src_files(&mut self) -> Vec<String> {
self.entries.iter().map(|(name, _)| String::from_utf8(name.clone()).unwrap()).collect()
}
fn remove_file(&mut self, name: &str) {
let index = self
.entries
.iter()
.position(|(entry_name, _)| entry_name == name.as_bytes())
.expect("Tried to remove file not existing in src archive");
self.entries.remove(index);
}
fn add_file(&mut self, file: &Path) { fn add_file(&mut self, file: &Path) {
self.entries.push(( self.entries.push((
file.file_name().unwrap().to_str().unwrap().to_string().into_bytes(), file.file_name().unwrap().to_str().unwrap().to_string().into_bytes(),
@ -105,7 +74,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
Ok(()) Ok(())
} }
fn build(mut self) { fn build(mut self) -> bool {
enum BuilderKind { enum BuilderKind {
Bsd(ar::Builder<File>), Bsd(ar::Builder<File>),
Gnu(ar::GnuBuilder<File>), Gnu(ar::GnuBuilder<File>),
@ -204,6 +173,8 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
) )
}; };
let any_members = !entries.is_empty();
// Add all files // Add all files
for (entry_name, data) in entries.into_iter() { for (entry_name, data) in entries.into_iter() {
let header = ar::Header::new(entry_name, data.len() as u64); let header = ar::Header::new(entry_name, data.len() as u64);
@ -229,6 +200,8 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
self.sess.fatal(&format!("Ranlib exited with code {:?}", status.code())); self.sess.fatal(&format!("Ranlib exited with code {:?}", status.code()));
} }
} }
any_members
} }
fn inject_dll_import_lib( fn inject_dll_import_lib(

47
compiler/rustc_codegen_gcc/src/archive.rs
View File

@ -32,7 +32,7 @@ pub struct ArArchiveBuilder<'a> {
} }
impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
fn new(sess: &'a Session, output: &Path, input: Option<&Path>) -> Self { fn new(sess: &'a Session, output: &Path) -> Self {
let config = ArchiveConfig { let config = ArchiveConfig {
sess, sess,
dst: output.to_path_buf(), dst: output.to_path_buf(),
@ -41,48 +41,13 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
use_gnu_style_archive: sess.target.options.archive_format == "gnu", use_gnu_style_archive: sess.target.options.archive_format == "gnu",
}; };
let (src_archives, entries) = if let Some(input) = input {
let mut archive = ar::Archive::new(File::open(input).unwrap());
let mut entries = Vec::new();
let mut i = 0;
while let Some(entry) = archive.next_entry() {
let entry = entry.unwrap();
entries.push((
String::from_utf8(entry.header().identifier().to_vec()).unwrap(),
ArchiveEntry::FromArchive {
archive_index: 0,
entry_index: i,
},
));
i += 1;
}
(vec![(input.to_owned(), archive)], entries)
} else {
(vec![], Vec::new())
};
ArArchiveBuilder { ArArchiveBuilder {
config, config,
src_archives, src_archives: vec![],
entries, entries: vec![],
} }
} }
fn src_files(&mut self) -> Vec<String> {
self.entries.iter().map(|(name, _)| name.clone()).collect()
}
fn remove_file(&mut self, name: &str) {
let index = self
.entries
.iter()
.position(|(entry_name, _)| entry_name == name)
.expect("Tried to remove file not existing in src archive");
self.entries.remove(index);
}
fn add_file(&mut self, file: &Path) { fn add_file(&mut self, file: &Path) {
self.entries.push(( self.entries.push((
file.file_name().unwrap().to_str().unwrap().to_string(), file.file_name().unwrap().to_str().unwrap().to_string(),
@ -113,7 +78,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
Ok(()) Ok(())
} }
fn build(mut self) { fn build(mut self) -> bool {
use std::process::Command; use std::process::Command;
fn add_file_using_ar(archive: &Path, file: &Path) { fn add_file_using_ar(archive: &Path, file: &Path) {
@ -146,6 +111,8 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
BuilderKind::Bsd(ar::Builder::new(File::create(&self.config.dst).unwrap())) BuilderKind::Bsd(ar::Builder::new(File::create(&self.config.dst).unwrap()))
}; };
let any_members = !self.entries.is_empty();
// Add all files // Add all files
for (entry_name, entry) in self.entries.into_iter() { for (entry_name, entry) in self.entries.into_iter() {
match entry { match entry {
@ -206,6 +173,8 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
if !status.success() { if !status.success() {
self.config.sess.fatal(&format!("Ranlib exited with code {:?}", status.code())); self.config.sess.fatal(&format!("Ranlib exited with code {:?}", status.code()));
} }
any_members
} }
fn inject_dll_import_lib(&mut self, _lib_name: &str, _dll_imports: &[DllImport], _tmpdir: &MaybeTempDir) { fn inject_dll_import_lib(&mut self, _lib_name: &str, _dll_imports: &[DllImport], _tmpdir: &MaybeTempDir) {

120
compiler/rustc_codegen_llvm/src/back/archive.rs
View File

@ -15,19 +15,12 @@ use rustc_data_structures::temp_dir::MaybeTempDir;
use rustc_session::cstore::{DllCallingConvention, DllImport}; use rustc_session::cstore::{DllCallingConvention, DllImport};
use rustc_session::Session; use rustc_session::Session;
struct ArchiveConfig<'a> {
pub sess: &'a Session,
pub dst: PathBuf,
pub src: Option<PathBuf>,
}
/// Helper for adding many files to an archive. /// Helper for adding many files to an archive.
#[must_use = "must call build() to finish building the archive"] #[must_use = "must call build() to finish building the archive"]
pub struct LlvmArchiveBuilder<'a> { pub struct LlvmArchiveBuilder<'a> {
config: ArchiveConfig<'a>, sess: &'a Session,
removals: Vec<String>, dst: PathBuf,
additions: Vec<Addition>, additions: Vec<Addition>,
src_archive: Option<Option<ArchiveRO>>,
} }
enum Addition { enum Addition {
@ -50,10 +43,6 @@ fn is_relevant_child(c: &Child<'_>) -> bool {
} }
} }
fn archive_config<'a>(sess: &'a Session, output: &Path, input: Option<&Path>) -> ArchiveConfig<'a> {
ArchiveConfig { sess, dst: output.to_path_buf(), src: input.map(|p| p.to_path_buf()) }
}
/// Map machine type strings to values of LLVM's MachineTypes enum. /// Map machine type strings to values of LLVM's MachineTypes enum.
fn llvm_machine_type(cpu: &str) -> LLVMMachineType { fn llvm_machine_type(cpu: &str) -> LLVMMachineType {
match cpu { match cpu {
@ -68,37 +57,8 @@ fn llvm_machine_type(cpu: &str) -> LLVMMachineType {
impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
/// Creates a new static archive, ready for modifying the archive specified /// Creates a new static archive, ready for modifying the archive specified
/// by `config`. /// by `config`.
fn new(sess: &'a Session, output: &Path, input: Option<&Path>) -> LlvmArchiveBuilder<'a> { fn new(sess: &'a Session, output: &Path) -> LlvmArchiveBuilder<'a> {
let config = archive_config(sess, output, input); LlvmArchiveBuilder { sess, dst: output.to_path_buf(), additions: Vec::new() }
LlvmArchiveBuilder {
config,
removals: Vec::new(),
additions: Vec::new(),
src_archive: None,
}
}
/// Removes a file from this archive
fn remove_file(&mut self, file: &str) {
self.removals.push(file.to_string());
}
/// Lists all files in an archive
fn src_files(&mut self) -> Vec<String> {
if self.src_archive().is_none() {
return Vec::new();
}
let archive = self.src_archive.as_ref().unwrap().as_ref().unwrap();
archive
.iter()
.filter_map(|child| child.ok())
.filter(is_relevant_child)
.filter_map(|child| child.name())
.filter(|name| !self.removals.iter().any(|x| x == name))
.map(|name| name.to_owned())
.collect()
} }
fn add_archive<F>(&mut self, archive: &Path, skip: F) -> io::Result<()> fn add_archive<F>(&mut self, archive: &Path, skip: F) -> io::Result<()>
@ -129,13 +89,10 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
/// Combine the provided files, rlibs, and native libraries into a single /// Combine the provided files, rlibs, and native libraries into a single
/// `Archive`. /// `Archive`.
fn build(mut self) { fn build(mut self) -> bool {
let kind = self.llvm_archive_kind().unwrap_or_else(|kind| { match self.build_with_llvm() {
self.config.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) Ok(any_members) => any_members,
}); Err(e) => self.sess.fatal(&format!("failed to build archive: {}", e)),
if let Err(e) = self.build_with_llvm(kind) {
self.config.sess.fatal(&format!("failed to build archive: {}", e));
} }
} }
@ -151,7 +108,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
output_path.with_extension("lib") output_path.with_extension("lib")
}; };
let target = &self.config.sess.target; let target = &self.sess.target;
let mingw_gnu_toolchain = target.vendor == "pc" let mingw_gnu_toolchain = target.vendor == "pc"
&& target.os == "windows" && target.os == "windows"
&& target.env == "gnu" && target.env == "gnu"
@ -160,7 +117,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
let import_name_and_ordinal_vector: Vec<(String, Option<u16>)> = dll_imports let import_name_and_ordinal_vector: Vec<(String, Option<u16>)> = dll_imports
.iter() .iter()
.map(|import: &DllImport| { .map(|import: &DllImport| {
if self.config.sess.target.arch == "x86" { if self.sess.target.arch == "x86" {
( (
LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain), LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain),
import.ordinal, import.ordinal,
@ -197,11 +154,11 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
match std::fs::write(&def_file_path, def_file_content) { match std::fs::write(&def_file_path, def_file_content) {
Ok(_) => {} Ok(_) => {}
Err(e) => { Err(e) => {
self.config.sess.fatal(&format!("Error writing .DEF file: {}", e)); self.sess.fatal(&format!("Error writing .DEF file: {}", e));
} }
}; };
let dlltool = find_binutils_dlltool(self.config.sess); let dlltool = find_binutils_dlltool(self.sess);
let result = std::process::Command::new(dlltool) let result = std::process::Command::new(dlltool)
.args([ .args([
"-d", "-d",
@ -215,9 +172,9 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
match result { match result {
Err(e) => { Err(e) => {
self.config.sess.fatal(&format!("Error calling dlltool: {}", e)); self.sess.fatal(&format!("Error calling dlltool: {}", e));
} }
Ok(output) if !output.status.success() => self.config.sess.fatal(&format!( Ok(output) if !output.status.success() => self.sess.fatal(&format!(
"Dlltool could not create import library: {}\n{}", "Dlltool could not create import library: {}\n{}",
String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr) String::from_utf8_lossy(&output.stderr)
@ -263,13 +220,13 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
output_path_z.as_ptr(), output_path_z.as_ptr(),
ffi_exports.as_ptr(), ffi_exports.as_ptr(),
ffi_exports.len(), ffi_exports.len(),
llvm_machine_type(&self.config.sess.target.arch) as u16, llvm_machine_type(&self.sess.target.arch) as u16,
!self.config.sess.target.is_like_msvc, !self.sess.target.is_like_msvc,
) )
}; };
if result == crate::llvm::LLVMRustResult::Failure { if result == crate::llvm::LLVMRustResult::Failure {
self.config.sess.fatal(&format!( self.sess.fatal(&format!(
"Error creating import library for {}: {}", "Error creating import library for {}: {}",
lib_name, lib_name,
llvm::last_error().unwrap_or("unknown LLVM error".to_string()) llvm::last_error().unwrap_or("unknown LLVM error".to_string())
@ -278,7 +235,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
}; };
self.add_archive(&output_path, |_| false).unwrap_or_else(|e| { self.add_archive(&output_path, |_| false).unwrap_or_else(|e| {
self.config.sess.fatal(&format!( self.sess.fatal(&format!(
"failed to add native library {}: {}", "failed to add native library {}: {}",
output_path.display(), output_path.display(),
e e
@ -288,46 +245,19 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
} }
impl<'a> LlvmArchiveBuilder<'a> { impl<'a> LlvmArchiveBuilder<'a> {
fn src_archive(&mut self) -> Option<&ArchiveRO> { fn build_with_llvm(&mut self) -> io::Result<bool> {
if let Some(ref a) = self.src_archive { let kind = &*self.sess.target.archive_format;
return a.as_ref(); let kind = kind.parse::<ArchiveKind>().map_err(|_| kind).unwrap_or_else(|kind| {
} self.sess.fatal(&format!("Don't know how to build archive of type: {}", kind))
let src = self.config.src.as_ref()?; });
self.src_archive = Some(ArchiveRO::open(src).ok());
self.src_archive.as_ref().unwrap().as_ref()
}
fn llvm_archive_kind(&self) -> Result<ArchiveKind, &str> {
let kind = &*self.config.sess.target.archive_format;
kind.parse().map_err(|_| kind)
}
fn build_with_llvm(&mut self, kind: ArchiveKind) -> io::Result<()> {
let removals = mem::take(&mut self.removals);
let mut additions = mem::take(&mut self.additions); let mut additions = mem::take(&mut self.additions);
let mut strings = Vec::new(); let mut strings = Vec::new();
let mut members = Vec::new(); let mut members = Vec::new();
let dst = CString::new(self.config.dst.to_str().unwrap())?; let dst = CString::new(self.dst.to_str().unwrap())?;
unsafe { unsafe {
if let Some(archive) = self.src_archive() {
for child in archive.iter() {
let child = child.map_err(string_to_io_error)?;
let Some(child_name) = child.name() else { continue };
if removals.iter().any(|r| r == child_name) {
continue;
}
let name = CString::new(child_name)?;
members.push(llvm::LLVMRustArchiveMemberNew(
ptr::null(),
name.as_ptr(),
Some(child.raw),
));
strings.push(name);
}
}
for addition in &mut additions { for addition in &mut additions {
match addition { match addition {
Addition::File { path, name_in_archive } => { Addition::File { path, name_in_archive } => {
@ -389,7 +319,7 @@ impl<'a> LlvmArchiveBuilder<'a> {
}; };
Err(io::Error::new(io::ErrorKind::Other, msg)) Err(io::Error::new(io::ErrorKind::Other, msg))
} else { } else {
Ok(()) Ok(!members.is_empty())
}; };
for member in members { for member in members {
llvm::LLVMRustArchiveMemberFree(member); llvm::LLVMRustArchiveMemberFree(member);

6
compiler/rustc_codegen_ssa/src/back/archive.rs
View File

@ -42,17 +42,15 @@ pub(super) fn find_library(
} }
pub trait ArchiveBuilder<'a> { pub trait ArchiveBuilder<'a> {
fn new(sess: &'a Session, output: &Path, input: Option<&Path>) -> Self; fn new(sess: &'a Session, output: &Path) -> Self;
fn add_file(&mut self, path: &Path); fn add_file(&mut self, path: &Path);
fn remove_file(&mut self, name: &str);
fn src_files(&mut self) -> Vec<String>;
fn add_archive<F>(&mut self, archive: &Path, skip: F) -> io::Result<()> fn add_archive<F>(&mut self, archive: &Path, skip: F) -> io::Result<()>
where where
F: FnMut(&str) -> bool + 'static; F: FnMut(&str) -> bool + 'static;
fn build(self); fn build(self) -> bool;
fn inject_dll_import_lib( fn inject_dll_import_lib(
&mut self, &mut self,

37
compiler/rustc_codegen_ssa/src/back/link.rs
View File

@ -270,7 +270,7 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(
let lib_search_paths = archive_search_paths(sess); let lib_search_paths = archive_search_paths(sess);
let mut ab = <B as ArchiveBuilder>::new(sess, out_filename, None); let mut ab = <B as ArchiveBuilder>::new(sess, out_filename);
let trailing_metadata = match flavor { let trailing_metadata = match flavor {
RlibFlavor::Normal => { RlibFlavor::Normal => {
@ -2466,17 +2466,19 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>(
let name = &name[3..name.len() - 5]; // chop off lib/.rlib let name = &name[3..name.len() - 5]; // chop off lib/.rlib
sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| { sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| {
let mut archive = <B as ArchiveBuilder>::new(sess, &dst, Some(cratepath)); let canonical_name = name.replace('-', "_");
let upstream_rust_objects_already_included =
are_upstream_rust_objects_already_included(sess);
let is_builtins = sess.target.no_builtins
|| !codegen_results.crate_info.is_no_builtins.contains(&cnum);
let mut any_objects = false; let mut archive = <B as ArchiveBuilder>::new(sess, &dst);
for f in archive.src_files() { if let Err(e) = archive.add_archive(cratepath, move |f| {
if f == METADATA_FILENAME { if f == METADATA_FILENAME {
archive.remove_file(&f); return true;
continue;
} }
let canonical = f.replace('-', "_"); let canonical = f.replace('-', "_");
let canonical_name = name.replace('-', "_");
let is_rust_object = let is_rust_object =
canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f); canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f);
@ -2490,23 +2492,20 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>(
// file, then we don't need the object file as it's part of the // file, then we don't need the object file as it's part of the
// LTO module. Note that `#![no_builtins]` is excluded from LTO, // LTO module. Note that `#![no_builtins]` is excluded from LTO,
// though, so we let that object file slide. // though, so we let that object file slide.
let skip_because_lto = are_upstream_rust_objects_already_included(sess) let skip_because_lto =
&& is_rust_object upstream_rust_objects_already_included && is_rust_object && is_builtins;
&& (sess.target.no_builtins
|| !codegen_results.crate_info.is_no_builtins.contains(&cnum));
if skip_because_cfg_say_so || skip_because_lto { if skip_because_cfg_say_so || skip_because_lto {
archive.remove_file(&f); return true;
} else {
any_objects = true;
} }
}
if !any_objects { false
return; }) {
sess.fatal(&format!("failed to build archive from rlib: {}", e));
}
if archive.build() {
link_upstream(&dst);
} }
archive.build();
link_upstream(&dst);
}); });
} }