mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #95867 - cjgillot:fixed-size, r=oli-obk

Browse Source 
Skip `Lazy` for some metadata tables

Some metadata tables encode their entries indirectly, through the Lazy construct. This is useful when dealing with variable length encoding, but incurs the extra cost of one u32.

Meanwhile, some fields can be encoded in a single u8, or can use a short fixed-length encoding. This PR proposes to do so, and avoid the overhead.
This commit is contained in:
bors 2022-04-12 16:12:48 +00:00
commit de56c295c3
5 changed files with 321 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
compiler/rustc_metadata/src/lib.rs
View File

@ -5,7 +5,9 @@
#![feature(nll)]
#![feature(once_cell)]
#![feature(proc_macro_internals)]
#![feature(macro_metavar_expr)]
#![feature(min_specialization)]
#![feature(slice_as_chunks)]
#![feature(try_blocks)]
#![feature(never_type)]
#![recursion_limit = "256"]

46
compiler/rustc_metadata/src/rmeta/decoder.rs
View File

@ -292,6 +292,18 @@ fn decode_query(
) -> T;
}
impl<'a, 'tcx, T> LazyQueryDecodable<'a, 'tcx, T> for T {
fn decode_query(self, _: CrateMetadataRef<'a>, _: TyCtxt<'tcx>, _: impl FnOnce() -> !) -> T {
self
}
}
impl<'a, 'tcx, T> LazyQueryDecodable<'a, 'tcx, T> for Option<T> {
fn decode_query(self, _: CrateMetadataRef<'a>, _: TyCtxt<'tcx>, err: impl FnOnce() -> !) -> T {
if let Some(l) = self { l } else { err() }
}
}
impl<'a, 'tcx, T> LazyQueryDecodable<'a, 'tcx, T> for Option<Lazy<T>>
where
T: Decodable<DecodeContext<'a, 'tcx>>,
@ -376,6 +388,17 @@ fn decode_query(
}
}
impl<'a, 'tcx> LazyQueryDecodable<'a, 'tcx, Option<DefId>> for Option<RawDefId> {
fn decode_query(
self,
cdata: CrateMetadataRef<'a>,
_: TyCtxt<'tcx>,
_: impl FnOnce() -> !,
) -> Option<DefId> {
self.map(|raw_def_id| raw_def_id.decode(cdata))
}
}
impl<'a, 'tcx> DecodeContext<'a, 'tcx> {
#[inline]
fn tcx(&self) -> TyCtxt<'tcx> {
@ -394,8 +417,9 @@ pub fn cdata(&self) -> CrateMetadataRef<'a> {
self.cdata.unwrap()
}
#[inline]
fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum {
if cnum == LOCAL_CRATE { self.cdata().cnum } else { self.cdata().cnum_map[cnum] }
self.cdata().map_encoded_cnum_to_current(cnum)
}
fn read_lazy_with_meta<T: ?Sized + LazyMeta>(&mut self, meta: T::Meta) -> Lazy<T> {
@ -706,8 +730,7 @@ fn decode(decoder: &mut DecodeContext<'a, 'tcx>) -> Self {
}
}
impl<'a, 'tcx, I: Idx, T: Decodable<DecodeContext<'a, 'tcx>>> Decodable<DecodeContext<'a, 'tcx>>
for Lazy<Table<I, T>>
impl<'a, 'tcx, I: Idx, T> Decodable<DecodeContext<'a, 'tcx>> for Lazy<Table<I, T>>
where
Option<T>: FixedSizeEncoding,
{
@ -844,6 +867,11 @@ fn maybe_kind(self, item_id: DefIndex) -> Option<EntryKind> {
self.root.tables.kind.get(self, item_id).map(|k| k.decode(self))
}
#[inline]
pub(super) fn map_encoded_cnum_to_current(self, cnum: CrateNum) -> CrateNum {
if cnum == LOCAL_CRATE { self.cnum } else { self.cnum_map[cnum] }
}
fn kind(self, item_id: DefIndex) -> EntryKind {
self.maybe_kind(item_id).unwrap_or_else(|| {
bug!(
@ -856,16 +884,14 @@ fn kind(self, item_id: DefIndex) -> EntryKind {
}
fn def_kind(self, item_id: DefIndex) -> DefKind {
self.root.tables.opt_def_kind.get(self, item_id).map(|k| k.decode(self)).unwrap_or_else(
|| {
self.root.tables.opt_def_kind.get(self, item_id).unwrap_or_else(|| {
bug!(
"CrateMetadata::def_kind({:?}): id not found, in crate {:?} with number {}",
item_id,
self.root.name,
self.cnum,
)
},
)
})
}
fn get_span(self, index: DefIndex, sess: &Session) -> Span {
@ -1449,9 +1475,9 @@ fn def_path_hash_unlocked(
index: DefIndex,
def_path_hashes: &mut FxHashMap<DefIndex, DefPathHash>,
) -> DefPathHash {
*def_path_hashes.entry(index).or_insert_with(|| {
self.root.tables.def_path_hashes.get(self, index).unwrap().decode(self)
})
*def_path_hashes
.entry(index)
.or_insert_with(|| self.root.tables.def_path_hashes.get(self, index).unwrap())
}
#[inline]

48
compiler/rustc_metadata/src/rmeta/encoder.rs
View File

@ -147,8 +147,7 @@ fn encode(&self, e: &mut EncodeContext<'a, 'tcx>) -> opaque::EncodeResult {
}
}
impl<'a, 'tcx, I: Idx, T: Encodable<EncodeContext<'a, 'tcx>>> Encodable<EncodeContext<'a, 'tcx>>
for Lazy<Table<I, T>>
impl<'a, 'tcx, I: Idx, T> Encodable<EncodeContext<'a, 'tcx>> for Lazy<Table<I, T>>
where
Option<T>: FixedSizeEncoding,
{
@ -461,16 +460,15 @@ fn encode_def_path_table(&mut self) {
.chain(self.tcx.resolutions(()).proc_macros.iter().map(|p| p.local_def_index))
{
let def_key = self.lazy(table.def_key(def_index));
let def_path_hash = self.lazy(table.def_path_hash(def_index));
let def_path_hash = table.def_path_hash(def_index);
self.tables.def_keys.set(def_index, def_key);
self.tables.def_path_hashes.set(def_index, def_path_hash);
}
} else {
for (def_index, def_key, def_path_hash) in table.enumerated_keys_and_path_hashes() {
let def_key = self.lazy(def_key);
let def_path_hash = self.lazy(def_path_hash);
self.tables.def_keys.set(def_index, def_key);
self.tables.def_path_hashes.set(def_index, def_path_hash);
self.tables.def_path_hashes.set(def_index, *def_path_hash);
}
}
}
@ -988,7 +986,7 @@ fn encode_def_ids(&mut self) {
let def_id = local_id.to_def_id();
let def_kind = tcx.opt_def_kind(local_id);
let Some(def_kind) = def_kind else { continue };
record!(self.tables.opt_def_kind[def_id] <- def_kind);
self.tables.opt_def_kind.set(def_id.index, def_kind);
record!(self.tables.def_span[def_id] <- tcx.def_span(def_id));
record!(self.tables.attributes[def_id] <- tcx.get_attrs(def_id));
record!(self.tables.expn_that_defined[def_id] <- self.tcx.expn_that_defined(def_id));
@ -1048,7 +1046,7 @@ fn encode_enum_variant_info(&mut self, def: ty::AdtDef<'tcx>, index: VariantIdx)
};
record!(self.tables.kind[def_id] <- EntryKind::Variant(self.lazy(data)));
record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
record!(self.tables.children[def_id] <- variant.fields.iter().map(|f| {
assert!(f.did.is_local());
f.did.index
@ -1078,7 +1076,7 @@ fn encode_enum_variant_ctor(&mut self, def: ty::AdtDef<'tcx>, index: VariantIdx)
};
record!(self.tables.kind[def_id] <- EntryKind::Variant(self.lazy(data)));
record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
self.encode_item_type(def_id);
if variant.ctor_kind == CtorKind::Fn {
record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
@ -1157,7 +1155,7 @@ fn encode_struct_ctor(&mut self, adt_def: ty::AdtDef<'tcx>, def_id: DefId) {
};
record!(self.tables.repr_options[def_id] <- adt_def.repr());
record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
record!(self.tables.kind[def_id] <- EntryKind::Struct(self.lazy(data)));
self.encode_item_type(def_id);
if variant.ctor_kind == CtorKind::Fn {
@ -1207,8 +1205,8 @@ fn encode_info_for_trait_item(&mut self, def_id: DefId) {
record!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body))
}
};
record!(self.tables.asyncness[def_id] <- m_sig.header.asyncness);
record!(self.tables.impl_constness[def_id] <- hir::Constness::NotConst);
self.tables.asyncness.set(def_id.index, m_sig.header.asyncness);
self.tables.impl_constness.set(def_id.index, hir::Constness::NotConst);
record!(self.tables.kind[def_id] <- EntryKind::AssocFn(self.lazy(AssocFnData {
container,
has_self: trait_item.fn_has_self_parameter,
@ -1265,7 +1263,7 @@ fn encode_info_for_impl_item(&mut self, def_id: DefId) {
}
ty::AssocKind::Fn => {
let hir::ImplItemKind::Fn(ref sig, body) = ast_item.kind else { bug!() };
record!(self.tables.asyncness[def_id] <- sig.header.asyncness);
self.tables.asyncness.set(def_id.index, sig.header.asyncness);
record!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body));
// Can be inside `impl const Trait`, so using sig.header.constness is not reliable
let constness = if self.tcx.is_const_fn_raw(def_id) {
@ -1273,7 +1271,7 @@ fn encode_info_for_impl_item(&mut self, def_id: DefId) {
} else {
hir::Constness::NotConst
};
record!(self.tables.impl_constness[def_id] <- constness);
self.tables.impl_constness.set(def_id.index, constness);
record!(self.tables.kind[def_id] <- EntryKind::AssocFn(self.lazy(AssocFnData {
container,
has_self: impl_item.fn_has_self_parameter,
@ -1286,7 +1284,7 @@ fn encode_info_for_impl_item(&mut self, def_id: DefId) {
self.encode_ident_span(def_id, impl_item.ident(self.tcx));
self.encode_item_type(def_id);
if let Some(trait_item_def_id) = impl_item.trait_item_def_id {
record!(self.tables.trait_item_def_id[def_id] <- trait_item_def_id);
self.tables.trait_item_def_id.set(def_id.index, trait_item_def_id.into());
}
if impl_item.kind == ty::AssocKind::Fn {
record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
@ -1394,9 +1392,9 @@ fn encode_info_for_item(&mut self, def_id: DefId, item: &'tcx hir::Item<'tcx>) {
EntryKind::Const
}
hir::ItemKind::Fn(ref sig, .., body) => {
record!(self.tables.asyncness[def_id] <- sig.header.asyncness);
self.tables.asyncness.set(def_id.index, sig.header.asyncness);
record!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body));
record!(self.tables.impl_constness[def_id] <- sig.header.constness);
self.tables.impl_constness.set(def_id.index, sig.header.constness);
EntryKind::Fn
}
hir::ItemKind::Macro(ref macro_def, _) => {
@ -1420,7 +1418,7 @@ fn encode_info_for_item(&mut self, def_id: DefId, item: &'tcx hir::Item<'tcx>) {
hir::ItemKind::Struct(ref struct_def, _) => {
let adt_def = self.tcx.adt_def(def_id);
record!(self.tables.repr_options[def_id] <- adt_def.repr());
record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
// Encode def_ids for each field and method
// for methods, write all the stuff get_trait_method
@ -1450,15 +1448,15 @@ fn encode_info_for_item(&mut self, def_id: DefId, item: &'tcx hir::Item<'tcx>) {
}))
}
hir::ItemKind::Impl(hir::Impl { defaultness, constness, .. }) => {
record!(self.tables.impl_defaultness[def_id] <- defaultness);
record!(self.tables.impl_constness[def_id] <- constness);
self.tables.impl_defaultness.set(def_id.index, defaultness);
self.tables.impl_constness.set(def_id.index, constness);
let trait_ref = self.tcx.impl_trait_ref(def_id);
if let Some(trait_ref) = trait_ref {
let trait_def = self.tcx.trait_def(trait_ref.def_id);
if let Some(mut an) = trait_def.ancestors(self.tcx, def_id).ok() {
if let Some(specialization_graph::Node::Impl(parent)) = an.nth(1) {
record!(self.tables.impl_parent[def_id] <- parent);
self.tables.impl_parent.set(def_id.index, parent.into());
}
}
@ -1472,7 +1470,7 @@ fn encode_info_for_item(&mut self, def_id: DefId, item: &'tcx hir::Item<'tcx>) {
}
let polarity = self.tcx.impl_polarity(def_id);
record!(self.tables.impl_polarity[def_id] <- polarity);
self.tables.impl_polarity.set(def_id.index, polarity);
EntryKind::Impl
}
@ -1644,7 +1642,7 @@ fn encode_proc_macros(&mut self) -> Option<ProcMacroData> {
self.tables.proc_macro_quoted_spans.set(i, span);
}
record!(self.tables.opt_def_kind[LOCAL_CRATE.as_def_id()] <- DefKind::Mod);
self.tables.opt_def_kind.set(LOCAL_CRATE.as_def_id().index, DefKind::Mod);
record!(self.tables.def_span[LOCAL_CRATE.as_def_id()] <- tcx.def_span(LOCAL_CRATE.as_def_id()));
record!(self.tables.attributes[LOCAL_CRATE.as_def_id()] <- tcx.get_attrs(LOCAL_CRATE.as_def_id()));
record!(self.tables.visibility[LOCAL_CRATE.as_def_id()] <- tcx.visibility(LOCAL_CRATE.as_def_id()));
@ -1685,7 +1683,7 @@ fn encode_proc_macros(&mut self) -> Option<ProcMacroData> {
def_key.disambiguated_data.data = DefPathData::MacroNs(name);
let def_id = id.to_def_id();
record!(self.tables.opt_def_kind[def_id] <- DefKind::Macro(macro_kind));
self.tables.opt_def_kind.set(def_id.index, DefKind::Macro(macro_kind));
record!(self.tables.kind[def_id] <- EntryKind::ProcMacro(macro_kind));
record!(self.tables.attributes[def_id] <- attrs);
record!(self.tables.def_keys[def_id] <- def_key);
@ -1886,14 +1884,14 @@ fn encode_info_for_foreign_item(&mut self, def_id: DefId, nitem: &hir::ForeignIt
match nitem.kind {
hir::ForeignItemKind::Fn(_, ref names, _) => {
record!(self.tables.asyncness[def_id] <- hir::IsAsync::NotAsync);
self.tables.asyncness.set(def_id.index, hir::IsAsync::NotAsync);
record!(self.tables.fn_arg_names[def_id] <- *names);
let constness = if self.tcx.is_const_fn_raw(def_id) {
hir::Constness::Const
} else {
hir::Constness::NotConst
};
record!(self.tables.impl_constness[def_id] <- constness);
self.tables.impl_constness.set(def_id.index, constness);
record!(self.tables.kind[def_id] <- EntryKind::ForeignFn);
}
hir::ForeignItemKind::Static(..) => {

42
compiler/rustc_metadata/src/rmeta/mod.rs
View File

@ -1,3 +1,4 @@
use crate::creader::CrateMetadataRef;
use decoder::Metadata;
use def_path_hash_map::DefPathHashMapRef;
use table::{Table, TableBuilder};
@ -8,7 +9,7 @@
use rustc_data_structures::sync::MetadataRef;
use rustc_hir as hir;
use rustc_hir::def::{CtorKind, DefKind};
use rustc_hir::def_id::{DefId, DefIndex, DefPathHash, StableCrateId};
use rustc_hir::def_id::{CrateNum, DefId, DefIndex, DefPathHash, StableCrateId};
use rustc_hir::definitions::DefKey;
use rustc_hir::lang_items;
use rustc_index::{bit_set::FiniteBitSet, vec::IndexVec};
@ -237,6 +238,29 @@ macro_rules! Lazy {
symbol_mangling_version: SymbolManglingVersion,
}
/// On-disk representation of `DefId`.
/// This creates a type-safe way to enforce that we remap the CrateNum between the on-disk
/// representation and the compilation session.
#[derive(Copy, Clone)]
crate struct RawDefId {
krate: u32,
index: u32,
}
impl Into<RawDefId> for DefId {
fn into(self) -> RawDefId {
RawDefId { krate: self.krate.as_u32(), index: self.index.as_u32() }
}
}
impl RawDefId {
fn decode(self, cdata: CrateMetadataRef<'_>) -> DefId {
let krate = CrateNum::from_u32(self.krate);
let krate = cdata.map_encoded_cnum_to_current(krate);
DefId { krate, index: DefIndex::from_u32(self.index) }
}
}
#[derive(Encodable, Decodable)]
crate struct CrateDep {
pub name: Symbol,
@ -286,7 +310,7 @@ fn encode(&self, buf: &mut Encoder) -> LazyTables<'tcx> {
attributes: Table<DefIndex, Lazy<[ast::Attribute]>>,
children: Table<DefIndex, Lazy<[DefIndex]>>,
opt_def_kind: Table<DefIndex, Lazy<DefKind>>,
opt_def_kind: Table<DefIndex, DefKind>,
visibility: Table<DefIndex, Lazy<ty::Visibility>>,
def_span: Table<DefIndex, Lazy<Span>>,
def_ident_span: Table<DefIndex, Lazy<Span>>,
@ -309,20 +333,20 @@ fn encode(&self, buf: &mut Encoder) -> LazyTables<'tcx> {
mir_for_ctfe: Table<DefIndex, Lazy!(mir::Body<'tcx>)>,
promoted_mir: Table<DefIndex, Lazy!(IndexVec<mir::Promoted, mir::Body<'tcx>>)>,
thir_abstract_const: Table<DefIndex, Lazy!(&'tcx [thir::abstract_const::Node<'tcx>])>,
impl_parent: Table<DefIndex, Lazy!(DefId)>,
impl_polarity: Table<DefIndex, Lazy!(ty::ImplPolarity)>,
impl_constness: Table<DefIndex, Lazy!(hir::Constness)>,
impl_defaultness: Table<DefIndex, Lazy!(hir::Defaultness)>,
impl_parent: Table<DefIndex, RawDefId>,
impl_polarity: Table<DefIndex, ty::ImplPolarity>,
impl_constness: Table<DefIndex, hir::Constness>,
impl_defaultness: Table<DefIndex, hir::Defaultness>,
// FIXME(eddyb) perhaps compute this on the fly if cheap enough?
coerce_unsized_info: Table<DefIndex, Lazy!(ty::adjustment::CoerceUnsizedInfo)>,
mir_const_qualif: Table<DefIndex, Lazy!(mir::ConstQualifs)>,
rendered_const: Table<DefIndex, Lazy!(String)>,
asyncness: Table<DefIndex, Lazy!(hir::IsAsync)>,
asyncness: Table<DefIndex, hir::IsAsync>,
fn_arg_names: Table<DefIndex, Lazy!([Ident])>,
generator_kind: Table<DefIndex, Lazy!(hir::GeneratorKind)>,
trait_def: Table<DefIndex, Lazy!(ty::TraitDef)>,
trait_item_def_id: Table<DefIndex, Lazy<DefId>>,
trait_item_def_id: Table<DefIndex, RawDefId>,
inherent_impls: Table<DefIndex, Lazy<[DefIndex]>>,
expn_that_defined: Table<DefIndex, Lazy<ExpnId>>,
unused_generic_params: Table<DefIndex, Lazy<FiniteBitSet<u32>>>,
@ -332,7 +356,7 @@ fn encode(&self, buf: &mut Encoder) -> LazyTables<'tcx> {
// `DefPathTable` up front, since we may only ever use a few
// definitions from any given crate.
def_keys: Table<DefIndex, Lazy<DefKey>>,
def_path_hashes: Table<DefIndex, Lazy<DefPathHash>>,
def_path_hashes: Table<DefIndex, DefPathHash>,
proc_macro_quoted_spans: Table<usize, Lazy<Span>>,
}

312
compiler/rustc_metadata/src/rmeta/table.rs
View File

@ -1,8 +1,11 @@
use crate::rmeta::*;
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_hir::def::{CtorKind, CtorOf};
use rustc_index::vec::Idx;
use rustc_serialize::opaque::Encoder;
use rustc_serialize::Encoder as _;
use rustc_span::hygiene::MacroKind;
use std::convert::TryInto;
use std::marker::PhantomData;
use std::num::NonZeroUsize;
@ -13,66 +16,173 @@
/// Unchecked invariant: `Self::default()` should encode as `[0; BYTE_LEN]`,
/// but this has no impact on safety.
pub(super) trait FixedSizeEncoding: Default {
const BYTE_LEN: usize;
/// This should be `[u8; BYTE_LEN]`;
type ByteArray;
// FIXME(eddyb) convert to and from `[u8; Self::BYTE_LEN]` instead,
// once that starts being allowed by the compiler (i.e. lazy normalization).
fn from_bytes(b: &[u8]) -> Self;
fn write_to_bytes(self, b: &mut [u8]);
// FIXME(eddyb) make these generic functions, or at least defaults here.
// (same problem as above, needs `[u8; Self::BYTE_LEN]`)
// For now, a macro (`fixed_size_encoding_byte_len_and_defaults`) is used.
/// Read a `Self` value (encoded as `Self::BYTE_LEN` bytes),
/// from `&b[i * Self::BYTE_LEN..]`, returning `None` if `i`
/// is not in bounds, or `Some(Self::from_bytes(...))` otherwise.
fn maybe_read_from_bytes_at(b: &[u8], i: usize) -> Option<Self>;
/// Write a `Self` value (encoded as `Self::BYTE_LEN` bytes),
/// at `&mut b[i * Self::BYTE_LEN..]`, using `Self::write_to_bytes`.
fn write_to_bytes_at(self, b: &mut [u8], i: usize);
}
// HACK(eddyb) this shouldn't be needed (see comments on the methods above).
macro_rules! fixed_size_encoding_byte_len_and_defaults {
($byte_len:expr) => {
const BYTE_LEN: usize = $byte_len;
fn maybe_read_from_bytes_at(b: &[u8], i: usize) -> Option<Self> {
const BYTE_LEN: usize = $byte_len;
// HACK(eddyb) ideally this would be done with fully safe code,
// but slicing `[u8]` with `i * N..` is optimized worse, due to the
// possibility of `i * N` overflowing, than indexing `[[u8; N]]`.
let b = unsafe {
std::slice::from_raw_parts(b.as_ptr() as *const [u8; BYTE_LEN], b.len() / BYTE_LEN)
};
b.get(i).map(|b| FixedSizeEncoding::from_bytes(b))
}
fn write_to_bytes_at(self, b: &mut [u8], i: usize) {
const BYTE_LEN: usize = $byte_len;
// HACK(eddyb) ideally this would be done with fully safe code,
// see similar comment in `read_from_bytes_at` for why it can't yet.
let b = unsafe {
std::slice::from_raw_parts_mut(
b.as_mut_ptr() as *mut [u8; BYTE_LEN],
b.len() / BYTE_LEN,
)
};
self.write_to_bytes(&mut b[i]);
}
};
fn from_bytes(b: &Self::ByteArray) -> Self;
fn write_to_bytes(self, b: &mut Self::ByteArray);
}
impl FixedSizeEncoding for u32 {
fixed_size_encoding_byte_len_and_defaults!(4);
type ByteArray = [u8; 4];
fn from_bytes(b: &[u8]) -> Self {
let mut bytes = [0; Self::BYTE_LEN];
bytes.copy_from_slice(&b[..Self::BYTE_LEN]);
Self::from_le_bytes(bytes)
#[inline]
fn from_bytes(b: &[u8; 4]) -> Self {
Self::from_le_bytes(*b)
}
fn write_to_bytes(self, b: &mut [u8]) {
b[..Self::BYTE_LEN].copy_from_slice(&self.to_le_bytes());
#[inline]
fn write_to_bytes(self, b: &mut [u8; 4]) {
*b = self.to_le_bytes();
}
}
macro_rules! fixed_size_enum {
($ty:ty { $(($($pat:tt)*))* }) => {
impl FixedSizeEncoding for Option<$ty> {
type ByteArray = [u8;1];
#[inline]
fn from_bytes(b: &[u8;1]) -> Self {
use $ty::*;
if b[0] == 0 {
return None;
}
match b[0] - 1 {
$(${index()} => Some($($pat)*),)*
_ => panic!("Unexpected ImplPolarity code: {:?}", b[0]),
}
}
#[inline]
fn write_to_bytes(self, b: &mut [u8;1]) {
use $ty::*;
b[0] = match self {
None => 0,
$(Some($($pat)*) => 1 + ${index()},)*
}
}
}
}
}
fixed_size_enum! {
DefKind {
( Mod )
( Struct )
( Union )
( Enum )
( Variant )
( Trait )
( TyAlias )
( ForeignTy )
( TraitAlias )
( AssocTy )
( TyParam )
( Fn )
( Const )
( ConstParam )
( AssocFn )
( AssocConst )
( ExternCrate )
( Use )
( ForeignMod )
( AnonConst )
( InlineConst )
( OpaqueTy )
( Field )
( LifetimeParam )
( GlobalAsm )
( Impl )
( Closure )
( Generator )
( Static(ast::Mutability::Not) )
( Static(ast::Mutability::Mut) )
( Ctor(CtorOf::Struct, CtorKind::Fn) )
( Ctor(CtorOf::Struct, CtorKind::Const) )
( Ctor(CtorOf::Struct, CtorKind::Fictive) )
( Ctor(CtorOf::Variant, CtorKind::Fn) )
( Ctor(CtorOf::Variant, CtorKind::Const) )
( Ctor(CtorOf::Variant, CtorKind::Fictive) )
( Macro(MacroKind::Bang) )
( Macro(MacroKind::Attr) )
( Macro(MacroKind::Derive) )
}
}
fixed_size_enum! {
ty::ImplPolarity {
( Positive )
( Negative )
( Reservation )
}
}
fixed_size_enum! {
hir::Constness {
( NotConst )
( Const )
}
}
fixed_size_enum! {
hir::Defaultness {
( Final )
( Default { has_value: false } )
( Default { has_value: true } )
}
}
fixed_size_enum! {
hir::IsAsync {
( NotAsync )
( Async )
}
}
// We directly encode `DefPathHash` because a `Lazy` would encur a 25% cost.
impl FixedSizeEncoding for Option<DefPathHash> {
type ByteArray = [u8; 16];
#[inline]
fn from_bytes(b: &[u8; 16]) -> Self {
Some(DefPathHash(Fingerprint::from_le_bytes(*b)))
}
#[inline]
fn write_to_bytes(self, b: &mut [u8; 16]) {
let Some(DefPathHash(fingerprint)) = self else {
panic!("Trying to encode absent DefPathHash.")
};
*b = fingerprint.to_le_bytes();
}
}
// We directly encode RawDefId because using a `Lazy` would incur a 50% overhead in the worst case.
impl FixedSizeEncoding for Option<RawDefId> {
type ByteArray = [u8; 8];
#[inline]
fn from_bytes(b: &[u8; 8]) -> Self {
let krate = u32::from_le_bytes(b[0..4].try_into().unwrap());
let index = u32::from_le_bytes(b[4..8].try_into().unwrap());
if krate == 0 {
return None;
}
Some(RawDefId { krate: krate - 1, index })
}
#[inline]
fn write_to_bytes(self, b: &mut [u8; 8]) {
match self {
None => *b = [0; 8],
Some(RawDefId { krate, index }) => {
// CrateNum is less than `CrateNum::MAX_AS_U32`.
debug_assert!(krate < u32::MAX);
b[0..4].copy_from_slice(&(1 + krate).to_le_bytes());
b[4..8].copy_from_slice(&index.to_le_bytes());
}
}
}
}
@ -80,44 +190,51 @@ fn write_to_bytes(self, b: &mut [u8]) {
// generic `Lazy<T>` impl, but in the general case we might not need / want to
// fit every `usize` in `u32`.
impl<T> FixedSizeEncoding for Option<Lazy<T>> {
fixed_size_encoding_byte_len_and_defaults!(u32::BYTE_LEN);
type ByteArray = [u8; 4];
fn from_bytes(b: &[u8]) -> Self {
Some(Lazy::from_position(NonZeroUsize::new(u32::from_bytes(b) as usize)?))
#[inline]
fn from_bytes(b: &[u8; 4]) -> Self {
let position = NonZeroUsize::new(u32::from_bytes(b) as usize)?;
Some(Lazy::from_position(position))
}
fn write_to_bytes(self, b: &mut [u8]) {
#[inline]
fn write_to_bytes(self, b: &mut [u8; 4]) {
let position = self.map_or(0, |lazy| lazy.position.get());
let position: u32 = position.try_into().unwrap();
position.write_to_bytes(b)
}
}
impl<T> FixedSizeEncoding for Option<Lazy<[T]>> {
fixed_size_encoding_byte_len_and_defaults!(u32::BYTE_LEN * 2);
type ByteArray = [u8; 8];
fn from_bytes(b: &[u8]) -> Self {
Some(Lazy::from_position_and_meta(
<Option<Lazy<T>>>::from_bytes(b)?.position,
u32::from_bytes(&b[u32::BYTE_LEN..]) as usize,
))
#[inline]
fn from_bytes(b: &[u8; 8]) -> Self {
let ([ref position_bytes, ref meta_bytes],[])= b.as_chunks::<4>() else { panic!() };
let position = NonZeroUsize::new(u32::from_bytes(position_bytes) as usize)?;
let len = u32::from_bytes(meta_bytes) as usize;
Some(Lazy::from_position_and_meta(position, len))
}
fn write_to_bytes(self, b: &mut [u8]) {
self.map(|lazy| Lazy::<T>::from_position(lazy.position)).write_to_bytes(b);
#[inline]
fn write_to_bytes(self, b: &mut [u8; 8]) {
let ([ref mut position_bytes, ref mut meta_bytes],[])= b.as_chunks_mut::<4>() else { panic!() };
let position = self.map_or(0, |lazy| lazy.position.get());
let position: u32 = position.try_into().unwrap();
position.write_to_bytes(position_bytes);
let len = self.map_or(0, |lazy| lazy.meta);
let len: u32 = len.try_into().unwrap();
len.write_to_bytes(&mut b[u32::BYTE_LEN..]);
len.write_to_bytes(meta_bytes);
}
}
/// Random-access table (i.e. offering constant-time `get`/`set`), similar to
/// `Vec<Option<T>>`, but without requiring encoding or decoding all the values
/// eagerly and in-order.
/// A total of `(max_idx + 1) * <Option<T> as FixedSizeEncoding>::BYTE_LEN` bytes
/// A total of `(max_idx + 1)` times `Option<T> as FixedSizeEncoding>::ByteArray`
/// are used for a table, where `max_idx` is the largest index passed to
/// `TableBuilder::set`.
pub(super) struct Table<I: Idx, T>
@ -135,12 +252,8 @@ pub(super) struct TableBuilder<I: Idx, T>
where
Option<T>: FixedSizeEncoding,
{
// FIXME(eddyb) use `IndexVec<I, [u8; <Option<T>>::BYTE_LEN]>` instead of
// `Vec<u8>`, once that starts working (i.e. lazy normalization).
// Then again, that has the downside of not allowing `TableBuilder::encode` to
// obtain a `&[u8]` entirely in safe code, for writing the bytes out.
bytes: Vec<u8>,
_marker: PhantomData<(fn(&I), T)>,
blocks: IndexVec<I, <Option<T> as FixedSizeEncoding>::ByteArray>,
_marker: PhantomData<T>,
}
impl<I: Idx, T> Default for TableBuilder<I, T>
@ -148,7 +261,7 @@ impl<I: Idx, T> Default for TableBuilder<I, T>
Option<T>: FixedSizeEncoding,
{
fn default() -> Self {
TableBuilder { bytes: vec![], _marker: PhantomData }
TableBuilder { blocks: Default::default(), _marker: PhantomData }
}
}
@ -156,25 +269,29 @@ impl<I: Idx, T> TableBuilder<I, T>
where
Option<T>: FixedSizeEncoding,
{
pub(crate) fn set(&mut self, i: I, value: T) {
pub(crate) fn set<const N: usize>(&mut self, i: I, value: T)
where
Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>,
{
// FIXME(eddyb) investigate more compact encodings for sparse tables.
// On the PR @michaelwoerister mentioned:
// > Space requirements could perhaps be optimized by using the HAMT `popcnt`
// > trick (i.e. divide things into buckets of 32 or 64 items and then
// > store bit-masks of which item in each bucket is actually serialized).
let i = i.index();
let needed = (i + 1) * <Option<T>>::BYTE_LEN;
if self.bytes.len() < needed {
self.bytes.resize(needed, 0);
self.blocks.ensure_contains_elem(i, || [0; N]);
Some(value).write_to_bytes(&mut self.blocks[i]);
}
Some(value).write_to_bytes_at(&mut self.bytes, i);
}
pub(crate) fn encode(&self, buf: &mut Encoder) -> Lazy<Table<I, T>> {
pub(crate) fn encode<const N: usize>(&self, buf: &mut Encoder) -> Lazy<Table<I, T>>
where
Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>,
{
let pos = buf.position();
buf.emit_raw_bytes(&self.bytes).unwrap();
Lazy::from_position_and_meta(NonZeroUsize::new(pos as usize).unwrap(), self.bytes.len())
for block in &self.blocks {
buf.emit_raw_bytes(block).unwrap();
}
let num_bytes = self.blocks.len() * N;
Lazy::from_position_and_meta(NonZeroUsize::new(pos as usize).unwrap(), num_bytes)
}
}
@ -182,6 +299,7 @@ impl<I: Idx, T> LazyMeta for Table<I, T>
where
Option<T>: FixedSizeEncoding,
{
/// Number of bytes in the data stream.
type Meta = usize;
}
@ -191,16 +309,28 @@ impl<I: Idx, T> Lazy<Table<I, T>>
{
/// Given the metadata, extract out the value at a particular index (if any).
#[inline(never)]
pub(super) fn get<'a, 'tcx, M: Metadata<'a, 'tcx>>(&self, metadata: M, i: I) -> Option<T> {
pub(super) fn get<'a, 'tcx, M: Metadata<'a, 'tcx>, const N: usize>(
&self,
metadata: M,
i: I,
) -> Option<T>
where
Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>,
{
debug!("Table::lookup: index={:?} len={:?}", i, self.meta);
let start = self.position.get();
let bytes = &metadata.blob()[start..start + self.meta];
<Option<T>>::maybe_read_from_bytes_at(bytes, i.index())?
let (bytes, []) = bytes.as_chunks::<N>() else { panic!() };
let bytes = bytes.get(i.index())?;
FixedSizeEncoding::from_bytes(bytes)
}
/// Size of the table in entries, including possible gaps.
pub(super) fn size(&self) -> usize {
self.meta / <Option<T>>::BYTE_LEN
pub(super) fn size<const N: usize>(&self) -> usize
where
Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>,
{
self.meta / N
}
}