diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index bb2dd290c6d..e2b07fad6e7 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -654,7 +654,7 @@ impl<'a, 'tcx, T> Decodable> for LazyValue { impl<'a, 'tcx, T> Decodable> for LazyArray { fn decode(decoder: &mut DecodeContext<'a, 'tcx>) -> Self { let len = decoder.read_usize(); - if len == 0 { LazyArray::empty() } else { decoder.read_lazy_array(len) } + if len == 0 { LazyArray::default() } else { decoder.read_lazy_array(len) } } } @@ -864,7 +864,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .tables .children .get(self, index) - .unwrap_or_else(LazyArray::empty) + .expect("fields are not encoded for a variant") .decode(self) .map(|index| ty::FieldDef { did: self.local_def_id(index), @@ -896,7 +896,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .tables .children .get(self, item_id) - .unwrap_or_else(LazyArray::empty) + .expect("variants are not encoded for an enum") .decode(self) .filter_map(|index| { let kind = self.def_kind(index); @@ -1045,7 +1045,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .tables .fn_arg_names .get(self, id) - .unwrap_or_else(LazyArray::empty) + .expect("argument names not encoded for a function") .decode((self, sess)) .nth(0) .map_or(false, |ident| ident.name == kw::SelfLower) @@ -1060,7 +1060,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .tables .children .get(self, id) - .unwrap_or_else(LazyArray::empty) + .expect("associated items not encoded for an item") .decode((self, sess)) .map(move |child_index| self.local_def_id(child_index)) } @@ -1068,13 +1068,12 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { fn get_associated_item(self, id: DefIndex, sess: &'a Session) -> ty::AssocItem { let name = self.item_name(id); - let kind = match self.def_kind(id) { - DefKind::AssocConst => ty::AssocKind::Const, - DefKind::AssocFn => ty::AssocKind::Fn, - DefKind::AssocTy => ty::AssocKind::Type, + let (kind, has_self) = match self.def_kind(id) { + DefKind::AssocConst => (ty::AssocKind::Const, false), + DefKind::AssocFn => (ty::AssocKind::Fn, self.get_fn_has_self_parameter(id, sess)), + DefKind::AssocTy => (ty::AssocKind::Type, false), _ => bug!("cannot get associated-item of `{:?}`", self.def_key(id)), }; - let has_self = self.get_fn_has_self_parameter(id, sess); let container = self.root.tables.assoc_container.get(self, id).unwrap(); ty::AssocItem { @@ -1131,7 +1130,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .tables .children .get(self, id) - .unwrap_or_else(LazyArray::empty) + .expect("fields not encoded for a struct") .decode(self) .map(move |index| respan(self.get_span(index, sess), self.item_name(index))) } @@ -1144,7 +1143,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .tables .children .get(self, id) - .unwrap_or_else(LazyArray::empty) + .expect("fields not encoded for a struct") .decode(self) .map(move |field_index| self.get_visibility(field_index)) } @@ -1159,7 +1158,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .tables .inherent_impls .get(self, id) - .unwrap_or_else(LazyArray::empty) .decode(self) .map(|index| self.local_def_id(index)), ) @@ -1174,7 +1172,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .tables .inherent_impls .get(self, ty_index) - .unwrap_or_else(LazyArray::empty) .decode(self) .map(move |impl_index| (ty_def_id, self.local_def_id(impl_index))) }) @@ -1322,7 +1319,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { ) -> DefPathHash { *def_path_hashes .entry(index) - .or_insert_with(|| self.root.tables.def_path_hashes.get(self, index).unwrap()) + .or_insert_with(|| self.root.tables.def_path_hashes.get(self, index)) } #[inline] diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index 9b1401f4a44..07cc84ab953 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -1,6 +1,7 @@ use crate::creader::{CStore, LoadedMacro}; use crate::foreign_modules; use crate::native_libs; +use crate::rmeta::table::IsDefault; use crate::rmeta::AttrFlags; use rustc_ast as ast; @@ -88,6 +89,14 @@ macro_rules! provide_one { } } }; + ($tcx:ident, $def_id:ident, $other:ident, $cdata:ident, $name:ident => { table_defaulted_array }) => { + provide_one! { + $tcx, $def_id, $other, $cdata, $name => { + let lazy = $cdata.root.tables.$name.get($cdata, $def_id.index); + if lazy.is_default() { &[] } else { $tcx.arena.alloc_from_iter(lazy.decode(($cdata, $tcx))) } + } + } + }; ($tcx:ident, $def_id:ident, $other:ident, $cdata:ident, $name:ident => { table_direct }) => { provide_one! { $tcx, $def_id, $other, $cdata, $name => { @@ -187,10 +196,10 @@ impl IntoArgs for (CrateNum, SimplifiedType) { } provide! { tcx, def_id, other, cdata, - explicit_item_bounds => { table } + explicit_item_bounds => { table_defaulted_array } explicit_predicates_of => { table } generics_of => { table } - inferred_outlives_of => { table } + inferred_outlives_of => { table_defaulted_array } super_predicates_of => { table } type_of => { table } variances_of => { table } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 9d8f14058f6..85e9ae9a983 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -76,13 +76,13 @@ pub(super) struct EncodeContext<'a, 'tcx> { symbol_table: FxHashMap, } -/// If the current crate is a proc-macro, returns early with `LazyArray::empty()`. +/// If the current crate is a proc-macro, returns early with `LazyArray::default()`. /// This is useful for skipping the encoding of things that aren't needed /// for proc-macro crates. macro_rules! empty_proc_macro { ($self:ident) => { if $self.is_proc_macro { - return LazyArray::empty(); + return LazyArray::default(); } }; } @@ -365,21 +365,31 @@ impl<'a, 'tcx> TyEncoder for EncodeContext<'a, 'tcx> { } } -// Shorthand for `$self.$tables.$table.set($def_id.index, $self.lazy_value($value))`, which would +// Shorthand for `$self.$tables.$table.set_some($def_id.index, $self.lazy_value($value))`, which would // normally need extra variables to avoid errors about multiple mutable borrows. macro_rules! record { ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ { let value = $value; let lazy = $self.lazy(value); - $self.$tables.$table.set($def_id.index, lazy); + $self.$tables.$table.set_some($def_id.index, lazy); } }}; } -// Shorthand for `$self.$tables.$table.set($def_id.index, $self.lazy_value($value))`, which would +// Shorthand for `$self.$tables.$table.set_some($def_id.index, $self.lazy_value($value))`, which would // normally need extra variables to avoid errors about multiple mutable borrows. macro_rules! record_array { + ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ + { + let value = $value; + let lazy = $self.lazy_array(value); + $self.$tables.$table.set_some($def_id.index, lazy); + } + }}; +} + +macro_rules! record_defaulted_array { ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ { let value = $value; @@ -467,13 +477,13 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { { let def_key = self.lazy(table.def_key(def_index)); let def_path_hash = table.def_path_hash(def_index); - self.tables.def_keys.set(def_index, def_key); + self.tables.def_keys.set_some(def_index, def_key); self.tables.def_path_hashes.set(def_index, def_path_hash); } } else { for (def_index, def_key, def_path_hash) in table.enumerated_keys_and_path_hashes() { let def_key = self.lazy(def_key); - self.tables.def_keys.set(def_index, def_key); + self.tables.def_keys.set_some(def_index, def_key); self.tables.def_path_hashes.set(def_index, *def_path_hash); } } @@ -548,7 +558,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { let on_disk_index: u32 = on_disk_index.try_into().expect("cannot export more than U32_MAX files"); - adapted.set(on_disk_index, self.lazy(source_file)); + adapted.set_some(on_disk_index, self.lazy(source_file)); } adapted.encode(&mut self.opaque) @@ -1147,9 +1157,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { if state.is_doc_hidden { attr_flags |= AttrFlags::IS_DOC_HIDDEN; } - if !attr_flags.is_empty() { - self.tables.attr_flags.set_nullable(def_id.local_def_index, attr_flags); - } + self.tables.attr_flags.set(def_id.local_def_index, attr_flags); } fn encode_def_ids(&mut self) { @@ -1161,7 +1169,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { let def_id = local_id.to_def_id(); let def_kind = tcx.opt_def_kind(local_id); let Some(def_kind) = def_kind else { continue }; - self.tables.opt_def_kind.set(def_id.index, def_kind); + self.tables.opt_def_kind.set_some(def_id.index, def_kind); let def_span = tcx.def_span(local_id); record!(self.tables.def_span[def_id] <- def_span); self.encode_attrs(local_id); @@ -1192,9 +1200,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { record!(self.tables.generics_of[def_id] <- g); record!(self.tables.explicit_predicates_of[def_id] <- self.tcx.explicit_predicates_of(def_id)); let inferred_outlives = self.tcx.inferred_outlives_of(def_id); - if !inferred_outlives.is_empty() { - record_array!(self.tables.inferred_outlives_of[def_id] <- inferred_outlives); - } + record_defaulted_array!(self.tables.inferred_outlives_of[def_id] <- inferred_outlives); } if should_encode_type(tcx, local_id, def_kind) { record!(self.tables.type_of[def_id] <- self.tcx.type_of(def_id)); @@ -1215,15 +1221,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { record!(self.tables.trait_impl_trait_tys[def_id] <- table); } } + let inherent_impls = tcx.with_stable_hashing_context(|hcx| { tcx.crate_inherent_impls(()).inherent_impls.to_sorted(&hcx, true) }); - - for (def_id, implementations) in inherent_impls { - if implementations.is_empty() { - continue; - } - record_array!(self.tables.inherent_impls[def_id.to_def_id()] <- implementations.iter().map(|&def_id| { + for (def_id, impls) in inherent_impls { + record_defaulted_array!(self.tables.inherent_impls[def_id.to_def_id()] <- impls.iter().map(|def_id| { assert!(def_id.is_local()); def_id.index })); @@ -1264,14 +1267,14 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { }; record!(self.tables.variant_data[variant.def_id] <- data); - self.tables.constness.set(variant.def_id.index, hir::Constness::Const); + self.tables.constness.set_some(variant.def_id.index, hir::Constness::Const); record_array!(self.tables.children[variant.def_id] <- variant.fields.iter().map(|f| { assert!(f.did.is_local()); f.did.index })); if let Some((CtorKind::Fn, ctor_def_id)) = variant.ctor { - self.tables.constness.set(ctor_def_id.index, hir::Constness::Const); + self.tables.constness.set_some(ctor_def_id.index, hir::Constness::Const); let fn_sig = tcx.fn_sig(ctor_def_id); record!(self.tables.fn_sig[ctor_def_id] <- fn_sig); // FIXME only encode signature for ctor_def_id @@ -1332,9 +1335,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { fn encode_explicit_item_bounds(&mut self, def_id: DefId) { debug!("EncodeContext::encode_explicit_item_bounds({:?})", def_id); let bounds = self.tcx.explicit_item_bounds(def_id); - if !bounds.is_empty() { - record_array!(self.tables.explicit_item_bounds[def_id] <- bounds); - } + record_defaulted_array!(self.tables.explicit_item_bounds[def_id] <- bounds); } fn encode_info_for_trait_item(&mut self, def_id: DefId) { @@ -1342,16 +1343,16 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { let tcx = self.tcx; let impl_defaultness = tcx.impl_defaultness(def_id.expect_local()); - self.tables.impl_defaultness.set(def_id.index, impl_defaultness); + self.tables.impl_defaultness.set_some(def_id.index, impl_defaultness); let trait_item = tcx.associated_item(def_id); - self.tables.assoc_container.set(def_id.index, trait_item.container); + self.tables.assoc_container.set_some(def_id.index, trait_item.container); match trait_item.kind { ty::AssocKind::Const => {} ty::AssocKind::Fn => { record_array!(self.tables.fn_arg_names[def_id] <- tcx.fn_arg_names(def_id)); - self.tables.asyncness.set(def_id.index, tcx.asyncness(def_id)); - self.tables.constness.set(def_id.index, hir::Constness::NotConst); + self.tables.asyncness.set_some(def_id.index, tcx.asyncness(def_id)); + self.tables.constness.set_some(def_id.index, hir::Constness::NotConst); } ty::AssocKind::Type => { self.encode_explicit_item_bounds(def_id); @@ -1367,14 +1368,14 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { let tcx = self.tcx; let ast_item = self.tcx.hir().expect_impl_item(def_id.expect_local()); - self.tables.impl_defaultness.set(def_id.index, ast_item.defaultness); + self.tables.impl_defaultness.set_some(def_id.index, ast_item.defaultness); let impl_item = self.tcx.associated_item(def_id); - self.tables.assoc_container.set(def_id.index, impl_item.container); + self.tables.assoc_container.set_some(def_id.index, impl_item.container); match impl_item.kind { ty::AssocKind::Fn => { let hir::ImplItemKind::Fn(ref sig, body) = ast_item.kind else { bug!() }; - self.tables.asyncness.set(def_id.index, sig.header.asyncness); + self.tables.asyncness.set_some(def_id.index, sig.header.asyncness); record_array!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body)); // Can be inside `impl const Trait`, so using sig.header.constness is not reliable let constness = if self.tcx.is_const_fn_raw(def_id) { @@ -1382,18 +1383,16 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { } else { hir::Constness::NotConst }; - self.tables.constness.set(def_id.index, constness); + self.tables.constness.set_some(def_id.index, constness); } ty::AssocKind::Const | ty::AssocKind::Type => {} } if let Some(trait_item_def_id) = impl_item.trait_item_def_id { - self.tables.trait_item_def_id.set(def_id.index, trait_item_def_id.into()); + self.tables.trait_item_def_id.set_some(def_id.index, trait_item_def_id.into()); } if impl_item.kind == ty::AssocKind::Fn { record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id)); - if tcx.is_intrinsic(def_id) { - self.tables.is_intrinsic.set_nullable(def_id.index, true); - } + self.tables.is_intrinsic.set(def_id.index, tcx.is_intrinsic(def_id)); } } @@ -1522,14 +1521,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { match item.kind { hir::ItemKind::Fn(ref sig, .., body) => { - self.tables.asyncness.set(def_id.index, sig.header.asyncness); + self.tables.asyncness.set_some(def_id.index, sig.header.asyncness); record_array!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body)); - self.tables.constness.set(def_id.index, sig.header.constness); + self.tables.constness.set_some(def_id.index, sig.header.constness); } hir::ItemKind::Macro(ref macro_def, _) => { - if macro_def.macro_rules { - self.tables.is_macro_rules.set_nullable(def_id.index, true); - } + self.tables.is_macro_rules.set(def_id.index, macro_def.macro_rules); record!(self.tables.macro_definition[def_id] <- &*macro_def.body); } hir::ItemKind::Mod(ref m) => { @@ -1537,20 +1534,20 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { } hir::ItemKind::OpaqueTy(ref opaque) => { self.encode_explicit_item_bounds(def_id); - if matches!(opaque.origin, hir::OpaqueTyOrigin::TyAlias) { - self.tables.is_type_alias_impl_trait.set_nullable(def_id.index, true); - } + self.tables + .is_type_alias_impl_trait + .set(def_id.index, matches!(opaque.origin, hir::OpaqueTyOrigin::TyAlias)); } hir::ItemKind::Impl(hir::Impl { defaultness, constness, .. }) => { - self.tables.impl_defaultness.set(def_id.index, *defaultness); - self.tables.constness.set(def_id.index, *constness); + self.tables.impl_defaultness.set_some(def_id.index, *defaultness); + self.tables.constness.set_some(def_id.index, *constness); let trait_ref = self.tcx.impl_trait_ref(def_id).map(ty::EarlyBinder::skip_binder); if let Some(trait_ref) = trait_ref { let trait_def = self.tcx.trait_def(trait_ref.def_id); if let Ok(mut an) = trait_def.ancestors(self.tcx, def_id) { if let Some(specialization_graph::Node::Impl(parent)) = an.nth(1) { - self.tables.impl_parent.set(def_id.index, parent.into()); + self.tables.impl_parent.set_some(def_id.index, parent.into()); } } @@ -1564,7 +1561,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { } let polarity = self.tcx.impl_polarity(def_id); - self.tables.impl_polarity.set(def_id.index, polarity); + self.tables.impl_polarity.set_some(def_id.index, polarity); } hir::ItemKind::Trait(..) => { let trait_def = self.tcx.trait_def(def_id); @@ -1601,9 +1598,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { } if let hir::ItemKind::Fn(..) = item.kind { record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id)); - if tcx.is_intrinsic(def_id) { - self.tables.is_intrinsic.set_nullable(def_id.index, true); - } + self.tables.is_intrinsic.set(def_id.index, tcx.is_intrinsic(def_id)); } if let hir::ItemKind::Impl { .. } = item.kind { if let Some(trait_ref) = self.tcx.impl_trait_ref(def_id) { @@ -1650,7 +1645,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { ty::Closure(_, substs) => { let constness = self.tcx.constness(def_id.to_def_id()); - self.tables.constness.set(def_id.to_def_id().index, constness); + self.tables.constness.set_some(def_id.to_def_id().index, constness); record!(self.tables.fn_sig[def_id.to_def_id()] <- ty::EarlyBinder(substs.as_closure().sig())); } @@ -1678,12 +1673,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { self.hygiene_ctxt.encode( &mut (&mut *self, &mut syntax_contexts, &mut expn_data_table, &mut expn_hash_table), |(this, syntax_contexts, _, _), index, ctxt_data| { - syntax_contexts.set(index, this.lazy(ctxt_data)); + syntax_contexts.set_some(index, this.lazy(ctxt_data)); }, |(this, _, expn_data_table, expn_hash_table), index, expn_data, hash| { if let Some(index) = index.as_local() { - expn_data_table.set(index.as_raw(), this.lazy(expn_data)); - expn_hash_table.set(index.as_raw(), this.lazy(hash)); + expn_data_table.set_some(index.as_raw(), this.lazy(expn_data)); + expn_hash_table.set_some(index.as_raw(), this.lazy(hash)); } }, ); @@ -1708,10 +1703,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { let spans = self.tcx.sess.parse_sess.proc_macro_quoted_spans(); for (i, span) in spans.into_iter().enumerate() { let span = self.lazy(span); - self.tables.proc_macro_quoted_spans.set(i, span); + self.tables.proc_macro_quoted_spans.set_some(i, span); } - self.tables.opt_def_kind.set(LOCAL_CRATE.as_def_id().index, DefKind::Mod); + self.tables.opt_def_kind.set_some(LOCAL_CRATE.as_def_id().index, DefKind::Mod); record!(self.tables.def_span[LOCAL_CRATE.as_def_id()] <- tcx.def_span(LOCAL_CRATE.as_def_id())); self.encode_attrs(LOCAL_CRATE.as_def_id().expect_local()); let vis = tcx.local_visibility(CRATE_DEF_ID).map_id(|def_id| def_id.local_def_index); @@ -1753,8 +1748,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { def_key.disambiguated_data.data = DefPathData::MacroNs(name); let def_id = id.to_def_id(); - self.tables.opt_def_kind.set(def_id.index, DefKind::Macro(macro_kind)); - self.tables.proc_macro.set(def_id.index, macro_kind); + self.tables.opt_def_kind.set_some(def_id.index, DefKind::Macro(macro_kind)); + self.tables.proc_macro.set_some(def_id.index, macro_kind); self.encode_attrs(id); record!(self.tables.def_keys[def_id] <- def_key); record!(self.tables.def_ident_span[def_id] <- span); @@ -1969,7 +1964,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { Linkage::Static => Some(LinkagePreference::RequireStatic), })); } - LazyArray::empty() + LazyArray::default() } fn encode_info_for_foreign_item(&mut self, def_id: DefId, nitem: &hir::ForeignItem<'_>) { @@ -1979,22 +1974,20 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { match nitem.kind { hir::ForeignItemKind::Fn(_, ref names, _) => { - self.tables.asyncness.set(def_id.index, hir::IsAsync::NotAsync); + self.tables.asyncness.set_some(def_id.index, hir::IsAsync::NotAsync); record_array!(self.tables.fn_arg_names[def_id] <- *names); let constness = if self.tcx.is_const_fn_raw(def_id) { hir::Constness::Const } else { hir::Constness::NotConst }; - self.tables.constness.set(def_id.index, constness); + self.tables.constness.set_some(def_id.index, constness); record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id)); } hir::ForeignItemKind::Static(..) | hir::ForeignItemKind::Type => {} } if let hir::ForeignItemKind::Fn(..) = nitem.kind { - if tcx.is_intrinsic(def_id) { - self.tables.is_intrinsic.set_nullable(def_id.index, true); - } + self.tables.is_intrinsic.set(def_id.index, tcx.is_intrinsic(def_id)); } } } diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 37af9e64e9a..a74aa381d9e 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -115,14 +115,16 @@ impl ParameterizedOverTcx for LazyArray { type Value<'tcx> = LazyArray>; } +impl Default for LazyArray { + fn default() -> LazyArray { + LazyArray::from_position_and_num_elems(NonZeroUsize::new(1).unwrap(), 0) + } +} + impl LazyArray { fn from_position_and_num_elems(position: NonZeroUsize, num_elems: usize) -> LazyArray { LazyArray { position, num_elems, _marker: PhantomData } } - - fn empty() -> LazyArray { - LazyArray::from_position_and_num_elems(NonZeroUsize::new(1).unwrap(), 0) - } } /// A list of lazily-decoded values, with the added capability of random access. @@ -316,7 +318,7 @@ pub(crate) struct IncoherentImpls { /// Define `LazyTables` and `TableBuilders` at the same time. macro_rules! define_tables { ( - - nullable: $($name1:ident: Table<$IDX1:ty, $T1:ty>,)+ + - defaulted: $($name1:ident: Table<$IDX1:ty, $T1:ty>,)+ - optional: $($name2:ident: Table<$IDX2:ty, $T2:ty>,)+ ) => { #[derive(MetadataEncodable, MetadataDecodable)] @@ -343,11 +345,15 @@ macro_rules! define_tables { } define_tables! { -- nullable: +- defaulted: is_intrinsic: Table, is_macro_rules: Table, is_type_alias_impl_trait: Table, attr_flags: Table, + def_path_hashes: Table, + explicit_item_bounds: Table, Span)>>, + inferred_outlives_of: Table, Span)>>, + inherent_impls: Table>, - optional: attributes: Table>, @@ -360,12 +366,8 @@ define_tables! { lookup_const_stability: Table>, lookup_default_body_stability: Table>, lookup_deprecation_entry: Table>, - // As an optimization, a missing entry indicates an empty `&[]`. - explicit_item_bounds: Table, Span)>>, explicit_predicates_of: Table>>, generics_of: Table>, - // As an optimization, a missing entry indicates an empty `&[]`. - inferred_outlives_of: Table, Span)>>, super_predicates_of: Table>>, type_of: Table>>, variances_of: Table>, @@ -393,7 +395,6 @@ define_tables! { generator_kind: Table>, trait_def: Table>, trait_item_def_id: Table, - inherent_impls: Table>, expn_that_defined: Table>, unused_generic_params: Table>, params_in_repr: Table>>, @@ -403,7 +404,6 @@ define_tables! { // `DefPathTable` up front, since we may only ever use a few // definitions from any given crate. def_keys: Table>, - def_path_hashes: Table, proc_macro_quoted_spans: Table>, generator_diagnostic_data: Table>>, variant_data: Table>, diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs index 70dbf6476e2..99bec570600 100644 --- a/compiler/rustc_metadata/src/rmeta/table.rs +++ b/compiler/rustc_metadata/src/rmeta/table.rs @@ -10,11 +10,51 @@ use rustc_span::hygiene::MacroKind; use std::marker::PhantomData; use std::num::NonZeroUsize; +pub(super) trait IsDefault: Default { + fn is_default(&self) -> bool; +} + +impl IsDefault for Option { + fn is_default(&self) -> bool { + self.is_none() + } +} + +impl IsDefault for AttrFlags { + fn is_default(&self) -> bool { + self.is_empty() + } +} + +impl IsDefault for bool { + fn is_default(&self) -> bool { + !self + } +} + +impl IsDefault for u32 { + fn is_default(&self) -> bool { + *self == 0 + } +} + +impl IsDefault for LazyArray { + fn is_default(&self) -> bool { + self.num_elems == 0 + } +} + +impl IsDefault for DefPathHash { + fn is_default(&self) -> bool { + self.0 == Fingerprint::ZERO + } +} + /// Helper trait, for encoding to, and decoding from, a fixed number of bytes. /// Used mainly for Lazy positions and lengths. /// Unchecked invariant: `Self::default()` should encode as `[0; BYTE_LEN]`, /// but this has no impact on safety. -pub(super) trait FixedSizeEncoding: Default { +pub(super) trait FixedSizeEncoding: IsDefault { /// This should be `[u8; BYTE_LEN]`; /// Cannot use an associated `const BYTE_LEN: usize` instead due to const eval limitations. type ByteArray; @@ -23,6 +63,8 @@ pub(super) trait FixedSizeEncoding: Default { fn write_to_bytes(self, b: &mut Self::ByteArray); } +/// This implementation is not used generically, but for reading/writing +/// concrete `u32` fields in `Lazy*` structures, which may be zero. impl FixedSizeEncoding for u32 { type ByteArray = [u8; 4]; @@ -58,7 +100,7 @@ macro_rules! fixed_size_enum { fn write_to_bytes(self, b: &mut [u8;1]) { use $ty::*; b[0] = match self { - None => 0, + None => unreachable!(), $(Some($($pat)*) => 1 + ${index()},)* } } @@ -155,20 +197,18 @@ fixed_size_enum! { } // We directly encode `DefPathHash` because a `LazyValue` would incur a 25% cost. -impl FixedSizeEncoding for Option { +impl FixedSizeEncoding for DefPathHash { type ByteArray = [u8; 16]; #[inline] fn from_bytes(b: &[u8; 16]) -> Self { - Some(DefPathHash(Fingerprint::from_le_bytes(*b))) + DefPathHash(Fingerprint::from_le_bytes(*b)) } #[inline] fn write_to_bytes(self, b: &mut [u8; 16]) { - let Some(DefPathHash(fingerprint)) = self else { - panic!("Trying to encode absent DefPathHash.") - }; - *b = fingerprint.to_le_bytes(); + debug_assert!(!self.is_default()); + *b = self.0.to_le_bytes(); } } @@ -179,17 +219,17 @@ impl FixedSizeEncoding for Option { #[inline] fn from_bytes(b: &[u8; 8]) -> Self { let krate = u32::from_le_bytes(b[0..4].try_into().unwrap()); - let index = u32::from_le_bytes(b[4..8].try_into().unwrap()); if krate == 0 { return None; } + let index = u32::from_le_bytes(b[4..8].try_into().unwrap()); Some(RawDefId { krate: krate - 1, index }) } #[inline] fn write_to_bytes(self, b: &mut [u8; 8]) { match self { - None => *b = [0; 8], + None => unreachable!(), Some(RawDefId { krate, index }) => { // CrateNum is less than `CrateNum::MAX_AS_U32`. debug_assert!(krate < u32::MAX); @@ -210,6 +250,7 @@ impl FixedSizeEncoding for AttrFlags { #[inline] fn write_to_bytes(self, b: &mut [u8; 1]) { + debug_assert!(!self.is_default()); b[0] = self.bits(); } } @@ -224,6 +265,7 @@ impl FixedSizeEncoding for bool { #[inline] fn write_to_bytes(self, b: &mut [u8; 1]) { + debug_assert!(!self.is_default()); b[0] = self as u8 } } @@ -242,9 +284,54 @@ impl FixedSizeEncoding for Option> { #[inline] fn write_to_bytes(self, b: &mut [u8; 4]) { - let position = self.map_or(0, |lazy| lazy.position.get()); + match self { + None => unreachable!(), + Some(lazy) => { + let position = lazy.position.get(); + let position: u32 = position.try_into().unwrap(); + position.write_to_bytes(b) + } + } + } +} + +impl LazyArray { + #[inline] + fn write_to_bytes_impl(self, b: &mut [u8; 8]) { + let ([position_bytes, meta_bytes],[])= b.as_chunks_mut::<4>() else { panic!() }; + + let position = self.position.get(); let position: u32 = position.try_into().unwrap(); - position.write_to_bytes(b) + position.write_to_bytes(position_bytes); + + let len = self.num_elems; + let len: u32 = len.try_into().unwrap(); + len.write_to_bytes(meta_bytes); + } + + fn from_bytes_impl(position_bytes: &[u8; 4], meta_bytes: &[u8; 4]) -> Option> { + let position = NonZeroUsize::new(u32::from_bytes(position_bytes) as usize)?; + let len = u32::from_bytes(meta_bytes) as usize; + Some(LazyArray::from_position_and_num_elems(position, len)) + } +} + +impl FixedSizeEncoding for LazyArray { + type ByteArray = [u8; 8]; + + #[inline] + fn from_bytes(b: &[u8; 8]) -> Self { + let ([position_bytes, meta_bytes],[])= b.as_chunks::<4>() else { panic!() }; + if *meta_bytes == [0; 4] { + return Default::default(); + } + LazyArray::from_bytes_impl(position_bytes, meta_bytes).unwrap() + } + + #[inline] + fn write_to_bytes(self, b: &mut [u8; 8]) { + assert!(!self.is_default()); + self.write_to_bytes_impl(b) } } @@ -253,23 +340,16 @@ impl FixedSizeEncoding for Option> { #[inline] fn from_bytes(b: &[u8; 8]) -> Self { - let ([ref position_bytes, ref meta_bytes],[])= b.as_chunks::<4>() else { panic!() }; - let position = NonZeroUsize::new(u32::from_bytes(position_bytes) as usize)?; - let len = u32::from_bytes(meta_bytes) as usize; - Some(LazyArray::from_position_and_num_elems(position, len)) + let ([position_bytes, meta_bytes],[])= b.as_chunks::<4>() else { panic!() }; + LazyArray::from_bytes_impl(position_bytes, meta_bytes) } #[inline] fn write_to_bytes(self, b: &mut [u8; 8]) { - let ([ref mut position_bytes, ref mut meta_bytes],[])= b.as_chunks_mut::<4>() else { panic!() }; - - let position = self.map_or(0, |lazy| lazy.position.get()); - let position: u32 = position.try_into().unwrap(); - position.write_to_bytes(position_bytes); - - let len = self.map_or(0, |lazy| lazy.num_elems); - let len: u32 = len.try_into().unwrap(); - len.write_to_bytes(meta_bytes); + match self { + None => unreachable!(), + Some(lazy) => lazy.write_to_bytes_impl(b), + } } } @@ -289,20 +369,27 @@ impl TableBuilder> where Option: FixedSizeEncoding, { - pub(crate) fn set(&mut self, i: I, value: T) { - self.set_nullable(i, Some(value)) + pub(crate) fn set_some(&mut self, i: I, value: T) { + self.set(i, Some(value)) } } impl> TableBuilder { - pub(crate) fn set_nullable(&mut self, i: I, value: T) { - // FIXME(eddyb) investigate more compact encodings for sparse tables. - // On the PR @michaelwoerister mentioned: - // > Space requirements could perhaps be optimized by using the HAMT `popcnt` - // > trick (i.e. divide things into buckets of 32 or 64 items and then - // > store bit-masks of which item in each bucket is actually serialized). - self.blocks.ensure_contains_elem(i, || [0; N]); - value.write_to_bytes(&mut self.blocks[i]); + /// Sets the table value if it is not default. + /// ATTENTION: For optimization default values are simply ignored by this function, because + /// right now metadata tables never need to reset non-default values to default. If such need + /// arises in the future then a new method (e.g. `clear` or `reset`) will need to be introduced + /// for doing that explicitly. + pub(crate) fn set(&mut self, i: I, value: T) { + if !value.is_default() { + // FIXME(eddyb) investigate more compact encodings for sparse tables. + // On the PR @michaelwoerister mentioned: + // > Space requirements could perhaps be optimized by using the HAMT `popcnt` + // > trick (i.e. divide things into buckets of 32 or 64 items and then + // > store bit-masks of which item in each bucket is actually serialized). + self.blocks.ensure_contains_elem(i, || [0; N]); + value.write_to_bytes(&mut self.blocks[i]); + } } pub(crate) fn encode(&self, buf: &mut FileEncoder) -> LazyTable { @@ -331,10 +418,7 @@ where let start = self.position.get(); let bytes = &metadata.blob()[start..start + self.encoded_size]; let (bytes, []) = bytes.as_chunks::() else { panic!() }; - match bytes.get(i.index()) { - Some(bytes) => FixedSizeEncoding::from_bytes(bytes), - None => FixedSizeEncoding::from_bytes(&[0; N]), - } + bytes.get(i.index()).map_or_else(Default::default, FixedSizeEncoding::from_bytes) } /// Size of the table in entries, including possible gaps. diff --git a/compiler/rustc_span/src/def_id.rs b/compiler/rustc_span/src/def_id.rs index 7c5e1427d1e..cdda052f529 100644 --- a/compiler/rustc_span/src/def_id.rs +++ b/compiler/rustc_span/src/def_id.rs @@ -119,6 +119,12 @@ impl DefPathHash { } } +impl Default for DefPathHash { + fn default() -> Self { + DefPathHash(Fingerprint::ZERO) + } +} + impl Borrow for DefPathHash { #[inline] fn borrow(&self) -> &Fingerprint { diff --git a/compiler/rustc_ty_utils/src/assoc.rs b/compiler/rustc_ty_utils/src/assoc.rs index 424b52309d3..a6e0f13f698 100644 --- a/compiler/rustc_ty_utils/src/assoc.rs +++ b/compiler/rustc_ty_utils/src/assoc.rs @@ -22,14 +22,17 @@ fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: DefId) -> &[DefId] { hir::ItemKind::Impl(ref impl_) => tcx.arena.alloc_from_iter( impl_.items.iter().map(|impl_item_ref| impl_item_ref.id.owner_id.to_def_id()), ), - hir::ItemKind::TraitAlias(..) => &[], _ => span_bug!(item.span, "associated_item_def_ids: not impl or trait"), } } fn associated_items(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AssocItems<'_> { - let items = tcx.associated_item_def_ids(def_id).iter().map(|did| tcx.associated_item(*did)); - ty::AssocItems::new(items) + if tcx.is_trait_alias(def_id) { + ty::AssocItems::new(Vec::new()) + } else { + let items = tcx.associated_item_def_ids(def_id).iter().map(|did| tcx.associated_item(*did)); + ty::AssocItems::new(items) + } } fn impl_item_implementor_ids(tcx: TyCtxt<'_>, impl_id: DefId) -> FxHashMap {