From 2ae197d5fcee57fb5bdc575a561fabeaee5eb5dd Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Mon, 10 Jul 2023 10:45:24 -0400 Subject: [PATCH 1/3] Adapt table sizes to the contents --- compiler/rustc_metadata/src/rmeta/decoder.rs | 7 +- compiler/rustc_metadata/src/rmeta/encoder.rs | 3 +- compiler/rustc_metadata/src/rmeta/mod.rs | 8 +- compiler/rustc_metadata/src/rmeta/table.rs | 129 +++++++++++++------ 4 files changed, 102 insertions(+), 45 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 8fa1d365728..d18e8c0fd3a 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -360,8 +360,8 @@ impl<'a, 'tcx> DecodeContext<'a, 'tcx> { self.read_lazy_offset_then(|pos| LazyArray::from_position_and_num_elems(pos, len)) } - fn read_lazy_table(&mut self, len: usize) -> LazyTable { - self.read_lazy_offset_then(|pos| LazyTable::from_position_and_encoded_size(pos, len)) + fn read_lazy_table(&mut self, width: usize, len: usize) -> LazyTable { + self.read_lazy_offset_then(|pos| LazyTable::from_position_and_encoded_size(pos, width, len)) } #[inline] @@ -665,8 +665,9 @@ impl<'a, 'tcx, T> Decodable> for LazyArray { impl<'a, 'tcx, I: Idx, T> Decodable> for LazyTable { fn decode(decoder: &mut DecodeContext<'a, 'tcx>) -> Self { + let width = decoder.read_usize(); let len = decoder.read_usize(); - decoder.read_lazy_table(len) + decoder.read_lazy_table(width, len) } } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index fc1062a48b4..3bb3116bdd7 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -129,7 +129,8 @@ impl<'a, 'tcx, T> Encodable> for LazyArray { impl<'a, 'tcx, I, T> Encodable> for LazyTable { fn encode(&self, e: &mut EncodeContext<'a, 'tcx>) { - e.emit_usize(self.encoded_size); + e.emit_usize(self.width); + e.emit_usize(self.len); e.emit_lazy_distance(self.position); } } diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 9cffd96f4a3..1bd86e54307 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -136,7 +136,8 @@ impl LazyArray { /// eagerly and in-order. struct LazyTable { position: NonZeroUsize, - encoded_size: usize, + width: usize, + len: usize, _marker: PhantomData T>, } @@ -147,9 +148,10 @@ impl ParameterizedOverTcx for LazyTable LazyTable { fn from_position_and_encoded_size( position: NonZeroUsize, - encoded_size: usize, + width: usize, + len: usize, ) -> LazyTable { - LazyTable { position, encoded_size, _marker: PhantomData } + LazyTable { position, width, len, _marker: PhantomData } } } diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs index 4287799a8e6..41be8434ead 100644 --- a/compiler/rustc_metadata/src/rmeta/table.rs +++ b/compiler/rustc_metadata/src/rmeta/table.rs @@ -38,6 +38,12 @@ impl IsDefault for u32 { } } +impl IsDefault for u64 { + fn is_default(&self) -> bool { + *self == 0 + } +} + impl IsDefault for LazyArray { fn is_default(&self) -> bool { self.num_elems == 0 @@ -89,6 +95,20 @@ impl FixedSizeEncoding for u32 { } } +impl FixedSizeEncoding for u64 { + type ByteArray = [u8; 8]; + + #[inline] + fn from_bytes(b: &[u8; 8]) -> Self { + Self::from_le_bytes(*b) + } + + #[inline] + fn write_to_bytes(self, b: &mut [u8; 8]) { + *b = self.to_le_bytes(); + } +} + macro_rules! fixed_size_enum { ($ty:ty { $(($($pat:tt)*))* }) => { impl FixedSizeEncoding for Option<$ty> { @@ -299,21 +319,21 @@ impl FixedSizeEncoding for UnusedGenericParams { // generic `LazyValue` impl, but in the general case we might not need / want // to fit every `usize` in `u32`. impl FixedSizeEncoding for Option> { - type ByteArray = [u8; 4]; + type ByteArray = [u8; 8]; #[inline] - fn from_bytes(b: &[u8; 4]) -> Self { - let position = NonZeroUsize::new(u32::from_bytes(b) as usize)?; + fn from_bytes(b: &[u8; 8]) -> Self { + let position = NonZeroUsize::new(u64::from_bytes(b) as usize)?; Some(LazyValue::from_position(position)) } #[inline] - fn write_to_bytes(self, b: &mut [u8; 4]) { + fn write_to_bytes(self, b: &mut [u8; 8]) { match self { None => unreachable!(), Some(lazy) => { let position = lazy.position.get(); - let position: u32 = position.try_into().unwrap(); + let position: u64 = position.try_into().unwrap(); position.write_to_bytes(b) } } @@ -322,55 +342,67 @@ impl FixedSizeEncoding for Option> { impl LazyArray { #[inline] - fn write_to_bytes_impl(self, b: &mut [u8; 8]) { - let ([position_bytes, meta_bytes], []) = b.as_chunks_mut::<4>() else { panic!() }; + fn write_to_bytes_impl(self, b: &mut [u8; 16]) { + let position = (self.position.get() as u64).to_le_bytes(); + let len = (self.num_elems as u64).to_le_bytes(); - let position = self.position.get(); - let position: u32 = position.try_into().unwrap(); - position.write_to_bytes(position_bytes); - - let len = self.num_elems; - let len: u32 = len.try_into().unwrap(); - len.write_to_bytes(meta_bytes); + for i in 0..8 { + b[2 * i] = position[i]; + b[2 * i + 1] = len[i]; + } } - fn from_bytes_impl(position_bytes: &[u8; 4], meta_bytes: &[u8; 4]) -> Option> { - let position = NonZeroUsize::new(u32::from_bytes(position_bytes) as usize)?; - let len = u32::from_bytes(meta_bytes) as usize; + fn from_bytes_impl(position: &[u8; 8], meta: &[u8; 8]) -> Option> { + let position = NonZeroUsize::new(u64::from_bytes(&position) as usize)?; + let len = u64::from_bytes(&meta) as usize; Some(LazyArray::from_position_and_num_elems(position, len)) } } impl FixedSizeEncoding for LazyArray { - type ByteArray = [u8; 8]; + type ByteArray = [u8; 16]; #[inline] - fn from_bytes(b: &[u8; 8]) -> Self { - let ([position_bytes, meta_bytes], []) = b.as_chunks::<4>() else { panic!() }; - if *meta_bytes == [0; 4] { + fn from_bytes(b: &[u8; 16]) -> Self { + let mut position = [0u8; 8]; + let mut meta = [0u8; 8]; + + for i in 0..8 { + position[i] = b[2 * i]; + meta[i] = b[2 * i + 1]; + } + + if meta == [0; 8] { return Default::default(); } - LazyArray::from_bytes_impl(position_bytes, meta_bytes).unwrap() + LazyArray::from_bytes_impl(&position, &meta).unwrap() } #[inline] - fn write_to_bytes(self, b: &mut [u8; 8]) { + fn write_to_bytes(self, b: &mut [u8; 16]) { assert!(!self.is_default()); self.write_to_bytes_impl(b) } } impl FixedSizeEncoding for Option> { - type ByteArray = [u8; 8]; + type ByteArray = [u8; 16]; #[inline] - fn from_bytes(b: &[u8; 8]) -> Self { - let ([position_bytes, meta_bytes], []) = b.as_chunks::<4>() else { panic!() }; - LazyArray::from_bytes_impl(position_bytes, meta_bytes) + fn from_bytes(b: &[u8; 16]) -> Self { + let mut position = [0u8; 8]; + let mut meta = [0u8; 8]; + + for i in 0..8 { + position[i] = b[2 * i]; + meta[i] = b[2 * i + 1]; + } + + LazyArray::from_bytes_impl(&position, &meta) } #[inline] - fn write_to_bytes(self, b: &mut [u8; 8]) { + fn write_to_bytes(self, b: &mut [u8; 16]) { match self { None => unreachable!(), Some(lazy) => lazy.write_to_bytes_impl(b), @@ -380,13 +412,14 @@ impl FixedSizeEncoding for Option> { /// Helper for constructing a table's serialization (also see `Table`). pub(super) struct TableBuilder { + width: usize, blocks: IndexVec, _marker: PhantomData, } impl Default for TableBuilder { fn default() -> Self { - TableBuilder { blocks: Default::default(), _marker: PhantomData } + TableBuilder { width: 0, blocks: Default::default(), _marker: PhantomData } } } @@ -414,22 +447,33 @@ impl> TableBui // > store bit-masks of which item in each bucket is actually serialized). let block = self.blocks.ensure_contains_elem(i, || [0; N]); value.write_to_bytes(block); + if self.width != N { + let width = N - trailing_zeros(block); + self.width = self.width.max(width); + } } } pub(crate) fn encode(&self, buf: &mut FileEncoder) -> LazyTable { let pos = buf.position(); + + let width = self.width; for block in &self.blocks { - buf.emit_raw_bytes(block); + buf.emit_raw_bytes(&block[..width]); } - let num_bytes = self.blocks.len() * N; + LazyTable::from_position_and_encoded_size( NonZeroUsize::new(pos as usize).unwrap(), - num_bytes, + width, + self.blocks.len(), ) } } +fn trailing_zeros(x: &[u8]) -> usize { + x.iter().rev().take_while(|b| **b == 0).count() +} + impl + ParameterizedOverTcx> LazyTable where @@ -438,16 +482,25 @@ where /// Given the metadata, extract out the value at a particular index (if any). #[inline(never)] pub(super) fn get<'a, 'tcx, M: Metadata<'a, 'tcx>>(&self, metadata: M, i: I) -> T::Value<'tcx> { - trace!("LazyTable::lookup: index={:?} len={:?}", i, self.encoded_size); + trace!("LazyTable::lookup: index={:?} len={:?}", i, self.len); - let start = self.position.get(); - let bytes = &metadata.blob()[start..start + self.encoded_size]; - let (bytes, []) = bytes.as_chunks::() else { panic!() }; - bytes.get(i.index()).map_or_else(Default::default, FixedSizeEncoding::from_bytes) + // Access past the end of the table returns a Default + if i.index() >= self.len { + return Default::default(); + } + + let width = self.width; + let start = self.position.get() + (width * i.index()); + let end = start + width; + let bytes = &metadata.blob()[start..end]; + + let mut fixed = [0u8; N]; + fixed[..width].copy_from_slice(bytes); + FixedSizeEncoding::from_bytes(&fixed) } /// Size of the table in entries, including possible gaps. pub(super) fn size(&self) -> usize { - self.encoded_size / N + self.len } } From e0103a7b3cd52457c698376e52d82bd087e3052c Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Fri, 14 Jul 2023 07:50:02 -0400 Subject: [PATCH 2/3] Micro-optimize --- compiler/rustc_metadata/src/rmeta/decoder.rs | 7 +++++++ compiler/rustc_metadata/src/rmeta/table.rs | 11 +++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index d18e8c0fd3a..c5fbec24b51 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -250,6 +250,7 @@ impl<'a, 'tcx> Metadata<'a, 'tcx> for (CrateMetadataRef<'a>, TyCtxt<'tcx>) { } impl LazyValue { + #[inline] fn decode<'a, 'tcx, M: Metadata<'a, 'tcx>>(self, metadata: M) -> T::Value<'tcx> where T::Value<'tcx>: Decodable>, @@ -294,6 +295,7 @@ unsafe impl<'a, 'tcx, T: Decodable>> TrustedLen } impl LazyArray { + #[inline] fn decode<'a, 'tcx, M: Metadata<'a, 'tcx>>( self, metadata: M, @@ -420,6 +422,7 @@ impl<'a, 'tcx> TyDecoder for DecodeContext<'a, 'tcx> { } impl<'a, 'tcx> Decodable> for CrateNum { + #[inline] fn decode(d: &mut DecodeContext<'a, 'tcx>) -> CrateNum { let cnum = CrateNum::from_u32(d.read_u32()); d.map_encoded_cnum_to_current(cnum) @@ -427,18 +430,21 @@ impl<'a, 'tcx> Decodable> for CrateNum { } impl<'a, 'tcx> Decodable> for DefIndex { + #[inline] fn decode(d: &mut DecodeContext<'a, 'tcx>) -> DefIndex { DefIndex::from_u32(d.read_u32()) } } impl<'a, 'tcx> Decodable> for ExpnIndex { + #[inline] fn decode(d: &mut DecodeContext<'a, 'tcx>) -> ExpnIndex { ExpnIndex::from_u32(d.read_u32()) } } impl<'a, 'tcx> Decodable> for ast::AttrId { + #[inline] fn decode(d: &mut DecodeContext<'a, 'tcx>) -> ast::AttrId { let sess = d.sess.expect("can't decode AttrId without Session"); sess.parse_sess.attr_id_generator.mk_attr_id() @@ -657,6 +663,7 @@ impl<'a, 'tcx, T> Decodable> for LazyValue { } impl<'a, 'tcx, T> Decodable> for LazyArray { + #[inline] fn decode(decoder: &mut DecodeContext<'a, 'tcx>) -> Self { let len = decoder.read_usize(); if len == 0 { LazyArray::default() } else { decoder.read_lazy_array(len) } diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs index 41be8434ead..2f807b8083e 100644 --- a/compiler/rustc_metadata/src/rmeta/table.rs +++ b/compiler/rustc_metadata/src/rmeta/table.rs @@ -480,7 +480,6 @@ where for<'tcx> T::Value<'tcx>: FixedSizeEncoding, { /// Given the metadata, extract out the value at a particular index (if any). - #[inline(never)] pub(super) fn get<'a, 'tcx, M: Metadata<'a, 'tcx>>(&self, metadata: M, i: I) -> T::Value<'tcx> { trace!("LazyTable::lookup: index={:?} len={:?}", i, self.len); @@ -494,9 +493,13 @@ where let end = start + width; let bytes = &metadata.blob()[start..end]; - let mut fixed = [0u8; N]; - fixed[..width].copy_from_slice(bytes); - FixedSizeEncoding::from_bytes(&fixed) + if let Ok(fixed) = bytes.try_into() { + FixedSizeEncoding::from_bytes(fixed) + } else { + let mut fixed = [0u8; N]; + fixed[..width].copy_from_slice(bytes); + FixedSizeEncoding::from_bytes(&fixed) + } } /// Size of the table in entries, including possible gaps. From 225b3c0556602e831ed691eab11ed1468cca80e1 Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Tue, 29 Aug 2023 20:16:57 -0400 Subject: [PATCH 3/3] Document in the code how this scheme works --- compiler/rustc_metadata/src/rmeta/mod.rs | 3 ++ compiler/rustc_metadata/src/rmeta/table.rs | 36 +++++++++++++--------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 1bd86e54307..7f022695e37 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -136,7 +136,10 @@ impl LazyArray { /// eagerly and in-order. struct LazyTable { position: NonZeroUsize, + /// The encoded size of the elements of a table is selected at runtime to drop + /// trailing zeroes. This is the number of bytes used for each table element. width: usize, + /// How many elements are in the table. len: usize, _marker: PhantomData T>, } diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs index 2f807b8083e..d572a65d004 100644 --- a/compiler/rustc_metadata/src/rmeta/table.rs +++ b/compiler/rustc_metadata/src/rmeta/table.rs @@ -346,6 +346,12 @@ impl LazyArray { let position = (self.position.get() as u64).to_le_bytes(); let len = (self.num_elems as u64).to_le_bytes(); + // Element width is selected at runtime on a per-table basis by omitting trailing + // zero bytes in table elements. This works very naturally when table elements are + // simple numbers but `LazyArray` is a pair of integers. If naively encoded, the second + // element would shield the trailing zeroes in the first. Interleaving the bytes + // of the position and length exposes trailing zeroes in both to the optimization. + // We encode length second because we generally expect it to be smaller. for i in 0..8 { b[2 * i] = position[i]; b[2 * i + 1] = len[i]; @@ -359,18 +365,26 @@ impl LazyArray { } } +// Decoding helper for the encoding scheme used by `LazyArray`. +// Interleaving the bytes of the two integers exposes trailing bytes in the first integer +// to the varint scheme that we use for tables. +#[inline] +fn decode_interleaved(encoded: &[u8; 16]) -> ([u8; 8], [u8; 8]) { + let mut first = [0u8; 8]; + let mut second = [0u8; 8]; + for i in 0..8 { + first[i] = encoded[2 * i]; + second[i] = encoded[2 * i + 1]; + } + (first, second) +} + impl FixedSizeEncoding for LazyArray { type ByteArray = [u8; 16]; #[inline] fn from_bytes(b: &[u8; 16]) -> Self { - let mut position = [0u8; 8]; - let mut meta = [0u8; 8]; - - for i in 0..8 { - position[i] = b[2 * i]; - meta[i] = b[2 * i + 1]; - } + let (position, meta) = decode_interleaved(b); if meta == [0; 8] { return Default::default(); @@ -390,13 +404,7 @@ impl FixedSizeEncoding for Option> { #[inline] fn from_bytes(b: &[u8; 16]) -> Self { - let mut position = [0u8; 8]; - let mut meta = [0u8; 8]; - - for i in 0..8 { - position[i] = b[2 * i]; - meta[i] = b[2 * i + 1]; - } + let (position, meta) = decode_interleaved(b); LazyArray::from_bytes_impl(&position, &meta) }