compare tagged/niche-filling layout and pick the best one

src/librustc_middle/lib.rs

@@ -27,6 +27,7 @@
 #![feature(bool_to_option)]
 #![feature(box_patterns)]
 #![feature(box_syntax)]
+#![feature(cmp_min_max_by)]
 #![feature(const_fn)]
 #![feature(const_panic)]
 #![feature(const_fn_transmute)]

src/librustc_middle/ty/layout.rs

@@ -1233,7 +1233,21 @@ fn layout_raw_uncached(&self, ty: Ty<'tcx>) -> Result<&'tcx Layout, LayoutError<
                     size,
                 };
 
-                tcx.intern_layout(niche_filling_layout.unwrap_or(tagged_layout))
+                let best_layout = match (tagged_layout, niche_filling_layout) {
+                    (tagged_layout, Some(niche_filling_layout)) => {
+                        // Pick the smaller layout; otherwise,
+                        // pick the layout with the larger niche; otherwise,
+                        // pick tagged as it has simpler codegen.
+                        cmp::min_by_key(tagged_layout, niche_filling_layout, |layout| {
+                            let niche_size =
+                                layout.largest_niche.as_ref().map_or(0, |n| n.available(dl));
+                            (layout.size, cmp::Reverse(niche_size))
+                        })
+                    }
+                    (tagged_layout, None) => tagged_layout,
+                };
+
+                tcx.intern_layout(best_layout)
             }
 
             // Types with no meaningful known layout.

src/test/ui/print_type_sizes/niche-filling.stdout

@@ -8,12 +8,12 @@ print-type-size     variant `Some`: 12 bytes
 print-type-size         field `.0`: 12 bytes
 print-type-size     variant `None`: 0 bytes
 print-type-size type: `EmbeddedDiscr`: 8 bytes, alignment: 4 bytes
+print-type-size     discriminant: 1 bytes
 print-type-size     variant `Record`: 7 bytes
-print-type-size         field `.val`: 4 bytes
-print-type-size         field `.post`: 2 bytes
 print-type-size         field `.pre`: 1 bytes
+print-type-size         field `.post`: 2 bytes
+print-type-size         field `.val`: 4 bytes
 print-type-size     variant `None`: 0 bytes
-print-type-size     end padding: 1 bytes
 print-type-size type: `MyOption<Union1<std::num::NonZeroU32>>`: 8 bytes, alignment: 4 bytes
 print-type-size     discriminant: 4 bytes
 print-type-size     variant `Some`: 4 bytes

src/test/ui/type-sizes.rs

@@ -5,6 +5,7 @@
 #![feature(never_type)]
 
 use std::mem::size_of;
+use std::num::NonZeroU8;
 
 struct t {a: u8, b: i8}
 struct u {a: u8, b: i8, c: u8}
@@ -102,6 +103,15 @@ enum Option2<A, B> {
     None
 }
 
+pub enum CanBeNicheFilledButShouldnt {
+    A(NonZeroU8, u32),
+    B
+}
+pub enum AlwaysTagged {
+    A(u8, u32),
+    B
+}
+
 pub fn main() {
     assert_eq!(size_of::<u8>(), 1 as usize);
     assert_eq!(size_of::<u32>(), 4 as usize);
@@ -145,4 +155,11 @@ pub fn main() {
     assert_eq!(size_of::<Option<Option<(&(), bool)>>>(), size_of::<(bool, &())>());
     assert_eq!(size_of::<Option<Option2<bool, &()>>>(), size_of::<(bool, &())>());
     assert_eq!(size_of::<Option<Option2<&(), bool>>>(), size_of::<(bool, &())>());
+
+    assert_eq!(size_of::<CanBeNicheFilledButShouldnt>(), 8);
+    assert_eq!(size_of::<Option<CanBeNicheFilledButShouldnt>>(), 8);
+    assert_eq!(size_of::<Option<Option<CanBeNicheFilledButShouldnt>>>(), 8);
+    assert_eq!(size_of::<AlwaysTagged>(), 8);
+    assert_eq!(size_of::<Option<AlwaysTagged>>(), 8);
+    assert_eq!(size_of::<Option<Option<AlwaysTagged>>>(), 8);
 }