From da33935c260bf8859d20b83dec40be7fc3d82310 Mon Sep 17 00:00:00 2001
From: Eduard-Mihai Burtescu <edy.burt@gmail.com>
Date: Wed, 22 Jan 2020 02:52:54 +0200
Subject: [PATCH] rustc_target: treat enum variants like union members, in call
 ABIs.

---
 src/librustc_target/abi/call/mod.rs    | 28 +++++++++++++++++++++++++-
 src/librustc_target/abi/call/x86_64.rs | 26 +++++++++++++++---------
 2 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/src/librustc_target/abi/call/mod.rs b/src/librustc_target/abi/call/mod.rs
index 748fd2b6579..e3cbf176c35 100644
--- a/src/librustc_target/abi/call/mod.rs
+++ b/src/librustc_target/abi/call/mod.rs
@@ -308,7 +308,7 @@ pub fn homogeneous_aggregate<C>(&self, cx: &C) -> Result<HomogeneousAggregate, H
 
             Abi::ScalarPair(..) | Abi::Aggregate { .. } => {
                 // Helper for computing `homogenous_aggregate`, allowing a custom
-                // starting offset (TODO(eddyb): use this to handle variants).
+                // starting offset (used below for handling variants).
                 let from_fields_at =
                     |layout: Self,
                      start: Size|
@@ -354,6 +354,32 @@ pub fn homogeneous_aggregate<C>(&self, cx: &C) -> Result<HomogeneousAggregate, H
 
                 let (mut result, mut total) = from_fields_at(*self, Size::ZERO)?;
 
+                match &self.variants {
+                    abi::Variants::Single { .. } => {}
+                    abi::Variants::Multiple { variants, .. } => {
+                        // Treat enum variants like union members.
+                        // HACK(eddyb) pretend the `enum` field (discriminant)
+                        // is at the start of every variant (otherwise the gap
+                        // at the start of all variants would disqualify them).
+                        //
+                        // NB: for all tagged `enum`s (which include all non-C-like
+                        // `enum`s with defined FFI representation), this will
+                        // match the homogenous computation on the equivalent
+                        // `struct { tag; union { variant1; ... } }` and/or
+                        // `union { struct { tag; variant1; } ... }`
+                        // (the offsets of variant fields should be identical
+                        // between the two for either to be a homogenous aggregate).
+                        let variant_start = total;
+                        for variant_idx in variants.indices() {
+                            let (variant_result, variant_total) =
+                                from_fields_at(self.for_variant(cx, variant_idx), variant_start)?;
+
+                            result = result.merge(variant_result)?;
+                            total = total.max(variant_total);
+                        }
+                    }
+                }
+
                 // There needs to be no padding.
                 if total != self.size {
                     Err(Heterogeneous)
diff --git a/src/librustc_target/abi/call/x86_64.rs b/src/librustc_target/abi/call/x86_64.rs
index a547d7262e2..4c192c46786 100644
--- a/src/librustc_target/abi/call/x86_64.rs
+++ b/src/librustc_target/abi/call/x86_64.rs
@@ -56,16 +56,24 @@ fn classify<'a, Ty, C>(
 
             Abi::Vector { .. } => Class::Sse,
 
-            Abi::ScalarPair(..) | Abi::Aggregate { .. } => match layout.variants {
-                abi::Variants::Single { .. } => {
-                    for i in 0..layout.fields.count() {
-                        let field_off = off + layout.fields.offset(i);
-                        classify(cx, layout.field(cx, i), cls, field_off)?;
-                    }
-                    return Ok(());
+            Abi::ScalarPair(..) | Abi::Aggregate { .. } => {
+                for i in 0..layout.fields.count() {
+                    let field_off = off + layout.fields.offset(i);
+                    classify(cx, layout.field(cx, i), cls, field_off)?;
                 }
-                abi::Variants::Multiple { .. } => return Err(Memory),
-            },
+
+                match &layout.variants {
+                    abi::Variants::Single { .. } => {}
+                    abi::Variants::Multiple { variants, .. } => {
+                        // Treat enum variants like union members.
+                        for variant_idx in variants.indices() {
+                            classify(cx, layout.for_variant(cx, variant_idx), cls, off)?;
+                        }
+                    }
+                }
+
+                return Ok(());
+            }
         };
 
         // Fill in `cls` for scalars (Int/Sse) and vectors (Sse).