make separate compilation respect #[inline] attributes

Adjust the handling of `#[inline]` items so that they get translated into every
compilation unit that uses them.  This is necessary to preserve the semantics
of `#[inline(always)]`.

Crate-local `#[inline]` functions and statics are blindly translated into every
compilation unit.  Cross-crate inlined items and monomorphizations of
`#[inline]` functions are translated the first time a reference is seen in each
compilation unit.  When using multiple compilation units, inlined items are
given `available_externally` linkage whenever possible to avoid duplicating
object code.

src/librustc/middle/trans/base.rs

@@ -2125,12 +2125,43 @@ impl<'a> Visitor<()> for TransItemVisitor<'a> {
     }
 }
 
+/// Enum describing the origin of an LLVM `Value`, for linkage purposes.
+pub enum ValueOrigin {
+    /// The LLVM `Value` is in this context because the corresponding item was
+    /// assigned to the current compilation unit.
+    OriginalTranslation,
+    /// The `Value`'s corresponding item was assigned to some other compilation
+    /// unit, but the `Value` was translated in this context anyway because the
+    /// item is marked `#[inline]`.
+    InlinedCopy,
+}
+
 /// Set the appropriate linkage for an LLVM `ValueRef` (function or global).
 /// If the `llval` is the direct translation of a specific Rust item, `id`
 /// should be set to the `NodeId` of that item.  (This mapping should be
 /// 1-to-1, so monomorphizations and drop/visit glue should have `id` set to
-/// `None`.)
-pub fn update_linkage(ccx: &CrateContext, llval: ValueRef, id: Option<ast::NodeId>) {
+/// `None`.)  `llval_origin` indicates whether `llval` is the translation of an
+/// item assigned to `ccx`'s compilation unit or an inlined copy of an item
+/// assigned to a different compilation unit.
+pub fn update_linkage(ccx: &CrateContext,
+                      llval: ValueRef,
+                      id: Option<ast::NodeId>,
+                      llval_origin: ValueOrigin) {
+    match llval_origin {
+        InlinedCopy => {
+            // `llval` is a translation of an item defined in a separate
+            // compilation unit.  This only makes sense if there are at least
+            // two compilation units.
+            assert!(ccx.sess().opts.cg.codegen_units > 1);
+            // `llval` is a copy of something defined elsewhere, so use
+            // `AvailableExternallyLinkage` to avoid duplicating code in the
+            // output.
+            llvm::SetLinkage(llval, llvm::AvailableExternallyLinkage);
+            return;
+        },
+        OriginalTranslation => {},
+    }
+
     match id {
         Some(id) if ccx.reachable().contains(&id) => {
             llvm::SetLinkage(llval, llvm::ExternalLinkage);
@@ -2149,29 +2180,41 @@ pub fn update_linkage(ccx: &CrateContext, llval: ValueRef, id: Option<ast::NodeI
 pub fn trans_item(ccx: &CrateContext, item: &ast::Item) {
     let _icx = push_ctxt("trans_item");
 
+    let from_external = ccx.external_srcs().borrow().contains_key(&item.id);
+
     match item.node {
       ast::ItemFn(ref decl, _fn_style, abi, ref generics, ref body) => {
         if !generics.is_type_parameterized() {
-            let llfn = get_item_val(ccx, item.id);
-            if abi != Rust {
-                foreign::trans_rust_fn_with_foreign_abi(ccx,
-                                                        &**decl,
-                                                        &**body,
-                                                        item.attrs.as_slice(),
-                                                        llfn,
-                                                        &param_substs::empty(),
-                                                        item.id,
-                                                        None);
-            } else {
-                trans_fn(ccx,
-                         &**decl,
-                         &**body,
-                         llfn,
-                         &param_substs::empty(),
-                         item.id,
-                         item.attrs.as_slice());
+            let trans_everywhere = attr::requests_inline(item.attrs.as_slice());
+            // Ignore `trans_everywhere` for cross-crate inlined items
+            // (`from_external`).  `trans_item` will be called once for each
+            // compilation unit that references the item, so it will still get
+            // translated everywhere it's needed.
+            for (ref ccx, is_origin) in ccx.maybe_iter(!from_external && trans_everywhere) {
+                let llfn = get_item_val(ccx, item.id);
+                if abi != Rust {
+                    foreign::trans_rust_fn_with_foreign_abi(ccx,
+                                                            &**decl,
+                                                            &**body,
+                                                            item.attrs.as_slice(),
+                                                            llfn,
+                                                            &param_substs::empty(),
+                                                            item.id,
+                                                            None);
+                } else {
+                    trans_fn(ccx,
+                             &**decl,
+                             &**body,
+                             llfn,
+                             &param_substs::empty(),
+                             item.id,
+                             item.attrs.as_slice());
+                }
+                update_linkage(ccx,
+                               llfn,
+                               Some(item.id),
+                               if is_origin { OriginalTranslation } else { InlinedCopy });
             }
-            update_linkage(ccx, llfn, Some(item.id));
         }
 
         // Be sure to travel more than just one layer deep to catch nested
@@ -2196,10 +2239,17 @@ pub fn trans_item(ccx: &CrateContext, item: &ast::Item) {
           // Recurse on the expression to catch items in blocks
           let mut v = TransItemVisitor{ ccx: ccx };
           v.visit_expr(&**expr, ());
-          consts::trans_const(ccx, m, item.id);
 
-          let g = get_item_val(ccx, item.id);
-          update_linkage(ccx, g, Some(item.id));
+          let trans_everywhere = attr::requests_inline(item.attrs.as_slice());
+          for (ref ccx, is_origin) in ccx.maybe_iter(!from_external && trans_everywhere) {
+              consts::trans_const(ccx, m, item.id);
+
+              let g = get_item_val(ccx, item.id);
+              update_linkage(ccx,
+                             g,
+                             Some(item.id),
+                             if is_origin { OriginalTranslation } else { InlinedCopy });
+          }
 
           // Do static_assert checking. It can't really be done much earlier
           // because we need to get the value of the bool out of LLVM

src/librustc/middle/trans/context.rs

@@ -155,6 +155,9 @@ pub struct LocalCrateContext {
 pub struct CrateContext<'a> {
     shared: &'a SharedCrateContext,
     local: &'a LocalCrateContext,
+    /// The index of `local` in `shared.local_ccxs`.  This is used in
+    /// `maybe_iter(true)` to identify the original `LocalCrateContext`.
+    index: uint,
 }
 
 pub struct CrateContextIterator<'a> {
@@ -174,10 +177,41 @@ impl<'a> Iterator<CrateContext<'a>> for CrateContextIterator<'a> {
         Some(CrateContext {
             shared: self.shared,
             local: &self.shared.local_ccxs[index],
+            index: index,
         })
     }
 }
 
+/// The iterator produced by `CrateContext::maybe_iter`.
+pub struct CrateContextMaybeIterator<'a> {
+    shared: &'a SharedCrateContext,
+    index: uint,
+    single: bool,
+    origin: uint,
+}
+
+impl<'a> Iterator<(CrateContext<'a>, bool)> for CrateContextMaybeIterator<'a> {
+    fn next(&mut self) -> Option<(CrateContext<'a>, bool)> {
+        if self.index >= self.shared.local_ccxs.len() {
+            return None;
+        }
+
+        let index = self.index;
+        self.index += 1;
+        if self.single {
+            self.index = self.shared.local_ccxs.len();
+        }
+
+        let ccx = CrateContext {
+            shared: self.shared,
+            local: &self.shared.local_ccxs[index],
+            index: index,
+        };
+        Some((ccx, index == self.origin))
+    }
+}
+
+
 unsafe fn create_context_and_module(sess: &Session, mod_name: &str) -> (ContextRef, ModuleRef) {
     let llcx = llvm::LLVMContextCreate();
     let llmod = mod_name.with_c_str(|buf| {
@@ -270,18 +304,21 @@ impl SharedCrateContext {
         CrateContext {
             shared: self,
             local: &self.local_ccxs[index],
+            index: index,
         }
     }
 
     fn get_smallest_ccx<'a>(&'a self) -> CrateContext<'a> {
-        let local_ccx =
+        let (local_ccx, index) =
             self.local_ccxs
                 .iter()
-                .min_by(|&local_ccx| local_ccx.n_llvm_insns.get())
+                .zip(range(0, self.local_ccxs.len()))
+                .min_by(|&(local_ccx, _idx)| local_ccx.n_llvm_insns.get())
                 .unwrap();
         CrateContext {
             shared: self,
             local: local_ccx,
+            index: index,
         }
     }
 
@@ -426,6 +463,7 @@ impl LocalCrateContext {
         CrateContext {
             shared: shared,
             local: self,
+            index: -1 as uint,
         }
     }
 }
@@ -446,6 +484,22 @@ impl<'b> CrateContext<'b> {
         self.shared.get_smallest_ccx()
     }
 
+    /// Either iterate over only `self`, or iterate over all `CrateContext`s in
+    /// the `SharedCrateContext`.  The iterator produces `(ccx, is_origin)`
+    /// pairs, where `is_origin` is `true` if `ccx` is `self` and `false`
+    /// otherwise.  This method is useful for avoiding code duplication in
+    /// cases where it may or may not be necessary to translate code into every
+    /// context.
+    pub fn maybe_iter(&self, iter_all: bool) -> CrateContextMaybeIterator<'b> {
+        CrateContextMaybeIterator {
+            shared: self.shared,
+            index: if iter_all { 0 } else { self.index },
+            single: !iter_all,
+            origin: self.index,
+        }
+    }
+
+
     pub fn tcx<'a>(&'a self) -> &'a ty::ctxt {
         &self.shared.tcx
     }

src/librustc/middle/trans/glue.rs

@@ -669,7 +669,7 @@ fn make_generic_glue(ccx: &CrateContext,
 
     let bcx = init_function(&fcx, false, ty::mk_nil());
 
-    update_linkage(ccx, llfn, None);
+    update_linkage(ccx, llfn, None, OriginalTranslation);
 
     ccx.stats().n_glues_created.set(ccx.stats().n_glues_created.get() + 1u);
     // All glue functions take values passed *by alias*; this is a

src/librustc/middle/trans/meth.rs

@@ -38,7 +38,7 @@ use util::ppaux::Repr;
 use std::c_str::ToCStr;
 use syntax::abi::{Rust, RustCall};
 use syntax::parse::token;
-use syntax::{ast, ast_map, visit};
+use syntax::{ast, ast_map, attr, visit};
 use syntax::ast_util::PostExpansionMethod;
 
 // drop_glue pointer, size, align.
@@ -77,15 +77,21 @@ pub fn trans_impl(ccx: &CrateContext,
         match *impl_item {
             ast::MethodImplItem(method) => {
                 if method.pe_generics().ty_params.len() == 0u {
-                    let llfn = get_item_val(ccx, method.id);
-                    trans_fn(ccx,
-                             &*method.pe_fn_decl(),
-                             &*method.pe_body(),
-                             llfn,
-                             &param_substs::empty(),
-                             method.id,
-                             []);
-                    update_linkage(ccx, llfn, Some(method.id));
+                    let trans_everywhere = attr::requests_inline(method.attrs.as_slice());
+                    for (ref ccx, is_origin) in ccx.maybe_iter(trans_everywhere) {
+                        let llfn = get_item_val(ccx, method.id);
+                        trans_fn(ccx,
+                                 &*method.pe_fn_decl(),
+                                 &*method.pe_body(),
+                                 llfn,
+                                 &param_substs::empty(),
+                                 method.id,
+                                 []);
+                        update_linkage(ccx,
+                                       llfn,
+                                       Some(method.id),
+                                       if is_origin { OriginalTranslation } else { InlinedCopy });
+                    }
                 }
                 let mut v = TransItemVisitor {
                     ccx: ccx,

src/librustc/middle/trans/monomorphize.rs

@@ -11,6 +11,7 @@
 use back::link::exported_name;
 use driver::session;
 use llvm::ValueRef;
+use llvm;
 use middle::subst;
 use middle::subst::Subst;
 use middle::trans::base::{set_llvm_fn_attrs, set_inline_hint};
@@ -27,6 +28,7 @@ use syntax::abi;
 use syntax::ast;
 use syntax::ast_map;
 use syntax::ast_util::{local_def, PostExpansionMethod};
+use syntax::attr;
 use std::hash::{sip, Hash};
 
 pub fn monomorphic_fn(ccx: &CrateContext,
@@ -150,6 +152,25 @@ pub fn monomorphic_fn(ccx: &CrateContext,
         ccx.monomorphized().borrow_mut().insert(hash_id.take().unwrap(), lldecl);
         lldecl
     };
+    let setup_lldecl = |lldecl, attrs: &[ast::Attribute]| {
+        base::update_linkage(ccx, lldecl, None, base::OriginalTranslation);
+        set_llvm_fn_attrs(attrs, lldecl);
+
+        let is_first = !ccx.available_monomorphizations().borrow().contains(&s);
+        if is_first {
+            ccx.available_monomorphizations().borrow_mut().insert(s.clone());
+        }
+
+        let trans_everywhere = attr::requests_inline(attrs);
+        if trans_everywhere && !is_first {
+            llvm::SetLinkage(lldecl, llvm::AvailableExternallyLinkage);
+        }
+
+        // If `true`, then `lldecl` should be given a function body.
+        // Otherwise, it should be left as a declaration of an external
+        // function, with no definition in the current compilation unit.
+        trans_everywhere || is_first
+    };
 
     let lldecl = match map_node {
         ast_map::NodeItem(i) => {
@@ -159,11 +180,8 @@ pub fn monomorphic_fn(ccx: &CrateContext,
                   ..
               } => {
                   let d = mk_lldecl(abi);
-                  base::update_linkage(ccx, d, None);
-                  set_llvm_fn_attrs(i.attrs.as_slice(), d);
-
-                  if !ccx.available_monomorphizations().borrow().contains(&s) {
-                      ccx.available_monomorphizations().borrow_mut().insert(s.clone());
+                  let needs_body = setup_lldecl(d, i.attrs.as_slice());
+                  if needs_body {
                       if abi != abi::Rust {
                           foreign::trans_rust_fn_with_foreign_abi(
                               ccx, &**decl, &**body, [], d, &psubsts, fn_id.node,
@@ -205,17 +223,15 @@ pub fn monomorphic_fn(ccx: &CrateContext,
             match *ii {
                 ast::MethodImplItem(mth) => {
                     let d = mk_lldecl(abi::Rust);
-                    base::update_linkage(ccx, d, None);
-                    set_llvm_fn_attrs(mth.attrs.as_slice(), d);
-                    if !ccx.available_monomorphizations().borrow().contains(&s) {
-                        ccx.available_monomorphizations().borrow_mut().insert(s.clone());
-                            trans_fn(ccx,
-                                     &*mth.pe_fn_decl(),
-                                     &*mth.pe_body(),
-                                     d,
-                                     &psubsts,
-                                     mth.id,
-                                     []);
+                    let needs_body = setup_lldecl(d, mth.attrs.as_slice());
+                    if needs_body {
+                        trans_fn(ccx,
+                                 &*mth.pe_fn_decl(),
+                                 &*mth.pe_body(),
+                                 d,
+                                 &psubsts,
+                                 mth.id,
+                                 []);
                     }
                     d
                 }
@@ -225,10 +241,8 @@ pub fn monomorphic_fn(ccx: &CrateContext,
             match *method {
                 ast::ProvidedMethod(mth) => {
                     let d = mk_lldecl(abi::Rust);
-                    base::update_linkage(ccx, d, None);
-                    set_llvm_fn_attrs(mth.attrs.as_slice(), d);
-                    if !ccx.available_monomorphizations().borrow().contains(&s) {
-                        ccx.available_monomorphizations().borrow_mut().insert(s.clone());
+                    let needs_body = setup_lldecl(d, mth.attrs.as_slice());
+                    if needs_body {
                         trans_fn(ccx, &*mth.pe_fn_decl(), &*mth.pe_body(), d,
                                  &psubsts, mth.id, []);
                     }

src/libsyntax/attr.rs

@@ -280,7 +280,7 @@ pub enum InlineAttr {
     InlineNever,
 }
 
-/// True if something like #[inline] is found in the list of attrs.
+/// Determine what `#[inline]` attribute is present in `attrs`, if any.
 pub fn find_inline_attr(attrs: &[Attribute]) -> InlineAttr {
     // FIXME (#2809)---validate the usage of #[inline] and #[inline]
     attrs.iter().fold(InlineNone, |ia,attr| {
@@ -304,6 +304,14 @@ pub fn find_inline_attr(attrs: &[Attribute]) -> InlineAttr {
     })
 }
 
+/// True if `#[inline]` or `#[inline(always)]` is present in `attrs`.
+pub fn requests_inline(attrs: &[Attribute]) -> bool {
+    match find_inline_attr(attrs) {
+        InlineHint | InlineAlways => true,
+        InlineNone | InlineNever => false,
+    }
+}
+
 /// Tests if any `cfg(...)` meta items in `metas` match `cfg`. e.g.
 ///
 /// test_cfg(`[foo="a", bar]`, `[cfg(foo), cfg(bar)]`) == true