Properly evaluate non-consts in const prop

compiler/rustc_mir/src/transform/const_prop.rs

@@ -393,8 +393,11 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             .filter(|ret_layout| {
                 !ret_layout.is_zst() && ret_layout.size < Size::from_bytes(MAX_ALLOC_LIMIT)
             })
-            // hopefully all types will allocate, since large types have already been removed
-            .and_then(|ret_layout| ecx.allocate(ret_layout, MemoryKind::Stack).ok())
+            .and_then(|ret_layout| {
+                let alloc = ecx.allocate(ret_layout, MemoryKind::Stack);
+                Self::check_interpresult(tcx, &alloc);
+                alloc.ok()
+            })
             .map(Into::into);
 
         ecx.push_stack_frame(
@@ -418,11 +421,27 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         }
     }
 
+    /// Some `InterpError`s could be ignored but must not be to ensure that queries are stable.
+    fn check_interpresult<T>(tcx: TyCtxt<'tcx>, error: &InterpResult<'tcx, T>) {
+        if let Err(e) = error {
+            if matches!(
+                e.kind(),
+                InterpError::ResourceExhaustion(ResourceExhaustionInfo::MemoryExhausted)
+            ) {
+                // Memory errors can't be ignored since otherwise the amount of available
+                // memory influences the result of optimization and the build. The error
+                // doesn't need to be fatal since no code will actually be generated anyways.
+                tcx.sess.fatal("memory exhausted during optimization");
+            }
+        }
+    }
+
     fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> {
         let op = match self.ecx.eval_place_to_op(place, None) {
             Ok(op) => op,
             Err(e) => {
                 trace!("get_const failed: {}", e);
+                Self::check_interpresult::<()>(self.tcx, &Err(e));
                 return None;
             }
         };
@@ -524,7 +543,12 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
     /// Returns the value, if any, of evaluating `place`.
     fn eval_place(&mut self, place: Place<'tcx>) -> Option<OpTy<'tcx>> {
         trace!("eval_place(place={:?})", place);
-        self.use_ecx(|this| this.ecx.eval_place_to_op(place, None))
+        let tcx = self.tcx;
+        self.use_ecx(|this| {
+            let val = this.ecx.eval_place_to_op(place, None);
+            Self::check_interpresult(tcx, &val);
+            val
+        })
     }
 
     /// Returns the value, if any, of evaluating `op`. Calls upon `eval_constant`
@@ -585,8 +609,17 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         right: &Operand<'tcx>,
         source_info: SourceInfo,
     ) -> Option<()> {
-        let r = self.use_ecx(|this| this.ecx.read_immediate(&this.ecx.eval_operand(right, None)?));
-        let l = self.use_ecx(|this| this.ecx.read_immediate(&this.ecx.eval_operand(left, None)?));
+        let tcx = self.tcx;
+        let r = self.use_ecx(|this| {
+            let val = this.ecx.read_immediate(&this.ecx.eval_operand(right, None)?);
+            Self::check_interpresult(tcx, &val);
+            val
+        });
+        let l = self.use_ecx(|this| {
+            let val = this.ecx.read_immediate(&this.ecx.eval_operand(left, None)?);
+            Self::check_interpresult(tcx, &val);
+            val
+        });
         // Check for exceeding shifts *even if* we cannot evaluate the LHS.
         if op == BinOp::Shr || op == BinOp::Shl {
             let r = r?;
@@ -752,18 +785,24 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         rvalue: &Rvalue<'tcx>,
         place: Place<'tcx>,
     ) -> Option<()> {
+        let tcx = self.tcx;
         self.use_ecx(|this| {
             match rvalue {
                 Rvalue::BinaryOp(op, box (left, right))
                 | Rvalue::CheckedBinaryOp(op, box (left, right)) => {
                     let l = this.ecx.eval_operand(left, None);
                     let r = this.ecx.eval_operand(right, None);
+                    Self::check_interpresult(tcx, &l);
+                    Self::check_interpresult(tcx, &r);
 
                     let const_arg = match (l, r) {
                         (Ok(ref x), Err(_)) | (Err(_), Ok(ref x)) => this.ecx.read_immediate(x)?,
                         (Err(e), Err(_)) => return Err(e),
                         (Ok(_), Ok(_)) => {
-                            this.ecx.eval_rvalue_into_place(rvalue, place)?;
+                            Self::check_interpresult(
+                                tcx,
+                                &this.ecx.eval_rvalue_into_place(rvalue, place),
+                            );
                             return Ok(());
                         }
                     };
@@ -799,12 +838,16 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
                             }
                         }
                         _ => {
-                            this.ecx.eval_rvalue_into_place(rvalue, place)?;
+                            let res = this.ecx.eval_rvalue_into_place(rvalue, place);
+                            Self::check_interpresult(tcx, &res);
+                            res?
                         }
                     }
                 }
                 _ => {
-                    this.ecx.eval_rvalue_into_place(rvalue, place)?;
+                    let res = this.ecx.eval_rvalue_into_place(rvalue, place);
+                    Self::check_interpresult(tcx, &res);
+                    res?
                 }
             }