diff --git a/src/machine.rs b/src/machine.rs
index 4fb08cd259b..51aa7ae3104 100644
--- a/src/machine.rs
+++ b/src/machine.rs
@@ -5,7 +5,7 @@ use std::borrow::Cow;
 use std::cell::RefCell;
 use std::num::NonZeroU64;
 use std::rc::Rc;
-use std::time::{Instant, SystemTime};
+use std::time::Instant;
 use std::fmt;
 
 use log::trace;
@@ -251,11 +251,6 @@ pub struct Evaluator<'mir, 'tcx> {
     /// The "time anchor" for this machine's monotone clock (for `Instant` simulation).
     pub(crate) time_anchor: Instant,
 
-    /// The approximate system time when "time anchor" was created. This is used
-    /// for converting system time to monotone time so that we can simplify the
-    /// thread scheduler to deal only with a single representation of time.
-    pub(crate) time_anchor_timestamp: SystemTime,
-
     /// The set of threads.
     pub(crate) threads: ThreadManager<'mir, 'tcx>,
 
@@ -286,7 +281,6 @@ impl<'mir, 'tcx> Evaluator<'mir, 'tcx> {
             dir_handler: Default::default(),
             panic_payload: None,
             time_anchor: Instant::now(),
-            time_anchor_timestamp: SystemTime::now(),
             layouts,
             threads: ThreadManager::default(),
         }
diff --git a/src/shims/sync.rs b/src/shims/sync.rs
index a586be8139b..5432c76dfe7 100644
--- a/src/shims/sync.rs
+++ b/src/shims/sync.rs
@@ -1,10 +1,11 @@
-use std::time::{Duration, SystemTime};
 use std::convert::TryInto;
+use std::time::{Duration, SystemTime};
 
 use rustc_middle::ty::{layout::TyAndLayout, TyKind, TypeAndMut};
 use rustc_target::abi::{LayoutOf, Size};
 
 use crate::stacked_borrows::Tag;
+use crate::thread::Time;
 
 use crate::*;
 
@@ -734,12 +735,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
         };
 
         let timeout_time = if clock_id == this.eval_libc_i32("CLOCK_REALTIME")? {
-            let time_anchor_since_epoch =
-                this.machine.time_anchor_timestamp.duration_since(SystemTime::UNIX_EPOCH).unwrap();
-            let duration_since_time_anchor = duration.checked_sub(time_anchor_since_epoch).unwrap();
-            this.machine.time_anchor.checked_add(duration_since_time_anchor).unwrap()
+            Time::RealTime(SystemTime::UNIX_EPOCH.checked_add(duration).unwrap())
         } else if clock_id == this.eval_libc_i32("CLOCK_MONOTONIC")? {
-            this.machine.time_anchor.checked_add(duration).unwrap()
+            Time::Monotonic(this.machine.time_anchor.checked_add(duration).unwrap())
         } else {
             throw_ub_format!("Unsupported clock id.");
         };
diff --git a/src/sync.rs b/src/sync.rs
index 88b5d6c060d..e05d111cb28 100644
--- a/src/sync.rs
+++ b/src/sync.rs
@@ -1,7 +1,6 @@
 use std::collections::{hash_map::Entry, HashMap, VecDeque};
 use std::convert::TryFrom;
 use std::num::NonZeroU32;
-use std::time::Instant;
 
 use rustc_index::vec::{Idx, IndexVec};
 
@@ -76,8 +75,6 @@ struct CondvarWaiter {
     thread: ThreadId,
     /// The mutex on which the thread is waiting.
     mutex: MutexId,
-    /// The moment in time when the waiter should time out.
-    timeout: Option<Instant>,
 }
 
 /// The conditional variable state.
@@ -280,7 +277,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
         let this = self.eval_context_mut();
         let waiters = &mut this.machine.threads.sync.condvars[id].waiters;
         assert!(waiters.iter().all(|waiter| waiter.thread != thread), "thread is already waiting");
-        waiters.push_back(CondvarWaiter { thread, mutex, timeout: None });
+        waiters.push_back(CondvarWaiter { thread, mutex });
     }
 
     /// Wake up some thread (if there is any) sleeping on the conditional
diff --git a/src/thread.rs b/src/thread.rs
index f67de48b710..69b31b541ae 100644
--- a/src/thread.rs
+++ b/src/thread.rs
@@ -4,7 +4,7 @@ use std::cell::RefCell;
 use std::collections::hash_map::Entry;
 use std::convert::TryFrom;
 use std::num::TryFromIntError;
-use std::time::Instant;
+use std::time::{Duration, Instant, SystemTime};
 
 use log::trace;
 
@@ -159,13 +159,30 @@ impl<'mir, 'tcx> Default for Thread<'mir, 'tcx> {
     }
 }
 
+#[derive(Debug)]
+pub enum Time {
+    Monotonic(Instant),
+    RealTime(SystemTime),
+}
+
+impl Time {
+    /// How long do we have to wait from now until the specified time?
+    fn get_wait_time(&self) -> Duration {
+        match self {
+            Time::Monotonic(instant) => instant.saturating_duration_since(Instant::now()),
+            Time::RealTime(time) =>
+                time.duration_since(SystemTime::now()).unwrap_or(Duration::new(0, 0)),
+        }
+    }
+}
+
 /// Callbacks are used to implement timeouts. For example, waiting on a
 /// conditional variable with a timeout creates a callback that is called after
 /// the specified time and unblocks the thread. If another thread signals on the
 /// conditional variable, the signal handler deletes the callback.
 struct TimeoutCallbackInfo<'mir, 'tcx> {
     /// The callback should be called no earlier than this time.
-    call_time: Instant,
+    call_time: Time,
     /// The called function.
     callback: TimeoutCallback<'mir, 'tcx>,
 }
@@ -362,11 +379,11 @@ impl<'mir, 'tcx: 'mir> ThreadManager<'mir, 'tcx> {
     fn register_timeout_callback(
         &mut self,
         thread: ThreadId,
-        call_time: Instant,
+        call_time: Time,
         callback: TimeoutCallback<'mir, 'tcx>,
     ) {
         self.timeout_callbacks
-            .insert(thread, TimeoutCallbackInfo { call_time: call_time, callback: callback })
+            .insert(thread, TimeoutCallbackInfo { call_time, callback })
             .unwrap_none();
     }
 
@@ -376,13 +393,12 @@ impl<'mir, 'tcx: 'mir> ThreadManager<'mir, 'tcx> {
     }
 
     /// Get a callback that is ready to be called.
-    fn get_callback(&mut self) -> Option<(ThreadId, TimeoutCallback<'mir, 'tcx>)> {
-        let current_time = Instant::now();
+    fn get_ready_callback(&mut self) -> Option<(ThreadId, TimeoutCallback<'mir, 'tcx>)> {
         // We use a for loop here to make the scheduler more deterministic.
         for thread in self.threads.indices() {
             match self.timeout_callbacks.entry(thread) {
                 Entry::Occupied(entry) =>
-                    if current_time >= entry.get().call_time {
+                    if entry.get().call_time.get_wait_time() == Duration::new(0, 0) {
                         return Some((thread, entry.remove().callback));
                     },
                 Entry::Vacant(_) => {}
@@ -445,18 +461,14 @@ impl<'mir, 'tcx: 'mir> ThreadManager<'mir, 'tcx> {
         }
         // We have not found a thread to execute.
         if self.threads.iter().all(|thread| thread.state == ThreadState::Terminated) {
-            unreachable!();
-        } else if let Some(next_call_time) =
-            self.timeout_callbacks.values().min_by_key(|info| info.call_time)
+            unreachable!("all threads terminated without the main thread terminating?!");
+        } else if let Some(sleep_time) =
+            self.timeout_callbacks.values().map(|info| info.call_time.get_wait_time()).min()
         {
             // All threads are currently blocked, but we have unexecuted
             // timeout_callbacks, which may unblock some of the threads. Hence,
             // sleep until the first callback.
-            if let Some(sleep_time) =
-                next_call_time.call_time.checked_duration_since(Instant::now())
-            {
-                std::thread::sleep(sleep_time);
-            }
+            std::thread::sleep(sleep_time);
             Ok(SchedulingAction::ExecuteTimeoutCallback)
         } else {
             throw_machine_stop!(TerminationInfo::Deadlock);
@@ -650,7 +662,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
     fn register_timeout_callback(
         &mut self,
         thread: ThreadId,
-        call_time: Instant,
+        call_time: Time,
         callback: TimeoutCallback<'mir, 'tcx>,
     ) -> InterpResult<'tcx> {
         let this = self.eval_context_mut();
@@ -669,7 +681,8 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
     #[inline]
     fn run_timeout_callback(&mut self) -> InterpResult<'tcx> {
         let this = self.eval_context_mut();
-        let (thread, callback) = this.machine.threads.get_callback().expect("no callback found");
+        let (thread, callback) =
+            this.machine.threads.get_ready_callback().expect("no callback found");
         let old_thread = this.set_active_thread(thread)?;
         callback(this)?;
         this.set_active_thread(old_thread)?;