diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 0f68b234c78..63518e67ff4 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -42,14 +42,6 @@ jobs:
       if: matrix.os == 'windows-latest'
       run: Rename-Item C:\Users\runneradmin\.rustup\toolchains\stable-x86_64-pc-windows-msvc C:\Users\runneradmin\.rustup\toolchains\stable-x86_64-pc-windows-msvc.old
 
-    # Work around https://github.com/actions/cache/issues/403 by using GNU tar
-    # instead of BSD tar.
-    - name: Install GNU tar
-      if: matrix.os == 'macos-latest'
-      run: |
-        brew install gnu-tar
-        echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV
-
     - name: Install Rust toolchain
       uses: actions-rs/toolchain@v1
       with:
@@ -58,19 +50,8 @@ jobs:
         override: true
         components: rustfmt, rust-src
 
-    - name: Cache cargo directories
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/.cargo/registry
-          ~/.cargo/git
-        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-
-    - name: Cache cargo target dir
-      uses: actions/cache@v2
-      with:
-        path: target
-        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+    - name: Cache Dependencies
+      uses: Swatinem/rust-cache@ce325b60658c1b38465c06cc965b79baf32c1e72
 
     - name: Compile
       run: cargo test --no-run --locked
@@ -78,9 +59,6 @@ jobs:
     - name: Test
       run: cargo test -- --nocapture
 
-    - name: Prepare cache
-      run: cargo xtask pre-cache
-
   # Weird targets to catch non-portable code
   rust-cross:
     name: Rust Cross
@@ -103,13 +81,8 @@ jobs:
     - name: Install Rust targets
       run: rustup target add ${{ env.targets }}
 
-    - name: Cache cargo directories
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/.cargo/registry
-          ~/.cargo/git
-        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+    - name: Cache Dependencies
+      uses: Swatinem/rust-cache@ce325b60658c1b38465c06cc965b79baf32c1e72
 
     - name: Check
       run: |
diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs
index f80a5dd1631..69b3cb9c175 100644
--- a/xtask/src/flags.rs
+++ b/xtask/src/flags.rs
@@ -28,7 +28,6 @@
         }
 
         cmd fuzz-tests {}
-        cmd pre-cache {}
 
         cmd release {
             optional --dry-run
@@ -62,7 +61,6 @@ pub enum XtaskCmd {
     Help(Help),
     Install(Install),
     FuzzTests(FuzzTests),
-    PreCache(PreCache),
     Release(Release),
     Promote(Promote),
     Dist(Dist),
@@ -87,9 +85,6 @@ pub struct Install {
 #[derive(Debug)]
 pub struct FuzzTests;
 
-#[derive(Debug)]
-pub struct PreCache;
-
 #[derive(Debug)]
 pub struct Release {
     pub dry_run: bool,
diff --git a/xtask/src/main.rs b/xtask/src/main.rs
index ce335341084..d0bef7b7ae3 100644
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@@ -18,7 +18,6 @@
 mod release;
 mod dist;
 mod metrics;
-mod pre_cache;
 
 use anyhow::{bail, Result};
 use std::{
@@ -39,7 +38,6 @@ fn main() -> Result<()> {
         }
         flags::XtaskCmd::Install(cmd) => cmd.run(),
         flags::XtaskCmd::FuzzTests(_) => run_fuzzer(),
-        flags::XtaskCmd::PreCache(cmd) => cmd.run(),
         flags::XtaskCmd::Release(cmd) => cmd.run(),
         flags::XtaskCmd::Promote(cmd) => cmd.run(),
         flags::XtaskCmd::Dist(cmd) => cmd.run(),
diff --git a/xtask/src/pre_cache.rs b/xtask/src/pre_cache.rs
deleted file mode 100644
index b456224fd69..00000000000
--- a/xtask/src/pre_cache.rs
+++ /dev/null
@@ -1,79 +0,0 @@
-use std::{
-    fs::FileType,
-    path::{Path, PathBuf},
-};
-
-use anyhow::Result;
-use xshell::rm_rf;
-
-use crate::flags;
-
-impl flags::PreCache {
-    /// Cleans the `./target` dir after the build such that only
-    /// dependencies are cached on CI.
-    pub(crate) fn run(self) -> Result<()> {
-        let slow_tests_cookie = Path::new("./target/.slow_tests_cookie");
-        if !slow_tests_cookie.exists() {
-            panic!("slow tests were skipped on CI!")
-        }
-        rm_rf(slow_tests_cookie)?;
-
-        for path in read_dir("./target/debug", FileType::is_file)? {
-            // Can't delete yourself on windows :-(
-            if !path.ends_with("xtask.exe") {
-                rm_rf(&path)?
-            }
-        }
-
-        rm_rf("./target/.rustc_info.json")?;
-
-        let to_delete = read_dir("./crates", FileType::is_dir)?
-            .into_iter()
-            .map(|path| path.file_name().unwrap().to_string_lossy().replace('-', "_"))
-            .collect::<Vec<_>>();
-
-        for &dir in ["./target/debug/deps", "target/debug/.fingerprint"].iter() {
-            for path in read_dir(dir, |_file_type| true)? {
-                if path.ends_with("xtask.exe") {
-                    continue;
-                }
-                let file_name = path.file_name().unwrap().to_string_lossy();
-                let (stem, _) = match rsplit_once(&file_name, '-') {
-                    Some(it) => it,
-                    None => {
-                        rm_rf(path)?;
-                        continue;
-                    }
-                };
-                let stem = stem.replace('-', "_");
-                if to_delete.contains(&stem) {
-                    rm_rf(path)?;
-                }
-            }
-        }
-
-        Ok(())
-    }
-}
-fn read_dir(path: impl AsRef<Path>, cond: impl Fn(&FileType) -> bool) -> Result<Vec<PathBuf>> {
-    read_dir_impl(path.as_ref(), &cond)
-}
-
-fn read_dir_impl(path: &Path, cond: &dyn Fn(&FileType) -> bool) -> Result<Vec<PathBuf>> {
-    let mut res = Vec::new();
-    for entry in path.read_dir()? {
-        let entry = entry?;
-        let file_type = entry.file_type()?;
-        if cond(&file_type) {
-            res.push(entry.path())
-        }
-    }
-    Ok(res)
-}
-
-fn rsplit_once(haystack: &str, delim: char) -> Option<(&str, &str)> {
-    let mut split = haystack.rsplitn(2, delim);
-    let suffix = split.next()?;
-    let prefix = split.next()?;
-    Some((prefix, suffix))
-}