From caacef29e6ad63d6b588715e809e78286d29db3f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 11 Jan 2023 14:14:14 +0000 Subject: [PATCH] Retry downloads on network failure Fixes #1280 --- build_system/prepare.rs | 20 +++++++++++++++++--- build_system/utils.rs | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 7c10b9b0c0f..106b06296b4 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -8,7 +8,7 @@ use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC}; use super::path::{Dirs, RelPath}; use super::rustc_info::{get_file_name, get_rustc_version}; -use super::utils::{copy_dir_recursively, spawn_and_wait, Compiler}; +use super::utils::{copy_dir_recursively, retry_spawn_and_wait, spawn_and_wait, Compiler}; pub(crate) fn prepare(dirs: &Dirs) { if RelPath::DOWNLOAD.to_path(dirs).exists() { @@ -140,8 +140,22 @@ fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo: // Download zip archive let mut download_cmd = Command::new("curl"); - download_cmd.arg("--location").arg("--output").arg(&archive_file).arg(archive_url); - spawn_and_wait(download_cmd); + download_cmd + .arg("--max-time") + .arg("600") + .arg("-y") + .arg("30") + .arg("-Y") + .arg("10") + .arg("--connect-timeout") + .arg("30") + .arg("--continue-at") + .arg("-") + .arg("--location") + .arg("--output") + .arg(&archive_file) + .arg(archive_url); + retry_spawn_and_wait(5, download_cmd); // Unpack tar archive let mut unpack_cmd = Command::new("tar"); diff --git a/build_system/utils.rs b/build_system/utils.rs index 995918ee143..3c27af0196e 100644 --- a/build_system/utils.rs +++ b/build_system/utils.rs @@ -188,6 +188,22 @@ pub(crate) fn spawn_and_wait(mut cmd: Command) { } } +// Based on the retry function in rust's src/ci/shared.sh +#[track_caller] +pub(crate) fn retry_spawn_and_wait(tries: u64, mut cmd: Command) { + for i in 1..tries+1 { + if i != 1 { + println!("Command failed. Attempt {i}/{tries}:"); + } + if cmd.spawn().unwrap().wait().unwrap().success() { + return; + } + std::thread::sleep(std::time::Duration::from_secs(i * 5)); + } + println!("The command has failed after {tries} attempts."); + process::exit(1); +} + #[track_caller] pub(crate) fn spawn_and_wait_with_input(mut cmd: Command, input: String) -> String { let mut child = cmd