A runner tries a fetch again after a network hiccup f53e6d47b56a

- **A runner tries a fetch again after a network hiccup (oxygenhub#111).** Fetching or cloning the job's repository or a subrepo is retried on DNS, connection, timeout and 502/503/504 errors, waiting 2, 4, 8, 16 and 30 seconds; any other error still ends the job at once.

/// Whether a network error is worth another try: DNS, a refused or reset

/// connection, a timeout, or a gateway answering 502, 503 or 504 while the

/// hub restarts (oxygenhub#111).

fn transient(msg: &str) -> bool {

    let m = msg.to_lowercase();

    [

        "dns failed",

        "name resolution",

        "connection refused",

        "connection reset",

        "timed out",

        "timeout",

        "http 502",

        "http 503",

        "http 504",

        "broken pipe",

        "unexpected eof",

    ]

    .iter()

    .any(|k| m.contains(k))

}

/// Run `f` again after a transient network error, waiting `waits` between

/// tries (about a minute in all), and give up on any other error at once.

fn with_retries<T>(waits: &[Duration], mut f: impl FnMut() -> Result<T>) -> Result<T> {

    let mut tries = waits.iter();

    loop {

        match f() {

            Ok(v) => return Ok(v),

            Err(e) => match tries.next() {

                Some(w) if transient(&format!("{e:#}")) => std::thread::sleep(*w),

                _ => return Err(e),

            },

        }

    }

}

/// The waits between tries of a fetch: 2, 4, 8, 16 and 30 seconds.

const FETCH_WAITS: [Duration; 5] =

    [Duration::from_secs(2), Duration::from_secs(4), Duration::from_secs(8), Duration::from_secs(16), Duration::from_secs(30)];

        with_retries(&FETCH_WAITS, || repo.fetch("origin").map(|_| ()).map_err(|e| anyhow::anyhow!("fetch {clone_url}: {e}")))?;

    let cloned = with_retries(&FETCH_WAITS, || {

        let _ = std::fs::remove_dir_all(&dir);

        Repo::clone(clone_url, &dir).map_err(|e| anyhow::anyhow!("{e}"))

    });

    #[test]

    fn transient_network_errors_are_tried_again_and_others_are_not() {

        assert!(transient("fetch x: Dns Failed: resolve dns name 'oxygenhub.io:443': Temporary failure in name resolution"));

        assert!(transient("HTTP 503: the hub is restarting"));

        assert!(!transient("HTTP 404: no such repository"));

        let mut n = 0;

        let got: Result<u32> = with_retries(&[Duration::ZERO, Duration::ZERO], || {

            n += 1;

            if n < 3 {

                Err(anyhow::anyhow!("connection refused"))

            } else {

                Ok(7)

            }

        });

        assert_eq!((got.unwrap(), n), (7, 3), "two retries, then success");

        let mut m = 0;

        let got: Result<u32> = with_retries(&[Duration::ZERO; 4], || {

            m += 1;

            Err(anyhow::anyhow!("HTTP 403: forbidden"))

        });

        assert!(got.is_err());

        assert_eq!(m, 1, "a refusal is not tried again");

        let mut k = 0;

        let got: Result<u32> = with_retries(&[Duration::ZERO; 2], || {

            k += 1;

            Err(anyhow::anyhow!("timed out"))

        });

        assert!(got.is_err());

        assert_eq!(k, 3, "gives up after the last wait");

    }