The runner tries hub requests again after server errors 1b2229dab1a4

    /// POST to the hub. A server error (5xx) means nothing was committed,

    /// so it is tried again after 1, 2 and 4 seconds before giving up: a

    /// busy hub costs a job a few seconds, not its log lines or its claim.

        self.post_with(path, body, &RETRY_DELAYS)

    }

    fn post_with(&self, path: &str, body: &impl serde::Serialize, delays: &[Duration]) -> Result<Option<ureq::Response>> {

        let mut delays = delays.iter();

        loop {

            match self.agent.post(&url).set("Authorization", &format!("Bearer {}", self.token)).send_json(body) {

                Ok(r) if r.status() == 204 => return Ok(None),

                Ok(r) => return Ok(Some(r)),

                Err(ureq::Error::Status(code, r)) => {

                    let text = r.into_string().unwrap_or_default();

                    match delays.next() {

                        Some(d) if code >= 500 => std::thread::sleep(*d),

                        _ => bail!("{url}: HTTP {code}: {text}"),

                    }

                }

                Err(e) => bail!("{url}: {e}"),

            }

/// Waits before trying a hub request again after a server error.

const RETRY_DELAYS: [Duration; 3] = [Duration::from_secs(1), Duration::from_secs(2), Duration::from_secs(4)];

    /// A hub that answers each request with the next of `codes`, counting them.

    fn fake_hub(codes: Vec<u16>) -> (String, std::sync::Arc<std::sync::atomic::AtomicUsize>) {

        use std::io::{Read, Write};

        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();

        let addr = listener.local_addr().unwrap();

        let seen = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));

        let count = seen.clone();

        std::thread::spawn(move || {

            for code in codes {

                let (mut s, _) = listener.accept().unwrap();

                let mut buf = [0u8; 8192];

                let _ = s.read(&mut buf);

                count.fetch_add(1, std::sync::atomic::Ordering::SeqCst);

                let reason = if code == 204 { "No Content" } else { "Error" };

                let _ = write!(s, "HTTP/1.1 {code} {reason}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");

            }

        });

        (format!("http://{addr}"), seen)

    }

    fn hub_at(base: String) -> Hub {

        Hub { base, token: "t".into(), agent: ureq::AgentBuilder::new().timeout(Duration::from_secs(5)).build() }

    }

    #[test]

    fn server_errors_are_tried_again_and_client_errors_are_not() {

        let quick = [Duration::from_millis(10); 3];

        let (base, seen) = fake_hub(vec![500, 503, 204]);

        let r = hub_at(base).post_with("/flow/jobs/1/log", &serde_json::json!({}), &quick).unwrap();

        assert!(r.is_none());

        assert_eq!(seen.load(std::sync::atomic::Ordering::SeqCst), 3, "two retries, then accepted");

        let (base, seen) = fake_hub(vec![500, 500, 500, 500]);

        let e = hub_at(base).post_with("/flow/claim", &serde_json::json!({}), &quick).unwrap_err().to_string();

        assert!(e.contains("HTTP 500"), "{e}");

        assert_eq!(seen.load(std::sync::atomic::Ordering::SeqCst), 4, "gives up after the last delay");

        let (base, seen) = fake_hub(vec![401]);

        assert!(hub_at(base).post_with("/flow/claim", &serde_json::json!({}), &quick).is_err());

        assert_eq!(seen.load(std::sync::atomic::Ordering::SeqCst), 1, "a refusal is final");

    }