feat: add backoff

Signed-off-by: kjuulh <contact@kjuulh.io>

Cargo.lock

@@ -186,6 +186,18 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "backon"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0"
+dependencies = [
+ "fastrand",
+ "futures-core",
+ "pin-project",
+ "tokio",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.71"
@@ -344,6 +356,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "axum",
+ "backon",
  "clap",
  "dagger-sdk",
  "dotenv",

crates/contractor/Cargo.toml

@@ -22,3 +22,4 @@ itertools = "0.12.1"
 regex = "1.10.4"
 serde_json = "1.0.115"
 dagger-sdk = "0.9.8"
+backon = "0.4.4"

crates/contractor/src/services/gitea.rs

@@ -241,18 +241,35 @@ impl DefaultGiteaClient {
 
         tracing::trace!("calling url: {}", &url);
 
-        let response = client
+        let response = (|| async {
+            client
                 .get(&url)
                 .header("Content-Type", "application/json")
                 .header("Authorization", format!("token {}", self.token))
                 .send()
+                .await
+        })
+        .retry(&ExponentialBuilder::default())
+        .notify(|err, dur| {
+            tracing::debug!("retrying job: {err}, in: {} seconds", dur.as_secs());
+        })
         .await?;
 
         match response.error_for_status() {
             Ok(_) => Ok(Some(())),
             Err(e) => match e.status() {
                 Some(StatusCode::NOT_FOUND) => Ok(None),
-                _ => anyhow::bail!(e),
+                Some(status) => {
+                    tracing::warn!(
+                        "failed to call fetch renovate for: {}, with error: {}",
+                        &repo,
+                        status
+                    );
+                    anyhow::bail!(e)
+                }
+                _ => {
+                    anyhow::bail!(e)
+                }
             },
         }
     }
@@ -267,11 +284,18 @@ impl DefaultGiteaClient {
 
         tracing::trace!("calling url: {}", &url);
 
-        let response = client
+        let response = (|| async {
+            client
                 .get(&url)
                 .header("Content-Type", "application/json")
                 .header("Authorization", format!("token {}", self.token))
                 .send()
+                .await
+        })
+        .retry(&ExponentialBuilder::default())
+        .notify(|err, dur| {
+            tracing::debug!("retrying job: {err}, in: {} seconds", dur.as_secs());
+        })
         .await?;
 
         let webhooks = response.json::<Vec<GiteaWebhook>>().await?;
@@ -301,13 +325,20 @@ impl DefaultGiteaClient {
             serde_json::to_string(&val)?
         );
 
-        let response = client
+        let response = (|| async {
+            client
                 .post(&url)
                 .header("Content-Type", "application/json")
                 .header("Accept", "application/json")
                 .header("Authorization", format!("token {}", self.token))
                 .json(&val)
                 .send()
+                .await
+        })
+        .retry(&ExponentialBuilder::default())
+        .notify(|err, dur| {
+            tracing::debug!("retrying job: {err}, in: {} seconds", dur.as_secs());
+        })
         .await?;
 
         if let Err(e) = response.error_for_status_ref() {
@@ -351,13 +382,20 @@ impl DefaultGiteaClient {
             serde_json::to_string(&val)?
         );
 
-        let response = client
+        let response = (|| async {
+            client
                 .patch(&url)
                 .header("Content-Type", "application/json")
                 .header("Accept", "application/json")
                 .header("Authorization", format!("token {}", self.token))
                 .json(&val)
                 .send()
+                .await
+        })
+        .retry(&ExponentialBuilder::default())
+        .notify(|err, dur| {
+            tracing::debug!("retrying job: {err}, in: {} seconds", dur.as_secs());
+        })
         .await?;
 
         if let Err(e) = response.error_for_status_ref() {
@@ -463,6 +501,7 @@ mod extensions;
 pub mod traits;
 
 use anyhow::Context;
+use backon::{ExponentialBuilder, Retryable};
 pub use extensions::*;
 use futures::{stream::FuturesUnordered, TryStreamExt};
 use reqwest::{StatusCode, Url};

crates/contractor/src/services/reconciler.rs

@@ -103,6 +103,7 @@ impl Reconciler {
         }
 
         let mut enabled = Vec::new();
+
         while let Some(res) = futures.next().await {
             let res = res?;