feat: with ultra caching

Signed-off-by: kjuulh <contact@kjuulh.io>

Cargo.lock

@@ -249,6 +249,7 @@ dependencies = [
  "color-eyre",
  "dagger-sdk",
  "eyre",
+ "futures",
  "tokio",
 ]
 

ci/Cargo.toml

@@ -11,3 +11,4 @@ eyre = "*"
 color-eyre = "*"
 tokio = "1"
 clap = {version = "4", features = ["derive"]}
+futures = "0.3.28"

ci/src/main.rs

@@ -7,6 +7,12 @@ use clap::Parser;
 use clap::Subcommand;
 use clap::ValueEnum;
 
+use dagger_sdk::ContainerId;
+use dagger_sdk::ContainerPublishOpts;
+use dagger_sdk::Platform;
+use dagger_sdk::QueryContainerOpts;
+use futures::StreamExt;
+
 #[derive(Parser)]
 #[command(author, version, about, long_about = None, subcommand_required = true)]
 pub struct Command {
@@ -34,11 +40,17 @@ pub enum LocalCommands {
     Build {
         #[arg(long, default_value = "debug")]
         profile: BuildProfile,
+        #[arg(long)]
+        bin_name: String,
     },
     Test,
     DockerImage {
         #[arg(long)]
-        tag: Option<String>,
+        image: String,
+        #[arg(long)]
+        tag: String,
+        #[arg(long)]
+        bin_name: String,
     },
     PleaseRelease,
 }
@@ -57,6 +69,9 @@ pub struct GlobalArgs {
     #[arg(long, global = true, help_heading = "Global")]
     rust_builder_image: Option<String>,
 
+    #[arg(long, global = true, help_heading = "Global")]
+    production_image: Option<String>,
+
     #[arg(long, global = true, help_heading = "Global")]
     source: Option<PathBuf>,
 }
@@ -71,12 +86,113 @@ async fn main() -> eyre::Result<()> {
 
     match cli.commands {
         Commands::Local { command } => match command {
-            LocalCommands::Build { .. } => {
+            LocalCommands::Build {
-                let base_image = base_rust_image(client.clone(), &cli.global).await?;
+                profile: _,
-                build::execute(client, &cli.global, base_image).await?;
+                bin_name,
+            } => {
+                let base_image =
+                    base_rust_image(client.clone(), &cli.global, None, bin_name.clone()).await?;
+                let prod_image = get_base_debian_image(client.clone(), &cli.global, None).await?;
+                build::execute(client, &cli.global, base_image, prod_image, bin_name, None).await?;
+            }
+            LocalCommands::Test => {
+                let base_image =
+                    base_rust_image(client.clone(), &cli.global, None, "cuddle-please".into())
+                        .await?;
+                test::execute(client, &cli.global, base_image).await?;
+            }
+            LocalCommands::DockerImage {
+                tag,
+                image,
+                bin_name,
+            } => {
+                // let containers = vec!["linux/amd64", "linux/arm64"];
+
+                let containers = vec!["linux/amd64"];
+
+                let stream = futures::stream::iter(containers.into_iter().map(|c| {
+                    let client = Arc::new(
+                        client
+                            .clone()
+                            .pipeline(format!("docker:build:platform:{c}")),
+                    );
+                    let args = cli.global.clone();
+                    let platform = c.to_string();
+                    let bin_name = bin_name.clone();
+                    tokio::spawn(async move {
+                        let base_image = match get_base_debian_image(
+                            client.clone(),
+                            &args.clone(),
+                            Some(platform.clone()),
+                        )
+                        .await
+                        {
+                            Ok(image) => image,
+                            Err(e) => {
+                                eprintln!("failed to get base image: {e}");
+                                return None;
+                            }
+                        };
+
+                        match base_rust_image(
+                            client.clone(),
+                            &args,
+                            Some(platform.clone()),
+                            bin_name.clone(),
+                        )
+                        .await
+                        {
+                            Ok(container) => {
+                                let build_image = match build::execute(
+                                    client,
+                                    &args,
+                                    container,
+                                    base_image,
+                                    bin_name,
+                                    Some(platform),
+                                )
+                                .await
+                                {
+                                    Ok(image) => image,
+                                    Err(e) => {
+                                        eprintln!("could not build image: {e}");
+
+                                        return None;
+                                    }
+                                };
+
+                                match build_image.id().await {
+                                    Ok(id) => return Some(id),
+                                    Err(e) => {
+                                        eprintln!("could not get id: {e}");
+                                    }
+                                }
+                            }
+                            Err(e) => {
+                                eprintln!("could not build container: {e}");
+                            }
+                        }
+
+                        None
+                    })
+                }))
+                .buffer_unordered(16)
+                .filter_map(|f| async move { f.ok() })
+                .collect::<Vec<_>>()
+                .await;
+
+                let _container = client
+                    .container()
+                    .publish_opts(
+                        format!("{image}:{tag}"),
+                        ContainerPublishOpts {
+                            platform_variants: stream
+                                .into_iter()
+                                .collect::<Option<Vec<ContainerId>>>(),
+                        },
+                    )
+                    .await?;
             }
-            LocalCommands::Test => todo!(),
-            LocalCommands::DockerImage { .. } => todo!(),
             LocalCommands::PleaseRelease => todo!(),
         },
         Commands::PullRequest => todo!(),
@@ -90,29 +206,103 @@ async fn main() -> eyre::Result<()> {
 mod build {
     use std::sync::Arc;
 
+    use dagger_sdk::Container;
+
     use crate::GlobalArgs;
 
     pub async fn execute(
-        client: Arc<dagger_sdk::Query>,
+        _client: Arc<dagger_sdk::Query>,
+        _args: &GlobalArgs,
+        container: dagger_sdk::Container,
+        base_image: dagger_sdk::Container,
+        bin_name: String,
+        platform: Option<String>,
+    ) -> eyre::Result<Container> {
+        let rust_target = match platform.unwrap_or("linux/amd64".to_string()).as_str() {
+            "linux/amd64" => "x86_64-unknown-linux-gnu",
+            "linux/arm64" => "aarch64-unknown-linux-gnu",
+            _ => eyre::bail!("architecture not supported"),
+        };
+        let build_image = container.with_exec(vec![
+            "cargo",
+            "build",
+            "--target",
+            rust_target,
+            "--release",
+            "-p",
+            &bin_name,
+        ]);
+
+        let final_image = base_image
+            .with_file(
+                format!("/usr/local/bin/{}", &bin_name),
+                build_image
+                    .file(format!("target/{}/release/{}", rust_target, &bin_name))
+                    .id()
+                    .await?,
+            )
+            .with_exec(vec![&bin_name, "--help"]);
+
+        let output = final_image.stdout().await?;
+        println!("{output}");
+
+        //.with_entrypoint(vec![&bin_name, "--log-level=debug"]);
+
+        Ok(final_image)
+    }
+}
+
+mod test {
+    use std::sync::Arc;
+
+    use crate::GlobalArgs;
+
+    pub async fn execute(
+        _client: Arc<dagger_sdk::Query>,
         _args: &GlobalArgs,
         container: dagger_sdk::Container,
     ) -> eyre::Result<()> {
-        let build_image = container
+        let test_image = container
-            .pipeline("rust:build")
+            .pipeline("rust:test")
-            .with_exec(vec!["cargo", "build"]);
+            .with_exec(vec!["cargo", "test"]);
 
-        build_image.exit_code().await?;
+        test_image.exit_code().await?;
-
-        let out = build_image
-            .with_exec(vec!["./target/debug/cuddle-please", "-h"])
-            .stdout()
-            .await?;
-        println!("stdout: {}", out);
 
         Ok(())
     }
 }
 
+pub async fn get_base_debian_image(
+    client: Arc<dagger_sdk::Query>,
+    args: &GlobalArgs,
+    platform: Option<String>,
+) -> eyre::Result<dagger_sdk::Container> {
+    let default_platform = client.default_platform().await?;
+    let platform = platform.map(Platform).unwrap_or(default_platform);
+
+    let image = client
+        .container_opts(QueryContainerOpts {
+            id: None,
+            platform: Some(platform),
+        })
+        .from(
+            args.production_image
+                .clone()
+                .unwrap_or("debian:bullseye".to_string()),
+        );
+
+    let base_image = image.with_exec(vec!["apt", "update"]).with_exec(vec![
+        "apt",
+        "install",
+        "-y",
+        "libssl-dev",
+        "pkg-config",
+        "openssl",
+    ]);
+
+    Ok(base_image)
+}
+
 pub fn get_src(
     client: Arc<dagger_sdk::Query>,
     args: &GlobalArgs,
@@ -146,6 +336,13 @@ pub async fn get_rust_dep_src(
             .build()?,
     );
 
+    return Ok(directory);
+}
+
+pub async fn get_rust_skeleton_files(
+    client: Arc<dagger_sdk::Query>,
+    args: &GlobalArgs,
+) -> eyre::Result<(dagger_sdk::Directory, Vec<String>)> {
     let mut rust_crates = vec![PathBuf::from("ci")];
     let mut dirs = tokio::fs::read_dir("crates").await?;
 
@@ -159,8 +356,13 @@ pub async fn get_rust_dep_src(
         directory: dagger_sdk::Directory,
         path: &Path,
     ) -> eyre::Result<dagger_sdk::Directory> {
-        let main_content = r#"fn main() {}"#;
+        println!("found crates: {}", path.display());
-        let lib_content = r#"fn some() {}"#;
+        let main_content = r#"
+        #[allow(dead_code)]
+        fn main() { panic!("should never be executed"); }"#;
+        let lib_content = r#"
+        #[allow(dead_code)]
+        fn some() { panic!("should never be executed"); }"#;
 
         let directory = directory.with_new_file(
             path.join("src").join("main.rs").display().to_string(),
@@ -174,38 +376,100 @@ pub async fn get_rust_dep_src(
         Ok(directory)
     }
 
-    let mut directory = directory;
+    let mut directory = client.directory();
+    let mut crate_names = Vec::new();
 
-    for rust_crate in rust_crates.into_iter() {
+    for rust_crate in rust_crates.iter() {
-        directory = create_skeleton_files(directory, &rust_crate)?
+        if let Some(file_name) = rust_crate.file_name() {
+            crate_names.push(file_name.to_str().unwrap().to_string());
+        }
+        directory = create_skeleton_files(directory, &rust_crate)?;
     }
 
-    Ok(directory)
+    Ok((directory, crate_names))
 }
 
 pub async fn base_rust_image(
     client: Arc<dagger_sdk::Query>,
     args: &GlobalArgs,
+    platform: Option<String>,
+    bin_name: String,
 ) -> eyre::Result<dagger_sdk::Container> {
     let dep_src = get_rust_dep_src(client.clone(), args).await?;
+    let (skeleton_files, crates) = get_rust_skeleton_files(client.clone(), args).await?;
     let src = get_src(client.clone(), args)?;
 
     let client = client.pipeline("rust_base_image");
 
-    let rust_build_image = client.container().from(
+    let rust_target = match platform.unwrap_or("linux/amd64".to_string()).as_str() {
-        args.rust_builder_image
+        "linux/amd64" => "x86_64-unknown-linux-gnu",
-            .as_ref()
+        "linux/arm64" => "aarch64-unknown-linux-gnu",
-            .unwrap_or(&"rustlang/rust:nightly".into()),
+        _ => eyre::bail!("architecture not supported"),
-    );
+    };
+    let rust_build_image = client
+        // .container_opts(QueryContainerOpts {
+        //     id: None,
+        //     platform: Some(
+        //         platform
+        //             .map(|p| Platform(p))
+        //             .unwrap_or(client.default_platform().await?),
+        //     ),
+        // })
+        .container()
+        .from(
+            args.rust_builder_image
+                .as_ref()
+                .unwrap_or(&"rustlang/rust:nightly".into()),
+        )
+        .with_exec(vec!["rustup", "target", "add", rust_target]);
 
     let target_cache = client.cache_volume("rust_target");
 
-    let rust_build_image = rust_build_image
+    let rust_prebuild = rust_build_image
         .with_workdir("/mnt/src")
         .with_directory("/mnt/src", dep_src.id().await?)
-        .with_exec(vec!["cargo", "build"])
+        .with_directory("/mnt/src/", skeleton_files.id().await?)
-        .with_mounted_cache("/mnt/src/target/", target_cache.id().await?)
+        .with_exec(vec![
-        .with_directory("/mnt/src/crates", src.directory("crates").id().await?);
+            "cargo",
+            "build",
+            "--target",
+            rust_target,
+            "--release",
+            "-p",
+            &bin_name,
+        ])
+        .with_mounted_cache("/mnt/src/target/", target_cache.id().await?);
 
-    Ok(rust_build_image)
+    let exclude = crates
+        .iter()
+        .filter(|c| **c != "ci")
+        .map(|c| format!("**/*{}*", c.replace("-", "_")))
+        .collect::<Vec<_>>();
+
+    let exclude = exclude.iter().map(|c| c.as_str()).collect();
+
+    let incremental_dir = client.directory().with_directory_opts(
+        ".",
+        rust_prebuild.directory("target").id().await?,
+        dagger_sdk::DirectoryWithDirectoryOpts {
+            exclude: Some(exclude),
+            include: None,
+        },
+    );
+
+    let rust_with_src = rust_build_image
+        .with_workdir("/mnt/src")
+        .with_directory(
+            "/usr/local/cargo",
+            rust_prebuild.directory("/usr/local/cargo").id().await?,
+        )
+        .with_directory(
+            //format!("/mnt/src/target/{}/release/build", rust_target),
+            "target/",
+            //rust_prebuild.id().await?,
+            incremental_dir.id().await?,
+        )
+        .with_directory("/mnt/src/", src.id().await?);
+
+    Ok(rust_with_src)
 }

crates/cuddle-please-commands/src/command.rs

@@ -12,7 +12,7 @@ use cuddle_please_misc::{
     VcsClient,
 };
 use tracing::Level;
-use tracing_subscriber::{prelude::__tracing_subscriber_SubscriberExt, EnvFilter};
+use tracing_subscriber::{EnvFilter};
 
 use crate::{
     config_command::{ConfigCommand, ConfigCommandHandler},

crates/cuddle-please-commands/src/release_command.rs

@@ -52,7 +52,7 @@ impl ReleaseCommandHandler {
         tracing::trace!("found current version: {}", current_version.to_string());
         self.ui.write_str_ln(&format!(
             "found current version: {}",
-            current_version.to_string()
+            current_version
         ));
 
         let conventional_commit_results = parse_conventional_commits(current_version, commits)?;
@@ -66,7 +66,7 @@ impl ReleaseCommandHandler {
         let (commit_strs, next_version) = conventional_commit_results.unwrap();
         self.ui.write_str_ln(&format!(
             "calculated next version: {}",
-            next_version.to_string()
+            next_version
         ));
 
         tracing::trace!("creating changelog");
@@ -112,7 +112,7 @@ impl ReleaseCommandHandler {
         next_version: &Version,
         changelog_last_changes: Option<String>,
     ) -> Result<(), anyhow::Error> {
-        Ok(if !dry_run {
+        if !dry_run {
             self.gitea_client.create_release(
                 owner,
                 repository,
@@ -122,7 +122,8 @@ impl ReleaseCommandHandler {
             )?;
         } else {
             tracing::debug!("creating release (dry_run)");
-        })
+        };
+        Ok(())
     }
 
     fn create_pull_request(
@@ -202,7 +203,7 @@ impl ReleaseCommandHandler {
     ) -> Result<Option<Tag>, anyhow::Error> {
         let tags = self.gitea_client.get_tags(owner, repository)?;
         let significant_tag = get_most_significant_version(tags.iter().collect());
-        Ok(significant_tag.map(|t| t.clone()))
+        Ok(significant_tag.cloned())
     }
 
     fn check_git_remote_connection(
@@ -252,8 +253,8 @@ fn parse_conventional_commits(
 }
 
 fn get_current_version(significant_tag: Option<Tag>) -> Version {
-    let current_version = significant_tag
+    
+    significant_tag
         .map(|st| Version::try_from(st).unwrap())
-        .unwrap_or(Version::new(0, 0, 0));
+        .unwrap_or(Version::new(0, 0, 0))
-    current_version
 }

crates/cuddle-please-frontend/src/gatheres/config_file.rs

@@ -31,10 +31,8 @@ const YAML_EXTENSION: &str = "yaml";
 
 pub fn get_config_from_config_file(current_dir: &Path) -> PleaseConfigBuilder {
     let current_cuddle_path = current_dir
-        .clone()
         .join(format!("{CUDDLE_FILE_NAME}.{YAML_EXTENSION}"));
     let current_cuddle_config_path = current_dir
-        .clone()
         .join(format!("{CUDDLE_CONFIG_FILE_NAME}.{YAML_EXTENSION}"));
     let mut please_config = PleaseConfigBuilder::default();
 

crates/cuddle-please-misc/src/git_client.rs

@@ -59,7 +59,7 @@ impl VcsClient {
             } => {
                 let checkout_branch = std::process::Command::new("git")
                     .current_dir(source.as_path())
-                    .args(&[
+                    .args([
                         "-c",
                         &format!("user.name={}", username),
                         "-c",

crates/cuddle-please/src/main.rs

@@ -8,5 +8,7 @@ fn main() -> anyhow::Result<()> {
 
     PleaseCommand::new().execute(current_dir)?;
 
+    //something else asdfa 123
+
     Ok(())
 }