From 8ee05136df1233fe99d555159546f5c86a0c76df Mon Sep 17 00:00:00 2001 From: kjuulh Date: Thu, 3 Aug 2023 16:09:45 +0200 Subject: [PATCH] feat: ci:main script for ci Signed-off-by: kjuulh --- Cargo.lock | 33 ++ ci/Cargo.toml | 1 + ci/src/main.rs | 352 +++++++++++++------ crates/cuddle-please-commands/src/command.rs | 2 +- crates/cuddle-please/tests/git.rs | 6 + cuddle.yaml | 10 + scripts/ci:main.sh | 10 + scripts/local:docker:docs.sh | 5 + scripts/publish:docker:docs.sh | 1 + templates/Caddyfile | 8 + 10 files changed, 314 insertions(+), 114 deletions(-) create mode 100755 scripts/ci:main.sh create mode 100755 scripts/local:docker:docs.sh create mode 100644 scripts/publish:docker:docs.sh create mode 100644 templates/Caddyfile diff --git a/Cargo.lock b/Cargo.lock index e818689..d19cf6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,18 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" +[[package]] +name = "async-scoped" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7a6a57c8aeb40da1ec037f5d455836852f7a57e69e1b1ad3d8f38ac1d6cadf" +dependencies = [ + "futures", + "pin-project", + "slab", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.72" @@ -245,6 +257,7 @@ dependencies = [ name = "ci" version = "0.1.0" dependencies = [ + "async-scoped", "clap", "color-eyre", "dagger-sdk", @@ -1728,6 +1741,26 @@ dependencies = [ "uncased", ] +[[package]] +name = "pin-project" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + [[package]] name = "pin-project-lite" version = "0.2.10" diff --git a/ci/Cargo.toml b/ci/Cargo.toml index 0412740..8ed23a8 100644 --- a/ci/Cargo.toml +++ b/ci/Cargo.toml @@ -12,3 +12,4 @@ color-eyre = "*" tokio = "1" clap = {version = "4", features = ["derive"]} futures = "0.3.28" +async-scoped = { version = "0.7.1", features = ["tokio", "use-tokio"] } diff --git a/ci/src/main.rs b/ci/src/main.rs index db0102d..f308556 100644 --- a/ci/src/main.rs +++ b/ci/src/main.rs @@ -2,18 +2,21 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; + use clap::Args; use clap::Parser; use clap::Subcommand; use clap::ValueEnum; -use dagger_sdk::ContainerId; -use dagger_sdk::ContainerPublishOpts; + + use dagger_sdk::Platform; use dagger_sdk::QueryContainerOpts; use futures::StreamExt; -#[derive(Parser)] + + +#[derive(Parser, Clone)] #[command(author, version, about, long_about = None, subcommand_required = true)] pub struct Command { #[command(subcommand)] @@ -23,7 +26,7 @@ pub struct Command { global: GlobalArgs, } -#[derive(Subcommand)] +#[derive(Subcommand, Clone)] pub enum Commands { #[command(subcommand_required = true)] Local { @@ -31,11 +34,18 @@ pub enum Commands { command: LocalCommands, }, PullRequest, - Main, + Main { + #[arg(long)] + image: String, + #[arg(long)] + tag: String, + #[arg(long)] + bin_name: String, + }, Release, } -#[derive(Subcommand)] +#[derive(Subcommand, Clone)] pub enum LocalCommands { Build { #[arg(long, default_value = "debug")] @@ -53,6 +63,7 @@ pub enum LocalCommands { bin_name: String, }, PleaseRelease, + BuildDocs {}, } #[derive(Debug, Clone, ValueEnum)] @@ -72,8 +83,20 @@ pub struct GlobalArgs { #[arg(long, global = true, help_heading = "Global")] production_image: Option, + #[arg(long, global = true, help_heading = "Global")] + mkdocs_image: Option, + + #[arg(long, global = true, help_heading = "Global")] + caddy_image: Option, + #[arg(long, global = true, help_heading = "Global")] source: Option, + + #[arg(long, global = true, help_heading = "Global")] + docs_image: Option, + + #[arg(long, global = true, help_heading = "Global")] + docs_image_tag: Option, } #[tokio::main] @@ -84,20 +107,28 @@ async fn main() -> eyre::Result<()> { let cli = Command::parse(); - match cli.commands { + match &cli.commands { Commands::Local { command } => match command { LocalCommands::Build { profile: _, bin_name, } => { let base_image = - base_rust_image(client.clone(), &cli.global, None, bin_name.clone()).await?; + base_rust_image(client.clone(), &cli.global, &None, &bin_name.clone()).await?; let prod_image = get_base_debian_image(client.clone(), &cli.global, None).await?; - build::execute(client, &cli.global, base_image, prod_image, bin_name, None).await?; + build::execute( + client, + &cli.global, + &base_image, + &prod_image, + bin_name, + &None, + ) + .await?; } LocalCommands::Test => { let base_image = - base_rust_image(client.clone(), &cli.global, None, "cuddle-please".into()) + base_rust_image(client.clone(), &cli.global, &None, &"cuddle-please".into()) .await?; test::execute(client, &cli.global, base_image).await?; } @@ -106,119 +137,210 @@ async fn main() -> eyre::Result<()> { image, bin_name, } => { - // let containers = vec!["linux/amd64", "linux/arm64"]; - - let containers = vec!["linux/amd64"]; - - let stream = futures::stream::iter(containers.into_iter().map(|c| { - let client = Arc::new( - client - .clone() - .pipeline(format!("docker:build:platform:{c}")), - ); - let args = cli.global.clone(); - let platform = c.to_string(); - let bin_name = bin_name.clone(); - tokio::spawn(async move { - let base_image = match get_base_debian_image( - client.clone(), - &args.clone(), - Some(platform.clone()), - ) - .await - { - Ok(image) => image, - Err(e) => { - eprintln!("failed to get base image: {e}"); - return None; - } - }; - - match base_rust_image( - client.clone(), - &args, - Some(platform.clone()), - bin_name.clone(), - ) - .await - { - Ok(container) => { - let build_image = match build::execute( - client, - &args, - container, - base_image, - bin_name, - Some(platform), - ) - .await - { - Ok(image) => image, - Err(e) => { - eprintln!("could not build image: {e}"); - - return None; - } - }; - - match build_image.id().await { - Ok(id) => return Some(id), - Err(e) => { - eprintln!("could not get id: {e}"); - } - } - } - Err(e) => { - eprintln!("could not build container: {e}"); - } - } - - None - }) - })) - .buffer_unordered(16) - .filter_map(|f| async move { f.ok() }) - .collect::>() - .await; - - let _container = client - .container() - .publish_opts( - format!("{image}:{tag}"), - ContainerPublishOpts { - platform_variants: stream - .into_iter() - .collect::>>(), - }, - ) - .await?; + build::build_and_deploy(client, &cli.global, bin_name, image, tag).await?; } LocalCommands::PleaseRelease => todo!(), + LocalCommands::BuildDocs {} => { + let _image = docs::execute( + client.clone(), + &cli.global, + &Some("linux/amd64".to_string()), + ) + .await?; + } }, Commands::PullRequest => todo!(), - Commands::Main => todo!(), + Commands::Main { + image, + tag, + bin_name, + } => { + async fn test(client: Arc, cli: &Command, bin_name: &String) { + let args = &cli.global; + + let base_image = base_rust_image(client.clone(), args, &None, bin_name) + .await + .unwrap(); + test::execute(client.clone(), args, base_image) + .await + .unwrap(); + } + async fn build( + client: Arc, + cli: &Command, + bin_name: &String, + image: &String, + tag: &String, + ) { + let args = &cli.global; + + build::build_and_deploy(client.clone(), args, bin_name, image, tag) + .await + .unwrap(); + } + + tokio::join!( + test(client.clone(), &cli, bin_name), + build(client.clone(), &cli, bin_name, image, tag) + ); + } Commands::Release => todo!(), } Ok(()) } +mod docs { + use std::sync::Arc; + + use dagger_sdk::Container; + + use crate::GlobalArgs; + + pub fn get_docs_src(client: Arc) -> eyre::Result { + let docs_content = client.host().directory_opts( + ".", + dagger_sdk::HostDirectoryOpts { + exclude: None, + include: Some(vec!["mkdocs.yml", "docs/"]), + }, + ); + + Ok(docs_content) + } + + pub async fn execute( + client: Arc, + args: &GlobalArgs, + _platform: &Option, + ) -> eyre::Result { + let mkdocs_container = client.container().from( + args.mkdocs_image + .as_ref() + .expect("--mkdocs-image to be set"), + ); + + let built_mkdocs_container = mkdocs_container + .with_directory("/docs", get_docs_src(client.clone())?.id().await?) + .with_exec(vec!["build"]); + + let site_output = built_mkdocs_container.directory("/docs/site").id().await?; + + let caddy_file = client.host().directory("templates").file("Caddyfile"); + + let dep_image = client + .container() + .from(args.caddy_image.as_ref().expect("--caddy-image to be set")) + .with_directory("/usr/share/caddy", site_output) + .with_file("/etc/caddy/Caddyfile", caddy_file.id().await?) + .with_exec(vec!["echo", "caddy"]); + + Ok(dep_image) + } + + pub async fn publish( + client: Arc, + args: &GlobalArgs, + containers: &Vec, + ) -> eyre::Result<()> { + let container_ids = + futures::future::join_all(containers.iter().map(|c| c.id()).collect::>()).await; + + let container_ids = container_ids + .into_iter() + .collect::>>()?; + + client + .container() + .publish_opts( + format!( + "{}:{}", + args.docs_image.as_ref().expect("--docs-image to be set"), + args.docs_image_tag + .as_ref() + .expect("--docs-image-tag to be set") + ), + dagger_sdk::ContainerPublishOpts { + platform_variants: Some(container_ids), + }, + ) + .await?; + + Ok(()) + } +} + mod build { use std::sync::Arc; use dagger_sdk::Container; - use crate::GlobalArgs; + + + + use crate::{base_rust_image, get_base_debian_image, GlobalArgs}; + + pub async fn build_and_deploy( + client: Arc, + args: &GlobalArgs, + bin_name: &String, + image: &String, + tag: &String, + ) -> eyre::Result<()> { + // let containers = vec!["linux/amd64", "linux/arm64"]; + + let base_image = get_base_debian_image( + client.clone(), + &args.clone(), + Some("linux/amd64".to_string()), + ) + .await?; + + let container = base_rust_image( + client.clone(), + args, + &Some("linux/amd64".to_string()), + &bin_name.clone(), + ) + .await?; + let build_image = execute( + client.clone(), + args, + &container, + &base_image, + bin_name, + &Some("linux/amd64".to_string()), + ) + .await?; + + let build_id = build_image.id().await?; + + let _container = client + .clone() + .container() + .publish_opts( + format!("{image}:{tag}"), + dagger_sdk::ContainerPublishOpts { + platform_variants: Some(vec![build_id]), + }, + ) + .await?; + Ok(()) + } pub async fn execute( _client: Arc, _args: &GlobalArgs, - container: dagger_sdk::Container, - base_image: dagger_sdk::Container, - bin_name: String, - platform: Option, + container: &dagger_sdk::Container, + base_image: &dagger_sdk::Container, + bin_name: &String, + platform: &Option, ) -> eyre::Result { - let rust_target = match platform.unwrap_or("linux/amd64".to_string()).as_str() { + let rust_target = match platform + .clone() + .unwrap_or("linux/amd64".to_string()) + .as_str() + { "linux/amd64" => "x86_64-unknown-linux-gnu", "linux/arm64" => "aarch64-unknown-linux-gnu", _ => eyre::bail!("architecture not supported"), @@ -230,7 +352,7 @@ mod build { rust_target, "--release", "-p", - &bin_name, + bin_name, ]); let final_image = base_image @@ -241,7 +363,7 @@ mod build { .id() .await?, ) - .with_exec(vec![&bin_name, "--help"]); + .with_exec(vec![bin_name, "--help"]); let output = final_image.stdout().await?; println!("{output}"); @@ -336,12 +458,12 @@ pub async fn get_rust_dep_src( .build()?, ); - return Ok(directory); + Ok(directory) } pub async fn get_rust_skeleton_files( client: Arc, - args: &GlobalArgs, + _args: &GlobalArgs, ) -> eyre::Result<(dagger_sdk::Directory, Vec)> { let mut rust_crates = vec![PathBuf::from("ci")]; let mut dirs = tokio::fs::read_dir("crates").await?; @@ -383,7 +505,7 @@ pub async fn get_rust_skeleton_files( if let Some(file_name) = rust_crate.file_name() { crate_names.push(file_name.to_str().unwrap().to_string()); } - directory = create_skeleton_files(directory, &rust_crate)?; + directory = create_skeleton_files(directory, rust_crate)?; } Ok((directory, crate_names)) @@ -392,8 +514,8 @@ pub async fn get_rust_skeleton_files( pub async fn base_rust_image( client: Arc, args: &GlobalArgs, - platform: Option, - bin_name: String, + platform: &Option, + bin_name: &String, ) -> eyre::Result { let dep_src = get_rust_dep_src(client.clone(), args).await?; let (skeleton_files, crates) = get_rust_skeleton_files(client.clone(), args).await?; @@ -401,7 +523,11 @@ pub async fn base_rust_image( let client = client.pipeline("rust_base_image"); - let rust_target = match platform.unwrap_or("linux/amd64".to_string()).as_str() { + let rust_target = match platform + .clone() + .unwrap_or("linux/amd64".to_string()) + .as_str() + { "linux/amd64" => "x86_64-unknown-linux-gnu", "linux/arm64" => "aarch64-unknown-linux-gnu", _ => eyre::bail!("architecture not supported"), @@ -428,14 +554,14 @@ pub async fn base_rust_image( rust_target, "--release", "-p", - &bin_name, + bin_name, ]) .with_mounted_cache("/mnt/src/target/", target_cache.id().await?); let exclude = crates .iter() .filter(|c| **c != "ci") - .map(|c| format!("**/*{}*", c.replace("-", "_"))) + .map(|c| format!("**/*{}*", c.replace('-', "_"))) .collect::>(); let exclude = exclude.iter().map(|c| c.as_str()).collect(); diff --git a/crates/cuddle-please-commands/src/command.rs b/crates/cuddle-please-commands/src/command.rs index d5bdc13..3079e64 100644 --- a/crates/cuddle-please-commands/src/command.rs +++ b/crates/cuddle-please-commands/src/command.rs @@ -139,7 +139,7 @@ impl Command { tracing_subscriber::fmt().with_env_filter(env_filter).init(); } - return Ok((config, git_client, gitea_client)); + Ok((config, git_client, gitea_client)) } fn build_config(&self, current_dir: Option<&Path>) -> Result { diff --git a/crates/cuddle-please/tests/git.rs b/crates/cuddle-please/tests/git.rs index 953c937..60de0b3 100644 --- a/crates/cuddle-please/tests/git.rs +++ b/crates/cuddle-please/tests/git.rs @@ -165,6 +165,12 @@ fn setup_git(path: &Path) -> anyhow::Result<()> { let stderr = std::str::from_utf8(&output.stderr)?; tracing::debug!(stdout = stdout, stderr = stderr, "git init"); + exec_git(path, &["checkout", "-b", "main"])?; + + exec_git(path, &["config", "user.name", "test"])?; + exec_git(path, &["config", "user.email", "test@test.com"])?; + exec_git(path, &["config", "init.defaultBranch", "main"])?; + Ok(()) } diff --git a/cuddle.yaml b/cuddle.yaml index 7f69b90..f428c6a 100644 --- a/cuddle.yaml +++ b/cuddle.yaml @@ -5,7 +5,9 @@ base: "git@git.front.kjuulh.io:kjuulh/cuddle-rust-cli-plan.git" vars: service: "cuddle-please" registry: kasperhermansen + mkdocs_image: "squidfunk/mkdocs-material:9.1" + caddy_image: "caddy:2.7" please: project: @@ -24,4 +26,12 @@ scripts: type: shell "local:docker": type: shell + "local:docker:docs": + type: shell + "ci:main": + type: shell + "ci:pr": + type: shell + "ci:release": + type: shell \ No newline at end of file diff --git a/scripts/ci:main.sh b/scripts/ci:main.sh new file mode 100755 index 0000000..6af4ba4 --- /dev/null +++ b/scripts/ci:main.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e + +cargo run -p ci -- main \ + --mkdocs-image "$MKDOCS_IMAGE" \ + --caddy-image "$CADDY_IMAGE" \ + --image "$REGISTRY/$SERVICE" \ + --tag "main-$(date +%s)" \ + --bin-name "$SERVICE" \ No newline at end of file diff --git a/scripts/local:docker:docs.sh b/scripts/local:docker:docs.sh new file mode 100755 index 0000000..4002ca9 --- /dev/null +++ b/scripts/local:docker:docs.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -e + +cargo run -p ci -- local build-docs --mkdocs-image $MKDOCS_IMAGE --caddy-image $CADDY_IMAGE diff --git a/scripts/publish:docker:docs.sh b/scripts/publish:docker:docs.sh new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/scripts/publish:docker:docs.sh @@ -0,0 +1 @@ + diff --git a/templates/Caddyfile b/templates/Caddyfile new file mode 100644 index 0000000..9cf2d06 --- /dev/null +++ b/templates/Caddyfile @@ -0,0 +1,8 @@ +{ + debug +} + +http://blog.kasperhermansen.com { + root * /usr/share/caddy + file_server +}