feat: ci:main script for ci
Some checks failed
continuous-integration/drone/push Build is failing

Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
Kasper Juul Hermansen 2023-08-03 16:09:45 +02:00
parent f7d02bad10
commit 8ee05136df
Signed by: kjuulh
GPG Key ID: 9AA7BC13CE474394
10 changed files with 314 additions and 114 deletions

33
Cargo.lock generated
View File

@ -102,6 +102,18 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e"
[[package]]
name = "async-scoped"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7a6a57c8aeb40da1ec037f5d455836852f7a57e69e1b1ad3d8f38ac1d6cadf"
dependencies = [
"futures",
"pin-project",
"slab",
"tokio",
]
[[package]]
name = "async-trait"
version = "0.1.72"
@ -245,6 +257,7 @@ dependencies = [
name = "ci"
version = "0.1.0"
dependencies = [
"async-scoped",
"clap",
"color-eyre",
"dagger-sdk",
@ -1728,6 +1741,26 @@ dependencies = [
"uncased",
]
[[package]]
name = "pin-project"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.28",
]
[[package]]
name = "pin-project-lite"
version = "0.2.10"

View File

@ -12,3 +12,4 @@ color-eyre = "*"
tokio = "1"
clap = {version = "4", features = ["derive"]}
futures = "0.3.28"
async-scoped = { version = "0.7.1", features = ["tokio", "use-tokio"] }

View File

@ -2,18 +2,21 @@ use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use clap::Args;
use clap::Parser;
use clap::Subcommand;
use clap::ValueEnum;
use dagger_sdk::ContainerId;
use dagger_sdk::ContainerPublishOpts;
use dagger_sdk::Platform;
use dagger_sdk::QueryContainerOpts;
use futures::StreamExt;
#[derive(Parser)]
#[derive(Parser, Clone)]
#[command(author, version, about, long_about = None, subcommand_required = true)]
pub struct Command {
#[command(subcommand)]
@ -23,7 +26,7 @@ pub struct Command {
global: GlobalArgs,
}
#[derive(Subcommand)]
#[derive(Subcommand, Clone)]
pub enum Commands {
#[command(subcommand_required = true)]
Local {
@ -31,11 +34,18 @@ pub enum Commands {
command: LocalCommands,
},
PullRequest,
Main,
Main {
#[arg(long)]
image: String,
#[arg(long)]
tag: String,
#[arg(long)]
bin_name: String,
},
Release,
}
#[derive(Subcommand)]
#[derive(Subcommand, Clone)]
pub enum LocalCommands {
Build {
#[arg(long, default_value = "debug")]
@ -53,6 +63,7 @@ pub enum LocalCommands {
bin_name: String,
},
PleaseRelease,
BuildDocs {},
}
#[derive(Debug, Clone, ValueEnum)]
@ -72,8 +83,20 @@ pub struct GlobalArgs {
#[arg(long, global = true, help_heading = "Global")]
production_image: Option<String>,
#[arg(long, global = true, help_heading = "Global")]
mkdocs_image: Option<String>,
#[arg(long, global = true, help_heading = "Global")]
caddy_image: Option<String>,
#[arg(long, global = true, help_heading = "Global")]
source: Option<PathBuf>,
#[arg(long, global = true, help_heading = "Global")]
docs_image: Option<String>,
#[arg(long, global = true, help_heading = "Global")]
docs_image_tag: Option<String>,
}
#[tokio::main]
@ -84,20 +107,28 @@ async fn main() -> eyre::Result<()> {
let cli = Command::parse();
match cli.commands {
match &cli.commands {
Commands::Local { command } => match command {
LocalCommands::Build {
profile: _,
bin_name,
} => {
let base_image =
base_rust_image(client.clone(), &cli.global, None, bin_name.clone()).await?;
base_rust_image(client.clone(), &cli.global, &None, &bin_name.clone()).await?;
let prod_image = get_base_debian_image(client.clone(), &cli.global, None).await?;
build::execute(client, &cli.global, base_image, prod_image, bin_name, None).await?;
build::execute(
client,
&cli.global,
&base_image,
&prod_image,
bin_name,
&None,
)
.await?;
}
LocalCommands::Test => {
let base_image =
base_rust_image(client.clone(), &cli.global, None, "cuddle-please".into())
base_rust_image(client.clone(), &cli.global, &None, &"cuddle-please".into())
.await?;
test::execute(client, &cli.global, base_image).await?;
}
@ -106,119 +137,210 @@ async fn main() -> eyre::Result<()> {
image,
bin_name,
} => {
// let containers = vec!["linux/amd64", "linux/arm64"];
let containers = vec!["linux/amd64"];
let stream = futures::stream::iter(containers.into_iter().map(|c| {
let client = Arc::new(
client
.clone()
.pipeline(format!("docker:build:platform:{c}")),
);
let args = cli.global.clone();
let platform = c.to_string();
let bin_name = bin_name.clone();
tokio::spawn(async move {
let base_image = match get_base_debian_image(
build::build_and_deploy(client, &cli.global, bin_name, image, tag).await?;
}
LocalCommands::PleaseRelease => todo!(),
LocalCommands::BuildDocs {} => {
let _image = docs::execute(
client.clone(),
&args.clone(),
Some(platform.clone()),
)
.await
{
Ok(image) => image,
Err(e) => {
eprintln!("failed to get base image: {e}");
return None;
}
};
match base_rust_image(
client.clone(),
&args,
Some(platform.clone()),
bin_name.clone(),
)
.await
{
Ok(container) => {
let build_image = match build::execute(
client,
&args,
container,
base_image,
bin_name,
Some(platform),
)
.await
{
Ok(image) => image,
Err(e) => {
eprintln!("could not build image: {e}");
return None;
}
};
match build_image.id().await {
Ok(id) => return Some(id),
Err(e) => {
eprintln!("could not get id: {e}");
}
}
}
Err(e) => {
eprintln!("could not build container: {e}");
}
}
None
})
}))
.buffer_unordered(16)
.filter_map(|f| async move { f.ok() })
.collect::<Vec<_>>()
.await;
let _container = client
.container()
.publish_opts(
format!("{image}:{tag}"),
ContainerPublishOpts {
platform_variants: stream
.into_iter()
.collect::<Option<Vec<ContainerId>>>(),
},
&cli.global,
&Some("linux/amd64".to_string()),
)
.await?;
}
LocalCommands::PleaseRelease => todo!(),
},
Commands::PullRequest => todo!(),
Commands::Main => todo!(),
Commands::Main {
image,
tag,
bin_name,
} => {
async fn test(client: Arc<dagger_sdk::Query>, cli: &Command, bin_name: &String) {
let args = &cli.global;
let base_image = base_rust_image(client.clone(), args, &None, bin_name)
.await
.unwrap();
test::execute(client.clone(), args, base_image)
.await
.unwrap();
}
async fn build(
client: Arc<dagger_sdk::Query>,
cli: &Command,
bin_name: &String,
image: &String,
tag: &String,
) {
let args = &cli.global;
build::build_and_deploy(client.clone(), args, bin_name, image, tag)
.await
.unwrap();
}
tokio::join!(
test(client.clone(), &cli, bin_name),
build(client.clone(), &cli, bin_name, image, tag)
);
}
Commands::Release => todo!(),
}
Ok(())
}
mod docs {
use std::sync::Arc;
use dagger_sdk::Container;
use crate::GlobalArgs;
pub fn get_docs_src(client: Arc<dagger_sdk::Query>) -> eyre::Result<dagger_sdk::Directory> {
let docs_content = client.host().directory_opts(
".",
dagger_sdk::HostDirectoryOpts {
exclude: None,
include: Some(vec!["mkdocs.yml", "docs/"]),
},
);
Ok(docs_content)
}
pub async fn execute(
client: Arc<dagger_sdk::Query>,
args: &GlobalArgs,
_platform: &Option<String>,
) -> eyre::Result<Container> {
let mkdocs_container = client.container().from(
args.mkdocs_image
.as_ref()
.expect("--mkdocs-image to be set"),
);
let built_mkdocs_container = mkdocs_container
.with_directory("/docs", get_docs_src(client.clone())?.id().await?)
.with_exec(vec!["build"]);
let site_output = built_mkdocs_container.directory("/docs/site").id().await?;
let caddy_file = client.host().directory("templates").file("Caddyfile");
let dep_image = client
.container()
.from(args.caddy_image.as_ref().expect("--caddy-image to be set"))
.with_directory("/usr/share/caddy", site_output)
.with_file("/etc/caddy/Caddyfile", caddy_file.id().await?)
.with_exec(vec!["echo", "caddy"]);
Ok(dep_image)
}
pub async fn publish(
client: Arc<dagger_sdk::Query>,
args: &GlobalArgs,
containers: &Vec<dagger_sdk::Container>,
) -> eyre::Result<()> {
let container_ids =
futures::future::join_all(containers.iter().map(|c| c.id()).collect::<Vec<_>>()).await;
let container_ids = container_ids
.into_iter()
.collect::<eyre::Result<Vec<dagger_sdk::ContainerId>>>()?;
client
.container()
.publish_opts(
format!(
"{}:{}",
args.docs_image.as_ref().expect("--docs-image to be set"),
args.docs_image_tag
.as_ref()
.expect("--docs-image-tag to be set")
),
dagger_sdk::ContainerPublishOpts {
platform_variants: Some(container_ids),
},
)
.await?;
Ok(())
}
}
mod build {
use std::sync::Arc;
use dagger_sdk::Container;
use crate::GlobalArgs;
use crate::{base_rust_image, get_base_debian_image, GlobalArgs};
pub async fn build_and_deploy(
client: Arc<dagger_sdk::Query>,
args: &GlobalArgs,
bin_name: &String,
image: &String,
tag: &String,
) -> eyre::Result<()> {
// let containers = vec!["linux/amd64", "linux/arm64"];
let base_image = get_base_debian_image(
client.clone(),
&args.clone(),
Some("linux/amd64".to_string()),
)
.await?;
let container = base_rust_image(
client.clone(),
args,
&Some("linux/amd64".to_string()),
&bin_name.clone(),
)
.await?;
let build_image = execute(
client.clone(),
args,
&container,
&base_image,
bin_name,
&Some("linux/amd64".to_string()),
)
.await?;
let build_id = build_image.id().await?;
let _container = client
.clone()
.container()
.publish_opts(
format!("{image}:{tag}"),
dagger_sdk::ContainerPublishOpts {
platform_variants: Some(vec![build_id]),
},
)
.await?;
Ok(())
}
pub async fn execute(
_client: Arc<dagger_sdk::Query>,
_args: &GlobalArgs,
container: dagger_sdk::Container,
base_image: dagger_sdk::Container,
bin_name: String,
platform: Option<String>,
container: &dagger_sdk::Container,
base_image: &dagger_sdk::Container,
bin_name: &String,
platform: &Option<String>,
) -> eyre::Result<Container> {
let rust_target = match platform.unwrap_or("linux/amd64".to_string()).as_str() {
let rust_target = match platform
.clone()
.unwrap_or("linux/amd64".to_string())
.as_str()
{
"linux/amd64" => "x86_64-unknown-linux-gnu",
"linux/arm64" => "aarch64-unknown-linux-gnu",
_ => eyre::bail!("architecture not supported"),
@ -230,7 +352,7 @@ mod build {
rust_target,
"--release",
"-p",
&bin_name,
bin_name,
]);
let final_image = base_image
@ -241,7 +363,7 @@ mod build {
.id()
.await?,
)
.with_exec(vec![&bin_name, "--help"]);
.with_exec(vec![bin_name, "--help"]);
let output = final_image.stdout().await?;
println!("{output}");
@ -336,12 +458,12 @@ pub async fn get_rust_dep_src(
.build()?,
);
return Ok(directory);
Ok(directory)
}
pub async fn get_rust_skeleton_files(
client: Arc<dagger_sdk::Query>,
args: &GlobalArgs,
_args: &GlobalArgs,
) -> eyre::Result<(dagger_sdk::Directory, Vec<String>)> {
let mut rust_crates = vec![PathBuf::from("ci")];
let mut dirs = tokio::fs::read_dir("crates").await?;
@ -383,7 +505,7 @@ pub async fn get_rust_skeleton_files(
if let Some(file_name) = rust_crate.file_name() {
crate_names.push(file_name.to_str().unwrap().to_string());
}
directory = create_skeleton_files(directory, &rust_crate)?;
directory = create_skeleton_files(directory, rust_crate)?;
}
Ok((directory, crate_names))
@ -392,8 +514,8 @@ pub async fn get_rust_skeleton_files(
pub async fn base_rust_image(
client: Arc<dagger_sdk::Query>,
args: &GlobalArgs,
platform: Option<String>,
bin_name: String,
platform: &Option<String>,
bin_name: &String,
) -> eyre::Result<dagger_sdk::Container> {
let dep_src = get_rust_dep_src(client.clone(), args).await?;
let (skeleton_files, crates) = get_rust_skeleton_files(client.clone(), args).await?;
@ -401,7 +523,11 @@ pub async fn base_rust_image(
let client = client.pipeline("rust_base_image");
let rust_target = match platform.unwrap_or("linux/amd64".to_string()).as_str() {
let rust_target = match platform
.clone()
.unwrap_or("linux/amd64".to_string())
.as_str()
{
"linux/amd64" => "x86_64-unknown-linux-gnu",
"linux/arm64" => "aarch64-unknown-linux-gnu",
_ => eyre::bail!("architecture not supported"),
@ -428,14 +554,14 @@ pub async fn base_rust_image(
rust_target,
"--release",
"-p",
&bin_name,
bin_name,
])
.with_mounted_cache("/mnt/src/target/", target_cache.id().await?);
let exclude = crates
.iter()
.filter(|c| **c != "ci")
.map(|c| format!("**/*{}*", c.replace("-", "_")))
.map(|c| format!("**/*{}*", c.replace('-', "_")))
.collect::<Vec<_>>();
let exclude = exclude.iter().map(|c| c.as_str()).collect();

View File

@ -139,7 +139,7 @@ impl Command {
tracing_subscriber::fmt().with_env_filter(env_filter).init();
}
return Ok((config, git_client, gitea_client));
Ok((config, git_client, gitea_client))
}
fn build_config(&self, current_dir: Option<&Path>) -> Result<PleaseConfig, anyhow::Error> {

View File

@ -165,6 +165,12 @@ fn setup_git(path: &Path) -> anyhow::Result<()> {
let stderr = std::str::from_utf8(&output.stderr)?;
tracing::debug!(stdout = stdout, stderr = stderr, "git init");
exec_git(path, &["checkout", "-b", "main"])?;
exec_git(path, &["config", "user.name", "test"])?;
exec_git(path, &["config", "user.email", "test@test.com"])?;
exec_git(path, &["config", "init.defaultBranch", "main"])?;
Ok(())
}

View File

@ -5,7 +5,9 @@ base: "git@git.front.kjuulh.io:kjuulh/cuddle-rust-cli-plan.git"
vars:
service: "cuddle-please"
registry: kasperhermansen
mkdocs_image: "squidfunk/mkdocs-material:9.1"
caddy_image: "caddy:2.7"
please:
project:
@ -24,4 +26,12 @@ scripts:
type: shell
"local:docker":
type: shell
"local:docker:docs":
type: shell
"ci:main":
type: shell
"ci:pr":
type: shell
"ci:release":
type: shell

10
scripts/ci:main.sh Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -e
cargo run -p ci -- main \
--mkdocs-image "$MKDOCS_IMAGE" \
--caddy-image "$CADDY_IMAGE" \
--image "$REGISTRY/$SERVICE" \
--tag "main-$(date +%s)" \
--bin-name "$SERVICE"

5
scripts/local:docker:docs.sh Executable file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -e
cargo run -p ci -- local build-docs --mkdocs-image $MKDOCS_IMAGE --caddy-image $CADDY_IMAGE

View File

@ -0,0 +1 @@

8
templates/Caddyfile Normal file
View File

@ -0,0 +1,8 @@
{
debug
}
http://blog.kasperhermansen.com {
root * /usr/share/caddy
file_server
}