feat: with rust build and test

Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
Kasper Juul Hermansen 2023-08-12 19:50:30 +02:00 committed by Kasper Juul Hermansen
parent a17e527b91
commit e1428a8fbb
26 changed files with 855 additions and 2 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
/target
.env
.cuddle/
target/

40
Cargo.lock generated
View File

@ -349,6 +349,16 @@ dependencies = [
"eyre",
]
[[package]]
name = "dagger-rust"
version = "0.1.0"
dependencies = [
"async-trait",
"dagger-sdk",
"eyre",
"tokio",
]
[[package]]
name = "dagger-sdk"
version = "0.2.22"
@ -1224,6 +1234,36 @@ dependencies = [
"winapi",
]
[[package]]
name = "rust-build"
version = "0.1.0"
dependencies = [
"dagger-rust",
"dagger-sdk",
"eyre",
"tokio",
]
[[package]]
name = "rust-src"
version = "0.1.0"
dependencies = [
"dagger-rust",
"dagger-sdk",
"eyre",
"tokio",
]
[[package]]
name = "rust-test"
version = "0.1.0"
dependencies = [
"dagger-rust",
"dagger-sdk",
"eyre",
"tokio",
]
[[package]]
name = "rustc-demangle"
version = "0.1.23"

View File

@ -10,9 +10,11 @@ resolver = "2"
cuddle-components = {path = "crates/cuddle-components"}
dagger-components = {path = "crates/dagger-components"}
dagger-cuddle-please = {path = "crates/dagger-cuddle-please"}
dagger-rust = {path = "crates/dagger-rust"}
ci = {path = "ci"}
dagger-sdk = "0.2.22"
eyre = "0.6.8"
tokio = "1.31.0"
dotenv = "*"
async-trait = "*"

View File

@ -186,7 +186,7 @@ pub fn get_src(
.display()
.to_string(),
dagger_sdk::HostDirectoryOptsBuilder::default()
.exclude(vec!["node_modules/", ".git/", "target/"])
.exclude(vec!["node_modules/", ".git/", "target/", ".cuddle/"])
.build()?,
);

View File

@ -8,4 +8,4 @@ edition = "2021"
[dependencies]
dagger-sdk.workspace = true
eyre.workspace = true
async-trait = "*"
async-trait.workspace = true

View File

@ -0,0 +1,12 @@
[package]
name = "dagger-rust"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
dagger-sdk.workspace = true
eyre.workspace = true
async-trait.workspace = true
tokio.workspace = true

View File

@ -0,0 +1 @@

View File

@ -0,0 +1,372 @@
use std::{path::PathBuf, sync::Arc};
use crate::source::RustSource;
#[allow(dead_code)]
pub struct RustBuild {
client: Arc<dagger_sdk::Query>,
registry: Option<String>,
}
impl RustBuild {
pub fn new(client: Arc<dagger_sdk::Query>) -> Self {
Self {
client,
registry: None,
}
}
pub async fn build(
&self,
source_path: Option<impl Into<PathBuf>>,
rust_version: impl AsRef<RustVersion>,
target: impl AsRef<BuildTarget>,
profile: impl AsRef<BuildProfile>,
crates: &[&str],
extra_deps: &[&str],
) -> eyre::Result<dagger_sdk::Container> {
let rust_version = rust_version.as_ref();
let target = target.as_ref();
let profile = profile.as_ref();
let source_path = source_path.map(|s| s.into());
let source = source_path.clone().unwrap_or(PathBuf::from("."));
let rust_source = RustSource::new(self.client.clone());
let (src, dep_src) = rust_source
.get_rust_src(source_path, crates.to_vec())
.await?;
let mut deps = vec!["apt", "install", "-y"];
deps.extend(extra_deps);
let rust_build_image = self
.client
.container()
.from(rust_version.to_string())
.with_exec(vec!["rustup", "target", "add", &target.to_string()])
.with_exec(vec!["apt", "update"])
.with_exec(deps);
let target_cache = self.client.cache_volume(format!(
"rust_target_{}_{}",
profile.to_string(),
target.to_string()
));
let target_str = target.to_string();
let mut build_options = vec!["cargo", "build", "--target", &target_str, "--workspace"];
let entries = dep_src
.directory("crates/example_bin/src")
.entries()
.await?;
for entry in entries {
println!("entry: {}", entry);
}
if matches!(profile, BuildProfile::Release) {
build_options.push("--release");
}
let rust_prebuild = rust_build_image
.with_workdir("/mnt/src")
.with_directory("/mnt/src", dep_src.id().await?)
.with_exec(build_options)
.with_mounted_cache("/mnt/src/target/", target_cache.id().await?);
let incremental_dir = rust_source
.get_rust_target_src(&source, rust_prebuild.clone(), crates.to_vec())
.await?;
let rust_with_src = rust_build_image
.with_workdir("/mnt/src")
.with_directory(
"/usr/local/cargo",
rust_prebuild.directory("/usr/local/cargo").id().await?,
)
.with_directory("/mnt/src/target", incremental_dir.id().await?)
.with_directory("/mnt/src/", src.id().await?);
Ok(rust_with_src)
}
pub async fn build_release(
&self,
source_path: Option<impl Into<PathBuf>>,
rust_version: impl AsRef<RustVersion>,
crates: &[&str],
extra_deps: &[&str],
images: impl IntoIterator<Item = SlimImage>,
bin_name: &str,
) -> eyre::Result<Vec<dagger_sdk::Container>> {
let images = images.into_iter().collect::<Vec<_>>();
let source_path = source_path.map(|s| s.into());
let mut containers = Vec::new();
for container_image in images {
let container = match &container_image {
SlimImage::Debian { image, deps, .. } => {
let target = BuildTarget::from_target(&container_image);
let build_container = self
.build(
source_path.clone(),
&rust_version,
BuildTarget::from_target(&container_image),
BuildProfile::Release,
crates,
extra_deps,
)
.await?;
let bin = build_container
.with_exec(vec![
"cargo",
"build",
"--target",
&target.to_string(),
"--release",
"-p",
bin_name,
])
.file(format!(
"target/{}/release/{}",
target.to_string(),
bin_name
));
self.build_debian_image(
bin,
image,
BuildTarget::from_target(&container_image),
deps.iter()
.map(|d| d.as_str())
.collect::<Vec<&str>>()
.as_slice(),
bin_name,
)
.await?
}
SlimImage::Alpine { image, deps, .. } => {
let target = BuildTarget::from_target(&container_image);
let build_container = self
.build(
source_path.clone(),
&rust_version,
BuildTarget::from_target(&container_image),
BuildProfile::Release,
crates,
extra_deps,
)
.await?;
let bin = build_container
.with_exec(vec![
"cargo",
"build",
"--target",
&target.to_string(),
"--release",
"-p",
bin_name,
])
.file(format!(
"target/{}/release/{}",
target.to_string(),
bin_name
));
self.build_alpine_image(
bin,
image,
BuildTarget::from_target(&container_image),
deps.iter()
.map(|d| d.as_str())
.collect::<Vec<&str>>()
.as_slice(),
bin_name,
)
.await?
}
};
containers.push(container);
}
Ok(containers)
}
async fn build_debian_image(
&self,
bin: dagger_sdk::File,
image: &str,
target: BuildTarget,
production_deps: &[&str],
bin_name: &str,
) -> eyre::Result<dagger_sdk::Container> {
let base_debian = self
.client
.container_opts(dagger_sdk::QueryContainerOpts {
id: None,
platform: Some(target.into_platform()),
})
.from(image);
let mut packages = vec!["apt", "install", "-y"];
packages.extend_from_slice(production_deps);
let base_debian = base_debian
.with_exec(vec!["apt", "update"])
.with_exec(packages);
let final_image = base_debian
.with_file(format!("/usr/local/bin/{}", bin_name), bin.id().await?)
.with_exec(vec![bin_name, "--help"]);
final_image.exit_code().await?;
Ok(final_image)
}
async fn build_alpine_image(
&self,
bin: dagger_sdk::File,
image: &str,
target: BuildTarget,
production_deps: &[&str],
bin_name: &str,
) -> eyre::Result<dagger_sdk::Container> {
let base_debian = self
.client
.container_opts(dagger_sdk::QueryContainerOpts {
id: None,
platform: Some(target.into_platform()),
})
.from(image);
let mut packages = vec!["apk", "add"];
packages.extend_from_slice(production_deps);
let base_debian = base_debian.with_exec(packages);
let final_image =
base_debian.with_file(format!("/usr/local/bin/{}", bin_name), bin.id().await?);
Ok(final_image)
}
}
pub enum RustVersion {
Nightly,
Stable(String),
}
impl AsRef<RustVersion> for RustVersion {
fn as_ref(&self) -> &RustVersion {
&self
}
}
impl ToString for RustVersion {
fn to_string(&self) -> String {
match self {
RustVersion::Nightly => "rustlang/rust:nightly".to_string(),
RustVersion::Stable(version) => format!("rust:{}", version),
}
}
}
pub enum BuildTarget {
LinuxAmd64,
LinuxArm64,
LinuxAmd64Musl,
LinuxArm64Musl,
MacOSAmd64,
MacOSArm64,
}
impl BuildTarget {
pub fn from_target(image: &SlimImage) -> Self {
match image {
SlimImage::Debian { architecture, .. } => match architecture {
BuildArchitecture::Amd64 => Self::LinuxAmd64,
BuildArchitecture::Arm64 => Self::LinuxArm64,
},
SlimImage::Alpine { architecture, .. } => match architecture {
BuildArchitecture::Amd64 => Self::LinuxAmd64Musl,
BuildArchitecture::Arm64 => Self::LinuxArm64Musl,
},
}
}
fn into_platform(&self) -> dagger_sdk::Platform {
let platform = match self {
BuildTarget::LinuxAmd64 => "linux/amd64",
BuildTarget::LinuxArm64 => "linux/arm64",
BuildTarget::LinuxAmd64Musl => "linux/amd64",
BuildTarget::LinuxArm64Musl => "linux/arm64",
BuildTarget::MacOSAmd64 => "darwin/amd64",
BuildTarget::MacOSArm64 => "darwin/arm64",
};
dagger_sdk::Platform(platform.into())
}
}
impl AsRef<BuildTarget> for BuildTarget {
fn as_ref(&self) -> &BuildTarget {
&self
}
}
impl ToString for BuildTarget {
fn to_string(&self) -> String {
let target = match self {
BuildTarget::LinuxAmd64 => "x86_64-unknown-linux-gnu",
BuildTarget::LinuxArm64 => "aarch64-unknown-linux-gnu",
BuildTarget::LinuxAmd64Musl => "x86_64-unknown-linux-musl",
BuildTarget::LinuxArm64Musl => "aarch64-unknown-linux-musl",
BuildTarget::MacOSAmd64 => "x86_64-apple-darwin",
BuildTarget::MacOSArm64 => "aarch64-apple-darwin",
};
target.into()
}
}
pub enum BuildProfile {
Debug,
Release,
}
impl AsRef<BuildProfile> for BuildProfile {
fn as_ref(&self) -> &BuildProfile {
&self
}
}
impl ToString for BuildProfile {
fn to_string(&self) -> String {
let profile = match self {
BuildProfile::Debug => "debug",
BuildProfile::Release => "release",
};
profile.into()
}
}
pub enum SlimImage {
Debian {
image: String,
deps: Vec<String>,
architecture: BuildArchitecture,
},
Alpine {
image: String,
deps: Vec<String>,
architecture: BuildArchitecture,
},
}
pub enum BuildArchitecture {
Amd64,
Arm64,
}

View File

@ -0,0 +1,3 @@
pub mod build;
pub mod source;
pub mod test;

View File

@ -0,0 +1,193 @@
use std::{
path::{Path, PathBuf},
sync::Arc,
};
use eyre::Context;
pub struct RustSource {
client: Arc<dagger_sdk::Query>,
exclude: Vec<String>,
}
impl RustSource {
pub fn new(client: Arc<dagger_sdk::Query>) -> Self {
Self {
client,
exclude: vec!["node_modules/", ".git/", "target/", ".cuddle/"]
.into_iter()
.map(|s| s.to_string())
.collect(),
}
}
pub fn with_exclude(
&mut self,
exclude: impl IntoIterator<Item = impl Into<String>>,
) -> &mut Self {
self.exclude = exclude.into_iter().map(|s| s.into()).collect();
self
}
pub fn append_exclude(
&mut self,
exclude: impl IntoIterator<Item = impl Into<String>>,
) -> &mut Self {
self.exclude
.append(&mut exclude.into_iter().map(|s| s.into()).collect::<Vec<_>>());
self
}
pub async fn get_rust_src<T, I>(
&self,
source: Option<T>,
crate_paths: I,
) -> eyre::Result<(dagger_sdk::Directory, dagger_sdk::Directory)>
where
T: Into<PathBuf>,
T: Clone,
I: IntoIterator,
I::Item: Into<String>,
{
let source_path = match source.clone() {
Some(s) => s.into(),
None => PathBuf::from("."),
};
let (skeleton_files, _crates) = self
.get_rust_skeleton_files(&source_path, crate_paths)
.await?;
let src = self.get_src(source.clone()).await?;
let rust_src = self.get_rust_dep_src(source).await?;
let rust_src = rust_src.with_directory(".", skeleton_files.id().await?);
Ok((src, rust_src))
}
pub async fn get_src(
&self,
source: Option<impl Into<PathBuf>>,
) -> eyre::Result<dagger_sdk::Directory> {
let source = source.map(|s| s.into()).unwrap_or(PathBuf::from("."));
let directory = self.client.host().directory_opts(
source.display().to_string(),
dagger_sdk::HostDirectoryOptsBuilder::default()
.exclude(self.exclude.iter().map(|s| s.as_str()).collect::<Vec<_>>())
.build()?,
);
Ok(directory)
}
pub async fn get_rust_dep_src(
&self,
source: Option<impl Into<PathBuf>>,
) -> eyre::Result<dagger_sdk::Directory> {
let source = source.map(|s| s.into()).unwrap_or(PathBuf::from("."));
let directory = self.client.host().directory_opts(
source.display().to_string(),
dagger_sdk::HostDirectoryOptsBuilder::default()
.include(vec!["**/Cargo.toml", "**/Cargo.lock"])
.build()?,
);
Ok(directory)
}
pub async fn get_rust_target_src(
&self,
source_path: &Path,
container: dagger_sdk::Container,
crate_paths: impl IntoIterator<Item = impl Into<String>>,
) -> eyre::Result<dagger_sdk::Directory> {
let (_skeleton_files, crates) = self
.get_rust_skeleton_files(source_path, crate_paths)
.await?;
let exclude = crates
.iter()
.map(|c| format!("**/*{}*", c.replace('-', "_")))
.collect::<Vec<_>>();
let exclude = exclude.iter().map(|c| c.as_str()).collect();
let incremental_dir = self.client.directory().with_directory_opts(
".",
container.directory("target").id().await?,
dagger_sdk::DirectoryWithDirectoryOpts {
exclude: Some(exclude),
include: None,
},
);
return Ok(incremental_dir);
}
pub async fn get_rust_skeleton_files(
&self,
source_path: &Path,
crate_paths: impl IntoIterator<Item = impl Into<String>>,
) -> eyre::Result<(dagger_sdk::Directory, Vec<String>)> {
let paths = crate_paths
.into_iter()
.map(|s| s.into())
.collect::<Vec<String>>();
let mut crates = Vec::new();
for path in paths {
if path.ends_with("/*") {
let mut dirs = tokio::fs::read_dir(source_path.join(path.trim_end_matches("/*")))
.await
.context(format!("failed to find path: {}", path.clone()))?;
while let Some(entry) = dirs.next_entry().await? {
if entry.metadata().await?.is_dir() {
crates.push(entry.path());
}
}
} else {
crates.push(PathBuf::from(path));
}
}
fn create_skeleton_files(
directory: dagger_sdk::Directory,
path: &Path,
) -> eyre::Result<dagger_sdk::Directory> {
let main_content = r#"
#[allow(dead_code)]
fn main() { panic!("should never be executed"); }"#;
let lib_content = r#"
#[allow(dead_code)]
fn some() { panic!("should never be executed"); }"#;
let directory = directory.with_new_file(
path.join("src").join("main.rs").display().to_string(),
main_content,
);
let directory = directory.with_new_file(
path.join("src").join("lib.rs").display().to_string(),
lib_content,
);
Ok(directory)
}
let mut directory = self.client.directory();
let mut crate_names = Vec::new();
for rust_crate in crates.iter() {
if let Some(file_name) = rust_crate.file_name() {
crate_names.push(file_name.to_str().unwrap().to_string());
}
directory = create_skeleton_files(directory, rust_crate.strip_prefix(source_path)?)?;
}
Ok((directory, crate_names))
}
}

View File

@ -0,0 +1,86 @@
use std::{path::PathBuf, sync::Arc};
use crate::{build::RustVersion, source::RustSource};
pub struct RustTest {
client: Arc<dagger_sdk::Query>,
registry: Option<String>,
}
impl RustTest {
pub fn new(client: Arc<dagger_sdk::Query>) -> Self {
Self {
client,
registry: None,
}
}
pub async fn test(
&self,
source_path: Option<impl Into<PathBuf>>,
rust_version: impl AsRef<RustVersion>,
crates: &[&str],
extra_deps: &[&str],
) -> eyre::Result<()> {
let rust_version = rust_version.as_ref();
let source_path = source_path.map(|s| s.into());
let source = source_path.clone().unwrap_or(PathBuf::from("."));
let rust_source = RustSource::new(self.client.clone());
let (src, dep_src) = rust_source
.get_rust_src(source_path, crates.to_vec())
.await?;
let mut deps = vec!["apt", "install", "-y"];
deps.extend(extra_deps);
let rust_build_image = self
.client
.container()
.from(rust_version.to_string())
.with_exec(vec!["apt", "update"])
.with_exec(deps);
let target_cache = self.client.cache_volume(format!("rust_target_test",));
let build_options = vec!["cargo", "build", "--workspace"];
let entries = dep_src
.directory("crates/example_bin/src")
.entries()
.await?;
for entry in entries {
println!("entry: {}", entry);
}
let rust_prebuild = rust_build_image
.with_workdir("/mnt/src")
.with_directory("/mnt/src", dep_src.id().await?)
.with_exec(build_options)
.with_mounted_cache("/mnt/src/target/", target_cache.id().await?);
let incremental_dir = rust_source
.get_rust_target_src(&source, rust_prebuild.clone(), crates.to_vec())
.await?;
let rust_with_src = rust_build_image
.with_workdir("/mnt/src")
.with_directory(
"/usr/local/cargo",
rust_prebuild.directory("/usr/local/cargo").id().await?,
)
.with_directory("/mnt/src/target", incremental_dir.id().await?)
.with_directory("/mnt/src/", src.id().await?);
let test = rust_with_src.with_exec(vec!["cargo", "test"]);
let stdout = test.stdout().await?;
let stderr = test.stderr().await?;
println!("stdout: {}, stderr: {}", stdout, stderr);
if 0 != test.exit_code().await? {
eyre::bail!("failed rust:test");
}
Ok(())
}
}

View File

@ -0,0 +1,13 @@
[package]
name = "rust-build"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
dagger-rust.workspace = true
eyre.workspace = true
dagger-sdk.workspace = true
tokio.workspace = true

View File

@ -0,0 +1,29 @@
use dagger_rust::build::{RustVersion, SlimImage};
#[tokio::main]
pub async fn main() -> eyre::Result<()> {
let client = dagger_sdk::connect().await?;
let rust_build = dagger_rust::build::RustBuild::new(client.clone());
let containers = rust_build
.build_release(
Some("testdata"),
RustVersion::Nightly,
&["crates/*"],
&["openssl"],
vec![SlimImage::Debian {
image: "debian:bookworm".into(),
deps: vec!["openssl".into()],
architecture: dagger_rust::build::BuildArchitecture::Amd64,
}],
"example_bin",
)
.await?;
for container in containers {
container.exit_code().await?;
}
Ok(())
}

View File

@ -0,0 +1,3 @@
[workspace]
members = ["crates/*"]
resolver = "2"

View File

@ -0,0 +1 @@
/target

View File

@ -0,0 +1,8 @@
[package]
name = "example_bin"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]

View File

@ -0,0 +1,3 @@
fn main() {
println!("Hello, world!");
}

View File

@ -0,0 +1,13 @@
[package]
name = "rust-src"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
dagger-rust.workspace = true
eyre.workspace = true
dagger-sdk.workspace = true
tokio.workspace = true

View File

@ -0,0 +1,14 @@
use std::path::PathBuf;
#[tokio::main]
pub async fn main() -> eyre::Result<()> {
let client = dagger_sdk::connect().await?;
let crates = ["some-crate"];
let dag = dagger_rust::source::RustSource::new(client.clone());
let (_src, _rust_src) = dag.get_rust_src(None::<PathBuf>, crates).await?;
let _full_src = dag.get_rust_target_src(client.container(), crates).await?;
Ok(())
}

View File

@ -0,0 +1,13 @@
[package]
name = "rust-test"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
dagger-rust.workspace = true
eyre.workspace = true
dagger-sdk.workspace = true
tokio.workspace = true

View File

@ -0,0 +1,16 @@
use dagger_rust::{build::RustVersion, test::RustTest};
#[tokio::main]
pub async fn main() -> eyre::Result<()> {
let client = dagger_sdk::connect().await?;
RustTest::new(client.clone())
.test(
Some("testdata"),
RustVersion::Nightly,
&["crates/*"],
&["openssl"],
)
.await?;
Ok(())
}

7
examples/rust-test/testdata/Cargo.lock generated vendored Normal file
View File

@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "example_bin"
version = "0.1.0"

View File

@ -0,0 +1,3 @@
[workspace]
members = ["crates/*"]
resolver = "2"

View File

@ -0,0 +1 @@
/target

View File

@ -0,0 +1,8 @@
[package]
name = "example_bin"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]

View File

@ -0,0 +1,11 @@
fn main() {
println!("Hello, world!");
}
#[cfg(test)]
mod tests {
#[test]
fn test_main() {
assert_eq!(1, 1)
}
}