From f04b0a2e54062ca59f55f35fc4a09312ab7e920e Mon Sep 17 00:00:00 2001 From: kjuulh Date: Mon, 20 May 2024 20:45:26 +0200 Subject: [PATCH] feat: add basic templates Signed-off-by: kjuulh --- .gitignore | 2 + Cargo.lock | 90 ++++++++++++ README.md | 7 + crates/cuddle-clusters/Cargo.toml | 4 + crates/cuddle-clusters/src/process.rs | 134 +++++++++++++++++- .../main.ncl => expected/dev/test.yaml} | 0 .../templates/clusters/test.yaml.jinja2 | 6 + crates/cuddle-clusters/tests/common.rs | 99 ++++++++++++- .../tests/raw_files/cuddle.yaml | 2 + .../raw_files/expected/dev/some-file.txt | 2 + .../raw_files/expected/dev/some-file.yaml | 2 + .../templates/clusters/raw/some-file.txt | 2 + .../templates/clusters/raw/some-file.yaml | 2 + crates/cuddle-clusters/tests/tests.rs | 9 ++ 14 files changed, 357 insertions(+), 4 deletions(-) rename crates/cuddle-clusters/tests/can_run_for_env/{templates/clusters/main.ncl => expected/dev/test.yaml} (100%) create mode 100644 crates/cuddle-clusters/tests/can_run_for_env/templates/clusters/test.yaml.jinja2 create mode 100644 crates/cuddle-clusters/tests/raw_files/cuddle.yaml create mode 100644 crates/cuddle-clusters/tests/raw_files/expected/dev/some-file.txt create mode 100644 crates/cuddle-clusters/tests/raw_files/expected/dev/some-file.yaml create mode 100644 crates/cuddle-clusters/tests/raw_files/templates/clusters/raw/some-file.txt create mode 100644 crates/cuddle-clusters/tests/raw_files/templates/clusters/raw/some-file.yaml diff --git a/.gitignore b/.gitignore index 9c4c004..c1e20cf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ target/ .cuddle/ + +**/actual/ diff --git a/Cargo.lock b/Cargo.lock index 01a1837..1d05f13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,6 +232,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata 0.1.10", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -302,6 +313,18 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -365,15 +388,19 @@ dependencies = [ "axum", "clap", "dotenv", + "minijinja", "serde", "serde_yaml", + "similar-asserts", "sqlx", "tokio", + "tokio-stream", "tower-http", "tracing", "tracing-subscriber", "tracing-test", "uuid", + "walkdir", ] [[package]] @@ -429,6 +456,12 @@ dependencies = [ "serde", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "equivalent" version = "1.0.1" @@ -885,6 +918,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minijinja" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7165d0e94806d52ad5295e4b54a95176d831814840bc067298ca647e1c956338" +dependencies = [ + "serde", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1336,6 +1378,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1468,6 +1519,26 @@ dependencies = [ "rand_core", ] +[[package]] +name = "similar" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640" +dependencies = [ + "bstr", + "unicode-segmentation", +] + +[[package]] +name = "similar-asserts" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e041bb827d1bfca18f213411d51b665309f1afb37a04a5d1464530e13779fc0f" +dependencies = [ + "console", + "similar", +] + [[package]] name = "slab" version = "0.4.9" @@ -2151,6 +2222,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2195,6 +2276,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/README.md b/README.md index 38ddce8..0d4a328 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,10 @@ some-service/: ``` For each env chosen, run code with own set of variables + +A function is run for each environment: + +- Input components which requires a set of variables +- Templates in templates.clusters.* +- All raw files in templates.clusters.raw.* + diff --git a/crates/cuddle-clusters/Cargo.toml b/crates/cuddle-clusters/Cargo.toml index 0e9bb5d..cfb6bc2 100644 --- a/crates/cuddle-clusters/Cargo.toml +++ b/crates/cuddle-clusters/Cargo.toml @@ -24,10 +24,14 @@ sqlx = { version = "0.7.3", features = [ uuid = { version = "1.7.0", features = ["v4"] } tower-http = { version = "0.5.2", features = ["cors", "trace"] } serde_yaml = "0.9.34" +tokio-stream = "0.1.15" +walkdir = "2.5.0" +minijinja = "2.0.1" [[test]] name = "integration" path = "tests/tests.rs" [dev-dependencies] +similar-asserts = "1.5.0" tracing-test = "0.2.4" diff --git a/crates/cuddle-clusters/src/process.rs b/crates/cuddle-clusters/src/process.rs index 4ed879d..76ed261 100644 --- a/crates/cuddle-clusters/src/process.rs +++ b/crates/cuddle-clusters/src/process.rs @@ -1,9 +1,12 @@ use std::{ collections::HashMap, + ops::Deref, path::{Path, PathBuf}, }; use anyhow::Context; +use tokio::io::AsyncWriteExt; +use tokio_stream::{wrappers::ReadDirStream, StreamExt}; pub async fn process() -> anyhow::Result<()> { process_opts(ProcessOpts::default()).await @@ -11,12 +14,17 @@ pub async fn process() -> anyhow::Result<()> { pub struct ProcessOpts { pub path: PathBuf, + pub output: PathBuf, } impl Default for ProcessOpts { fn default() -> Self { Self { path: std::env::current_dir().expect("to be able to get current dir"), + output: std::env::current_dir() + .expect("to be able to get current dir") + .join("cuddle-clusters") + .join("k8s"), } } } @@ -36,7 +44,13 @@ pub async fn process_opts(opts: ProcessOpts) -> anyhow::Result<()> { let clusters = read_cuddle_section(&cuddle_path).await?; tracing::debug!("found clusters: {:?}", clusters); - load_template_files(&template).await?; + let template_files = load_template_files(&template).await?; + tracing::debug!("found files: {:?}", template_files); + + tokio::fs::remove_dir_all(&opts.output).await?; + tokio::fs::create_dir_all(&opts.output).await?; + + process_templates(&clusters, &template_files, &opts.output).await?; Ok(()) } @@ -44,8 +58,18 @@ pub async fn process_opts(opts: ProcessOpts) -> anyhow::Result<()> { #[derive(serde::Deserialize, Default, Debug, Clone)] struct CuddleClusters(HashMap); +impl Deref for CuddleClusters { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + async fn read_cuddle_section(path: &Path) -> anyhow::Result { - let cuddle_file = tokio::fs::read(path).await?; + let cuddle_file = tokio::fs::read(path) + .await + .context(format!("failed to read: {}", path.display()))?; let value: serde_yaml::Value = serde_yaml::from_slice(&cuddle_file)?; @@ -63,6 +87,110 @@ async fn read_cuddle_section(path: &Path) -> anyhow::Result { Ok(cuddle_clusters) } -async fn load_template_files(path: &Path) -> anyhow::Result<()> { +#[derive(Debug, Clone)] +struct TemplateFiles { + templates: Vec, + raw: Vec, +} + +async fn load_template_files(path: &Path) -> anyhow::Result { + Ok(TemplateFiles { + templates: read_dir(path) + .await? + .into_iter() + .filter(|i| !i.ends_with(".jinja2")) + .collect(), + raw: read_dir(&path.join("raw")).await.unwrap_or_default(), + }) +} + +async fn read_dir(path: &Path) -> anyhow::Result> { + let template_dir = tokio::fs::read_dir(path).await?; + let mut template_dir_stream = ReadDirStream::new(template_dir); + + let mut paths = Vec::new(); + while let Some(entry) = template_dir_stream.next().await { + let entry = entry?; + + if entry.metadata().await?.is_file() { + paths.push(entry.path()); + } + } + + Ok(paths) +} + +async fn process_templates( + clusters: &CuddleClusters, + template_files: &TemplateFiles, + dest: &Path, +) -> anyhow::Result<()> { + for (cluster_name, _value) in clusters.iter() { + process_cluster(cluster_name, template_files, &dest.join(cluster_name)).await?; + } + + Ok(()) +} + +async fn process_cluster( + cluster_name: &str, + template_files: &TemplateFiles, + dest: &Path, +) -> anyhow::Result<()> { + for template_file in &template_files.templates { + process_template_file(cluster_name, template_file, dest).await?; + } + + for raw_file in &template_files.raw { + process_raw_file(cluster_name, raw_file, dest).await?; + } + + Ok(()) +} + +async fn process_template_file( + cluster_name: &str, + template_file: &PathBuf, + dest: &Path, +) -> anyhow::Result<()> { + // TODO: use mini jinja + let file = tokio::fs::read_to_string(template_file) + .await + .context(format!("failed to find file: {}", template_file.display()))?; + + if !dest.exists() { + tokio::fs::create_dir_all(dest).await?; + } + + let file_name = template_file + .file_stem() + .ok_or(anyhow::anyhow!("file didn't have a jinja2 format"))?; + + let mut dest_file = tokio::fs::File::create(dest.join(file_name)).await?; + dest_file.write_all(file.as_bytes()).await?; + + Ok(()) +} + +async fn process_raw_file( + _cluster_name: &str, + raw_file: &PathBuf, + dest: &Path, +) -> anyhow::Result<()> { + let file = tokio::fs::read_to_string(raw_file) + .await + .context(format!("failed to find file: {}", raw_file.display()))?; + + if !dest.exists() { + tokio::fs::create_dir_all(dest).await?; + } + + let file_name = raw_file + .file_name() + .ok_or(anyhow::anyhow!("file didn't have a file name"))?; + + let mut dest_file = tokio::fs::File::create(dest.join(file_name)).await?; + dest_file.write_all(file.as_bytes()).await?; + Ok(()) } diff --git a/crates/cuddle-clusters/tests/can_run_for_env/templates/clusters/main.ncl b/crates/cuddle-clusters/tests/can_run_for_env/expected/dev/test.yaml similarity index 100% rename from crates/cuddle-clusters/tests/can_run_for_env/templates/clusters/main.ncl rename to crates/cuddle-clusters/tests/can_run_for_env/expected/dev/test.yaml diff --git a/crates/cuddle-clusters/tests/can_run_for_env/templates/clusters/test.yaml.jinja2 b/crates/cuddle-clusters/tests/can_run_for_env/templates/clusters/test.yaml.jinja2 new file mode 100644 index 0000000..5418072 --- /dev/null +++ b/crates/cuddle-clusters/tests/can_run_for_env/templates/clusters/test.yaml.jinja2 @@ -0,0 +1,6 @@ +{ + hello = "world", + some = { + thing = "some" + } +} diff --git a/crates/cuddle-clusters/tests/common.rs b/crates/cuddle-clusters/tests/common.rs index f7f02a9..462a1ff 100644 --- a/crates/cuddle-clusters/tests/common.rs +++ b/crates/cuddle-clusters/tests/common.rs @@ -1,4 +1,7 @@ +use std::{cmp::Ordering, path::Path}; + use cuddle_clusters::process::ProcessOpts; +use walkdir::DirEntry; pub(crate) async fn run_test(name: &str) -> anyhow::Result<()> { let _ = tracing_subscriber::fmt::try_init(); @@ -7,10 +10,104 @@ pub(crate) async fn run_test(name: &str) -> anyhow::Result<()> { let current_dir = std::env::current_dir()?; + let test_folder = current_dir.join("tests").join(name); + + let actual = test_folder.join("actual"); + tokio::fs::create_dir_all(&actual).await?; + let expected = test_folder.join("expected"); + tokio::fs::create_dir_all(&expected).await?; + cuddle_clusters::process_opts(ProcessOpts { - path: current_dir.join("tests").join(name), + path: test_folder.clone(), + output: actual.clone(), }) .await?; + if std::env::var("TEST_OVERRIDE") == Ok("true".to_string()) { + cuddle_clusters::process_opts(ProcessOpts { + path: test_folder, + output: expected.clone(), + }) + .await?; + } + + compare(&expected, &actual).await?; + Ok(()) } + +async fn compare(expected: &Path, actual: &Path) -> anyhow::Result<()> { + let mut exp = walk_dir(expected)?; + let mut act = walk_dir(actual)?; + + for (exp, act) in (&mut exp).zip(&mut act) { + let exp = exp?; + let act = act?; + + if exp.depth() != act.depth() { + return Err(anyhow::anyhow!( + "path(different depth): expected {} is different from actual: {}", + exp.path().display(), + act.path().display() + )); + } + + if exp.file_type() != act.file_type() { + return Err(anyhow::anyhow!( + "path(different filetype): expected {} is different from actual: {}", + exp.path().display(), + act.path().display() + )); + } + + if exp.file_name() != act.file_name() { + return Err(anyhow::anyhow!( + "path(different filename): expected {} is different from actual: {}", + exp.path().display(), + act.path().display() + )); + } + + if exp.metadata()?.is_file() { + let exp_file = tokio::fs::read_to_string(exp.path()).await?; + let act_file = tokio::fs::read_to_string(act.path()).await?; + + similar_asserts::assert_eq!(exp_file, act_file); + } + } + + if exp.next().is_some() || act.next().is_some() { + return Err(anyhow::anyhow!( + "path(uneven amount of items): expected: {}, actual: {}", + exp.next() + .map(|o| match o { + Ok(o) => o.path().display().to_string(), + Err(_) => "expected: not-found".to_string(), + }) + .unwrap_or("expected: not-found".into()), + act.next() + .map(|o| match o { + Ok(o) => o.path().display().to_string(), + Err(_) => "actual: not-found".to_string(), + }) + .unwrap_or("actual: not-found".into()), + )); + } + + Ok(()) +} + +fn walk_dir(path: &Path) -> anyhow::Result { + let mut walkdir = walkdir::WalkDir::new(path) + .sort_by(compare_by_file_name) + .into_iter(); + if let Some(Err(e)) = walkdir.next() { + Err(e.into()) + } else { + Ok(walkdir) + } +} + +fn compare_by_file_name(a: &DirEntry, b: &DirEntry) -> Ordering { + a.file_name().cmp(b.file_name()) +} diff --git a/crates/cuddle-clusters/tests/raw_files/cuddle.yaml b/crates/cuddle-clusters/tests/raw_files/cuddle.yaml new file mode 100644 index 0000000..b81909a --- /dev/null +++ b/crates/cuddle-clusters/tests/raw_files/cuddle.yaml @@ -0,0 +1,2 @@ +cuddle/clusters: + dev: diff --git a/crates/cuddle-clusters/tests/raw_files/expected/dev/some-file.txt b/crates/cuddle-clusters/tests/raw_files/expected/dev/some-file.txt new file mode 100644 index 0000000..f5efcf4 --- /dev/null +++ b/crates/cuddle-clusters/tests/raw_files/expected/dev/some-file.txt @@ -0,0 +1,2 @@ +some +file diff --git a/crates/cuddle-clusters/tests/raw_files/expected/dev/some-file.yaml b/crates/cuddle-clusters/tests/raw_files/expected/dev/some-file.yaml new file mode 100644 index 0000000..9e85e1e --- /dev/null +++ b/crates/cuddle-clusters/tests/raw_files/expected/dev/some-file.yaml @@ -0,0 +1,2 @@ +some: file +another: entry diff --git a/crates/cuddle-clusters/tests/raw_files/templates/clusters/raw/some-file.txt b/crates/cuddle-clusters/tests/raw_files/templates/clusters/raw/some-file.txt new file mode 100644 index 0000000..f5efcf4 --- /dev/null +++ b/crates/cuddle-clusters/tests/raw_files/templates/clusters/raw/some-file.txt @@ -0,0 +1,2 @@ +some +file diff --git a/crates/cuddle-clusters/tests/raw_files/templates/clusters/raw/some-file.yaml b/crates/cuddle-clusters/tests/raw_files/templates/clusters/raw/some-file.yaml new file mode 100644 index 0000000..9e85e1e --- /dev/null +++ b/crates/cuddle-clusters/tests/raw_files/templates/clusters/raw/some-file.yaml @@ -0,0 +1,2 @@ +some: file +another: entry diff --git a/crates/cuddle-clusters/tests/tests.rs b/crates/cuddle-clusters/tests/tests.rs index c0929b8..b4664e8 100644 --- a/crates/cuddle-clusters/tests/tests.rs +++ b/crates/cuddle-clusters/tests/tests.rs @@ -1,3 +1,12 @@ pub mod common; mod can_run_for_env; + +use crate::common::run_test; + +#[tokio::test] +async fn raw_files() -> anyhow::Result<()> { + run_test("raw_files").await?; + + Ok(()) +}