feat: add basic templates

Signed-off-by: kjuulh <contact@kjuulh.io>

.gitignore

@@ -1,2 +1,4 @@
 target/
 .cuddle/
+
+**/actual/

Cargo.lock

@@ -232,6 +232,17 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "bstr"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
+dependencies = [
+ "lazy_static",
+ "memchr",
+ "regex-automata 0.1.10",
+]
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -302,6 +313,18 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
 
+[[package]]
+name = "console"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
+dependencies = [
+ "encode_unicode",
+ "lazy_static",
+ "libc",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "const-oid"
 version = "0.9.6"
@@ -365,15 +388,19 @@ dependencies = [
  "axum",
  "clap",
  "dotenv",
+ "minijinja",
  "serde",
  "serde_yaml",
+ "similar-asserts",
  "sqlx",
  "tokio",
+ "tokio-stream",
  "tower-http",
  "tracing",
  "tracing-subscriber",
  "tracing-test",
  "uuid",
+ "walkdir",
 ]
 
 [[package]]
@@ -429,6 +456,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "encode_unicode"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+
 [[package]]
 name = "equivalent"
 version = "1.0.1"
@@ -885,6 +918,15 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
+[[package]]
+name = "minijinja"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7165d0e94806d52ad5295e4b54a95176d831814840bc067298ca647e1c956338"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -1336,6 +1378,15 @@ version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -1468,6 +1519,26 @@ dependencies = [
  "rand_core",
 ]
 
+[[package]]
+name = "similar"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640"
+dependencies = [
+ "bstr",
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "similar-asserts"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e041bb827d1bfca18f213411d51b665309f1afb37a04a5d1464530e13779fc0f"
+dependencies = [
+ "console",
+ "similar",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -2151,6 +2222,16 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -2195,6 +2276,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"

README.md

@@ -14,3 +14,10 @@ some-service/:
 ```
 
 For each env chosen, run code with own set of variables
+
+A function is run for each environment:
+
+- Input components which requires a set of variables
+- Templates in templates.clusters.*
+- All raw files in templates.clusters.raw.*
+

crates/cuddle-clusters/Cargo.toml

@@ -24,10 +24,14 @@ sqlx = { version = "0.7.3", features = [
 uuid = { version = "1.7.0", features = ["v4"] }
 tower-http = { version = "0.5.2", features = ["cors", "trace"] }
 serde_yaml = "0.9.34"
+tokio-stream = "0.1.15"
+walkdir = "2.5.0"
+minijinja = "2.0.1"
 
 [[test]]
 name = "integration"
 path = "tests/tests.rs"
 
 [dev-dependencies]
+similar-asserts = "1.5.0"
 tracing-test = "0.2.4"

crates/cuddle-clusters/src/process.rs

@@ -1,9 +1,12 @@
 use std::{
     collections::HashMap,
+    ops::Deref,
     path::{Path, PathBuf},
 };
 
 use anyhow::Context;
+use tokio::io::AsyncWriteExt;
+use tokio_stream::{wrappers::ReadDirStream, StreamExt};
 
 pub async fn process() -> anyhow::Result<()> {
     process_opts(ProcessOpts::default()).await
@@ -11,12 +14,17 @@ pub async fn process() -> anyhow::Result<()> {
 
 pub struct ProcessOpts {
     pub path: PathBuf,
+    pub output: PathBuf,
 }
 
 impl Default for ProcessOpts {
     fn default() -> Self {
         Self {
             path: std::env::current_dir().expect("to be able to get current dir"),
+            output: std::env::current_dir()
+                .expect("to be able to get current dir")
+                .join("cuddle-clusters")
+                .join("k8s"),
         }
     }
 }
@@ -36,7 +44,13 @@ pub async fn process_opts(opts: ProcessOpts) -> anyhow::Result<()> {
     let clusters = read_cuddle_section(&cuddle_path).await?;
     tracing::debug!("found clusters: {:?}", clusters);
 
-    load_template_files(&template).await?;
+    let template_files = load_template_files(&template).await?;
+    tracing::debug!("found files: {:?}", template_files);
+
+    tokio::fs::remove_dir_all(&opts.output).await?;
+    tokio::fs::create_dir_all(&opts.output).await?;
+
+    process_templates(&clusters, &template_files, &opts.output).await?;
 
     Ok(())
 }
@@ -44,8 +58,18 @@ pub async fn process_opts(opts: ProcessOpts) -> anyhow::Result<()> {
 #[derive(serde::Deserialize, Default, Debug, Clone)]
 struct CuddleClusters(HashMap<String, serde_yaml::Value>);
 
+impl Deref for CuddleClusters {
+    type Target = HashMap<String, serde_yaml::Value>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
 async fn read_cuddle_section(path: &Path) -> anyhow::Result<CuddleClusters> {
-    let cuddle_file = tokio::fs::read(path).await?;
+    let cuddle_file = tokio::fs::read(path)
+        .await
+        .context(format!("failed to read: {}", path.display()))?;
 
     let value: serde_yaml::Value = serde_yaml::from_slice(&cuddle_file)?;
 
@@ -63,6 +87,110 @@ async fn read_cuddle_section(path: &Path) -> anyhow::Result<CuddleClusters> {
     Ok(cuddle_clusters)
 }
 
-async fn load_template_files(path: &Path) -> anyhow::Result<()> {
+#[derive(Debug, Clone)]
+struct TemplateFiles {
+    templates: Vec<PathBuf>,
+    raw: Vec<PathBuf>,
+}
+
+async fn load_template_files(path: &Path) -> anyhow::Result<TemplateFiles> {
+    Ok(TemplateFiles {
+        templates: read_dir(path)
+            .await?
+            .into_iter()
+            .filter(|i| !i.ends_with(".jinja2"))
+            .collect(),
+        raw: read_dir(&path.join("raw")).await.unwrap_or_default(),
+    })
+}
+
+async fn read_dir(path: &Path) -> anyhow::Result<Vec<PathBuf>> {
+    let template_dir = tokio::fs::read_dir(path).await?;
+    let mut template_dir_stream = ReadDirStream::new(template_dir);
+
+    let mut paths = Vec::new();
+    while let Some(entry) = template_dir_stream.next().await {
+        let entry = entry?;
+
+        if entry.metadata().await?.is_file() {
+            paths.push(entry.path());
+        }
+    }
+
+    Ok(paths)
+}
+
+async fn process_templates(
+    clusters: &CuddleClusters,
+    template_files: &TemplateFiles,
+    dest: &Path,
+) -> anyhow::Result<()> {
+    for (cluster_name, _value) in clusters.iter() {
+        process_cluster(cluster_name, template_files, &dest.join(cluster_name)).await?;
+    }
+
+    Ok(())
+}
+
+async fn process_cluster(
+    cluster_name: &str,
+    template_files: &TemplateFiles,
+    dest: &Path,
+) -> anyhow::Result<()> {
+    for template_file in &template_files.templates {
+        process_template_file(cluster_name, template_file, dest).await?;
+    }
+
+    for raw_file in &template_files.raw {
+        process_raw_file(cluster_name, raw_file, dest).await?;
+    }
+
+    Ok(())
+}
+
+async fn process_template_file(
+    cluster_name: &str,
+    template_file: &PathBuf,
+    dest: &Path,
+) -> anyhow::Result<()> {
+    // TODO: use mini jinja
+    let file = tokio::fs::read_to_string(template_file)
+        .await
+        .context(format!("failed to find file: {}", template_file.display()))?;
+
+    if !dest.exists() {
+        tokio::fs::create_dir_all(dest).await?;
+    }
+
+    let file_name = template_file
+        .file_stem()
+        .ok_or(anyhow::anyhow!("file didn't have a jinja2 format"))?;
+
+    let mut dest_file = tokio::fs::File::create(dest.join(file_name)).await?;
+    dest_file.write_all(file.as_bytes()).await?;
+
+    Ok(())
+}
+
+async fn process_raw_file(
+    _cluster_name: &str,
+    raw_file: &PathBuf,
+    dest: &Path,
+) -> anyhow::Result<()> {
+    let file = tokio::fs::read_to_string(raw_file)
+        .await
+        .context(format!("failed to find file: {}", raw_file.display()))?;
+
+    if !dest.exists() {
+        tokio::fs::create_dir_all(dest).await?;
+    }
+
+    let file_name = raw_file
+        .file_name()
+        .ok_or(anyhow::anyhow!("file didn't have a file name"))?;
+
+    let mut dest_file = tokio::fs::File::create(dest.join(file_name)).await?;
+    dest_file.write_all(file.as_bytes()).await?;
+
     Ok(())
 }

crates/cuddle-clusters/tests/can_run_for_env/templates/clusters/test.yaml.jinja2

@@ -0,0 +1,6 @@
+{
+  hello = "world",
+  some = {
+    thing = "some"
+  }
+}

crates/cuddle-clusters/tests/common.rs

@@ -1,4 +1,7 @@
+use std::{cmp::Ordering, path::Path};
+
 use cuddle_clusters::process::ProcessOpts;
+use walkdir::DirEntry;
 
 pub(crate) async fn run_test(name: &str) -> anyhow::Result<()> {
     let _ = tracing_subscriber::fmt::try_init();
@@ -7,10 +10,104 @@ pub(crate) async fn run_test(name: &str) -> anyhow::Result<()> {
 
     let current_dir = std::env::current_dir()?;
 
+    let test_folder = current_dir.join("tests").join(name);
+
+    let actual = test_folder.join("actual");
+    tokio::fs::create_dir_all(&actual).await?;
+    let expected = test_folder.join("expected");
+    tokio::fs::create_dir_all(&expected).await?;
+
     cuddle_clusters::process_opts(ProcessOpts {
-        path: current_dir.join("tests").join(name),
+        path: test_folder.clone(),
+        output: actual.clone(),
     })
     .await?;
 
+    if std::env::var("TEST_OVERRIDE") == Ok("true".to_string()) {
+        cuddle_clusters::process_opts(ProcessOpts {
+            path: test_folder,
+            output: expected.clone(),
+        })
+        .await?;
+    }
+
+    compare(&expected, &actual).await?;
+
     Ok(())
 }
+
+async fn compare(expected: &Path, actual: &Path) -> anyhow::Result<()> {
+    let mut exp = walk_dir(expected)?;
+    let mut act = walk_dir(actual)?;
+
+    for (exp, act) in (&mut exp).zip(&mut act) {
+        let exp = exp?;
+        let act = act?;
+
+        if exp.depth() != act.depth() {
+            return Err(anyhow::anyhow!(
+                "path(different depth): expected {} is different from actual: {}",
+                exp.path().display(),
+                act.path().display()
+            ));
+        }
+
+        if exp.file_type() != act.file_type() {
+            return Err(anyhow::anyhow!(
+                "path(different filetype): expected {} is different from actual: {}",
+                exp.path().display(),
+                act.path().display()
+            ));
+        }
+
+        if exp.file_name() != act.file_name() {
+            return Err(anyhow::anyhow!(
+                "path(different filename): expected {} is different from actual: {}",
+                exp.path().display(),
+                act.path().display()
+            ));
+        }
+
+        if exp.metadata()?.is_file() {
+            let exp_file = tokio::fs::read_to_string(exp.path()).await?;
+            let act_file = tokio::fs::read_to_string(act.path()).await?;
+
+            similar_asserts::assert_eq!(exp_file, act_file);
+        }
+    }
+
+    if exp.next().is_some() || act.next().is_some() {
+        return Err(anyhow::anyhow!(
+            "path(uneven amount of items): expected: {}, actual: {}",
+            exp.next()
+                .map(|o| match o {
+                    Ok(o) => o.path().display().to_string(),
+                    Err(_) => "expected: not-found".to_string(),
+                })
+                .unwrap_or("expected: not-found".into()),
+            act.next()
+                .map(|o| match o {
+                    Ok(o) => o.path().display().to_string(),
+                    Err(_) => "actual: not-found".to_string(),
+                })
+                .unwrap_or("actual: not-found".into()),
+        ));
+    }
+
+    Ok(())
+}
+
+fn walk_dir(path: &Path) -> anyhow::Result<walkdir::IntoIter> {
+    let mut walkdir = walkdir::WalkDir::new(path)
+        .sort_by(compare_by_file_name)
+        .into_iter();
+    if let Some(Err(e)) = walkdir.next() {
+        Err(e.into())
+    } else {
+        Ok(walkdir)
+    }
+}
+
+fn compare_by_file_name(a: &DirEntry, b: &DirEntry) -> Ordering {
+    a.file_name().cmp(b.file_name())
+}

crates/cuddle-clusters/tests/raw_files/cuddle.yaml

@@ -0,0 +1,2 @@
+cuddle/clusters:
+  dev:

crates/cuddle-clusters/tests/raw_files/expected/dev/some-file.txt

@@ -0,0 +1,2 @@
+some
+file

crates/cuddle-clusters/tests/raw_files/expected/dev/some-file.yaml

@@ -0,0 +1,2 @@
+some: file
+another: entry

crates/cuddle-clusters/tests/raw_files/templates/clusters/raw/some-file.txt

@@ -0,0 +1,2 @@
+some
+file

crates/cuddle-clusters/tests/raw_files/templates/clusters/raw/some-file.yaml

@@ -0,0 +1,2 @@
+some: file
+another: entry

crates/cuddle-clusters/tests/tests.rs

@@ -1,3 +1,12 @@
 pub mod common;
 
 mod can_run_for_env;
+
+use crate::common::run_test;
+
+#[tokio::test]
+async fn raw_files() -> anyhow::Result<()> {
+    run_test("raw_files").await?;
+
+    Ok(())
+}