feat: add basic templates

Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
Kasper Juul Hermansen 2024-05-20 20:45:26 +02:00
parent fe48259d06
commit f04b0a2e54
Signed by: kjuulh
GPG Key ID: 57B6E1465221F912
14 changed files with 357 additions and 4 deletions

2
.gitignore vendored
View File

@ -1,2 +1,4 @@
target/ target/
.cuddle/ .cuddle/
**/actual/

90
Cargo.lock generated
View File

@ -232,6 +232,17 @@ dependencies = [
"generic-array", "generic-array",
] ]
[[package]]
name = "bstr"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
dependencies = [
"lazy_static",
"memchr",
"regex-automata 0.1.10",
]
[[package]] [[package]]
name = "byteorder" name = "byteorder"
version = "1.5.0" version = "1.5.0"
@ -302,6 +313,18 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
[[package]]
name = "console"
version = "0.15.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
dependencies = [
"encode_unicode",
"lazy_static",
"libc",
"windows-sys 0.52.0",
]
[[package]] [[package]]
name = "const-oid" name = "const-oid"
version = "0.9.6" version = "0.9.6"
@ -365,15 +388,19 @@ dependencies = [
"axum", "axum",
"clap", "clap",
"dotenv", "dotenv",
"minijinja",
"serde", "serde",
"serde_yaml", "serde_yaml",
"similar-asserts",
"sqlx", "sqlx",
"tokio", "tokio",
"tokio-stream",
"tower-http", "tower-http",
"tracing", "tracing",
"tracing-subscriber", "tracing-subscriber",
"tracing-test", "tracing-test",
"uuid", "uuid",
"walkdir",
] ]
[[package]] [[package]]
@ -429,6 +456,12 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]] [[package]]
name = "equivalent" name = "equivalent"
version = "1.0.1" version = "1.0.1"
@ -885,6 +918,15 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "minijinja"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7165d0e94806d52ad5295e4b54a95176d831814840bc067298ca647e1c956338"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "minimal-lexical" name = "minimal-lexical"
version = "0.2.1" version = "0.2.1"
@ -1336,6 +1378,15 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]] [[package]]
name = "scopeguard" name = "scopeguard"
version = "1.2.0" version = "1.2.0"
@ -1468,6 +1519,26 @@ dependencies = [
"rand_core", "rand_core",
] ]
[[package]]
name = "similar"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640"
dependencies = [
"bstr",
"unicode-segmentation",
]
[[package]]
name = "similar-asserts"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e041bb827d1bfca18f213411d51b665309f1afb37a04a5d1464530e13779fc0f"
dependencies = [
"console",
"similar",
]
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.9" version = "0.4.9"
@ -2151,6 +2222,16 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]] [[package]]
name = "wasi" name = "wasi"
version = "0.11.0+wasi-snapshot-preview1" version = "0.11.0+wasi-snapshot-preview1"
@ -2195,6 +2276,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
dependencies = [
"windows-sys 0.52.0",
]
[[package]] [[package]]
name = "winapi-x86_64-pc-windows-gnu" name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0" version = "0.4.0"

View File

@ -14,3 +14,10 @@ some-service/:
``` ```
For each env chosen, run code with own set of variables For each env chosen, run code with own set of variables
A function is run for each environment:
- Input components which requires a set of variables
- Templates in templates.clusters.*
- All raw files in templates.clusters.raw.*

View File

@ -24,10 +24,14 @@ sqlx = { version = "0.7.3", features = [
uuid = { version = "1.7.0", features = ["v4"] } uuid = { version = "1.7.0", features = ["v4"] }
tower-http = { version = "0.5.2", features = ["cors", "trace"] } tower-http = { version = "0.5.2", features = ["cors", "trace"] }
serde_yaml = "0.9.34" serde_yaml = "0.9.34"
tokio-stream = "0.1.15"
walkdir = "2.5.0"
minijinja = "2.0.1"
[[test]] [[test]]
name = "integration" name = "integration"
path = "tests/tests.rs" path = "tests/tests.rs"
[dev-dependencies] [dev-dependencies]
similar-asserts = "1.5.0"
tracing-test = "0.2.4" tracing-test = "0.2.4"

View File

@ -1,9 +1,12 @@
use std::{ use std::{
collections::HashMap, collections::HashMap,
ops::Deref,
path::{Path, PathBuf}, path::{Path, PathBuf},
}; };
use anyhow::Context; use anyhow::Context;
use tokio::io::AsyncWriteExt;
use tokio_stream::{wrappers::ReadDirStream, StreamExt};
pub async fn process() -> anyhow::Result<()> { pub async fn process() -> anyhow::Result<()> {
process_opts(ProcessOpts::default()).await process_opts(ProcessOpts::default()).await
@ -11,12 +14,17 @@ pub async fn process() -> anyhow::Result<()> {
pub struct ProcessOpts { pub struct ProcessOpts {
pub path: PathBuf, pub path: PathBuf,
pub output: PathBuf,
} }
impl Default for ProcessOpts { impl Default for ProcessOpts {
fn default() -> Self { fn default() -> Self {
Self { Self {
path: std::env::current_dir().expect("to be able to get current dir"), path: std::env::current_dir().expect("to be able to get current dir"),
output: std::env::current_dir()
.expect("to be able to get current dir")
.join("cuddle-clusters")
.join("k8s"),
} }
} }
} }
@ -36,7 +44,13 @@ pub async fn process_opts(opts: ProcessOpts) -> anyhow::Result<()> {
let clusters = read_cuddle_section(&cuddle_path).await?; let clusters = read_cuddle_section(&cuddle_path).await?;
tracing::debug!("found clusters: {:?}", clusters); tracing::debug!("found clusters: {:?}", clusters);
load_template_files(&template).await?; let template_files = load_template_files(&template).await?;
tracing::debug!("found files: {:?}", template_files);
tokio::fs::remove_dir_all(&opts.output).await?;
tokio::fs::create_dir_all(&opts.output).await?;
process_templates(&clusters, &template_files, &opts.output).await?;
Ok(()) Ok(())
} }
@ -44,8 +58,18 @@ pub async fn process_opts(opts: ProcessOpts) -> anyhow::Result<()> {
#[derive(serde::Deserialize, Default, Debug, Clone)] #[derive(serde::Deserialize, Default, Debug, Clone)]
struct CuddleClusters(HashMap<String, serde_yaml::Value>); struct CuddleClusters(HashMap<String, serde_yaml::Value>);
impl Deref for CuddleClusters {
type Target = HashMap<String, serde_yaml::Value>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
async fn read_cuddle_section(path: &Path) -> anyhow::Result<CuddleClusters> { async fn read_cuddle_section(path: &Path) -> anyhow::Result<CuddleClusters> {
let cuddle_file = tokio::fs::read(path).await?; let cuddle_file = tokio::fs::read(path)
.await
.context(format!("failed to read: {}", path.display()))?;
let value: serde_yaml::Value = serde_yaml::from_slice(&cuddle_file)?; let value: serde_yaml::Value = serde_yaml::from_slice(&cuddle_file)?;
@ -63,6 +87,110 @@ async fn read_cuddle_section(path: &Path) -> anyhow::Result<CuddleClusters> {
Ok(cuddle_clusters) Ok(cuddle_clusters)
} }
async fn load_template_files(path: &Path) -> anyhow::Result<()> { #[derive(Debug, Clone)]
struct TemplateFiles {
templates: Vec<PathBuf>,
raw: Vec<PathBuf>,
}
async fn load_template_files(path: &Path) -> anyhow::Result<TemplateFiles> {
Ok(TemplateFiles {
templates: read_dir(path)
.await?
.into_iter()
.filter(|i| !i.ends_with(".jinja2"))
.collect(),
raw: read_dir(&path.join("raw")).await.unwrap_or_default(),
})
}
async fn read_dir(path: &Path) -> anyhow::Result<Vec<PathBuf>> {
let template_dir = tokio::fs::read_dir(path).await?;
let mut template_dir_stream = ReadDirStream::new(template_dir);
let mut paths = Vec::new();
while let Some(entry) = template_dir_stream.next().await {
let entry = entry?;
if entry.metadata().await?.is_file() {
paths.push(entry.path());
}
}
Ok(paths)
}
async fn process_templates(
clusters: &CuddleClusters,
template_files: &TemplateFiles,
dest: &Path,
) -> anyhow::Result<()> {
for (cluster_name, _value) in clusters.iter() {
process_cluster(cluster_name, template_files, &dest.join(cluster_name)).await?;
}
Ok(())
}
async fn process_cluster(
cluster_name: &str,
template_files: &TemplateFiles,
dest: &Path,
) -> anyhow::Result<()> {
for template_file in &template_files.templates {
process_template_file(cluster_name, template_file, dest).await?;
}
for raw_file in &template_files.raw {
process_raw_file(cluster_name, raw_file, dest).await?;
}
Ok(())
}
async fn process_template_file(
cluster_name: &str,
template_file: &PathBuf,
dest: &Path,
) -> anyhow::Result<()> {
// TODO: use mini jinja
let file = tokio::fs::read_to_string(template_file)
.await
.context(format!("failed to find file: {}", template_file.display()))?;
if !dest.exists() {
tokio::fs::create_dir_all(dest).await?;
}
let file_name = template_file
.file_stem()
.ok_or(anyhow::anyhow!("file didn't have a jinja2 format"))?;
let mut dest_file = tokio::fs::File::create(dest.join(file_name)).await?;
dest_file.write_all(file.as_bytes()).await?;
Ok(())
}
async fn process_raw_file(
_cluster_name: &str,
raw_file: &PathBuf,
dest: &Path,
) -> anyhow::Result<()> {
let file = tokio::fs::read_to_string(raw_file)
.await
.context(format!("failed to find file: {}", raw_file.display()))?;
if !dest.exists() {
tokio::fs::create_dir_all(dest).await?;
}
let file_name = raw_file
.file_name()
.ok_or(anyhow::anyhow!("file didn't have a file name"))?;
let mut dest_file = tokio::fs::File::create(dest.join(file_name)).await?;
dest_file.write_all(file.as_bytes()).await?;
Ok(()) Ok(())
} }

View File

@ -0,0 +1,6 @@
{
hello = "world",
some = {
thing = "some"
}
}

View File

@ -1,4 +1,7 @@
use std::{cmp::Ordering, path::Path};
use cuddle_clusters::process::ProcessOpts; use cuddle_clusters::process::ProcessOpts;
use walkdir::DirEntry;
pub(crate) async fn run_test(name: &str) -> anyhow::Result<()> { pub(crate) async fn run_test(name: &str) -> anyhow::Result<()> {
let _ = tracing_subscriber::fmt::try_init(); let _ = tracing_subscriber::fmt::try_init();
@ -7,10 +10,104 @@ pub(crate) async fn run_test(name: &str) -> anyhow::Result<()> {
let current_dir = std::env::current_dir()?; let current_dir = std::env::current_dir()?;
let test_folder = current_dir.join("tests").join(name);
let actual = test_folder.join("actual");
tokio::fs::create_dir_all(&actual).await?;
let expected = test_folder.join("expected");
tokio::fs::create_dir_all(&expected).await?;
cuddle_clusters::process_opts(ProcessOpts { cuddle_clusters::process_opts(ProcessOpts {
path: current_dir.join("tests").join(name), path: test_folder.clone(),
output: actual.clone(),
}) })
.await?; .await?;
if std::env::var("TEST_OVERRIDE") == Ok("true".to_string()) {
cuddle_clusters::process_opts(ProcessOpts {
path: test_folder,
output: expected.clone(),
})
.await?;
}
compare(&expected, &actual).await?;
Ok(()) Ok(())
} }
async fn compare(expected: &Path, actual: &Path) -> anyhow::Result<()> {
let mut exp = walk_dir(expected)?;
let mut act = walk_dir(actual)?;
for (exp, act) in (&mut exp).zip(&mut act) {
let exp = exp?;
let act = act?;
if exp.depth() != act.depth() {
return Err(anyhow::anyhow!(
"path(different depth): expected {} is different from actual: {}",
exp.path().display(),
act.path().display()
));
}
if exp.file_type() != act.file_type() {
return Err(anyhow::anyhow!(
"path(different filetype): expected {} is different from actual: {}",
exp.path().display(),
act.path().display()
));
}
if exp.file_name() != act.file_name() {
return Err(anyhow::anyhow!(
"path(different filename): expected {} is different from actual: {}",
exp.path().display(),
act.path().display()
));
}
if exp.metadata()?.is_file() {
let exp_file = tokio::fs::read_to_string(exp.path()).await?;
let act_file = tokio::fs::read_to_string(act.path()).await?;
similar_asserts::assert_eq!(exp_file, act_file);
}
}
if exp.next().is_some() || act.next().is_some() {
return Err(anyhow::anyhow!(
"path(uneven amount of items): expected: {}, actual: {}",
exp.next()
.map(|o| match o {
Ok(o) => o.path().display().to_string(),
Err(_) => "expected: not-found".to_string(),
})
.unwrap_or("expected: not-found".into()),
act.next()
.map(|o| match o {
Ok(o) => o.path().display().to_string(),
Err(_) => "actual: not-found".to_string(),
})
.unwrap_or("actual: not-found".into()),
));
}
Ok(())
}
fn walk_dir(path: &Path) -> anyhow::Result<walkdir::IntoIter> {
let mut walkdir = walkdir::WalkDir::new(path)
.sort_by(compare_by_file_name)
.into_iter();
if let Some(Err(e)) = walkdir.next() {
Err(e.into())
} else {
Ok(walkdir)
}
}
fn compare_by_file_name(a: &DirEntry, b: &DirEntry) -> Ordering {
a.file_name().cmp(b.file_name())
}

View File

@ -0,0 +1,2 @@
cuddle/clusters:
dev:

View File

@ -0,0 +1,2 @@
some
file

View File

@ -0,0 +1,2 @@
some: file
another: entry

View File

@ -0,0 +1,2 @@
some
file

View File

@ -0,0 +1,2 @@
some: file
another: entry

View File

@ -1,3 +1,12 @@
pub mod common; pub mod common;
mod can_run_for_env; mod can_run_for_env;
use crate::common::run_test;
#[tokio::test]
async fn raw_files() -> anyhow::Result<()> {
run_test("raw_files").await?;
Ok(())
}