From 461f8acb73ee8c867450167348646fa480e73953 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Tue, 21 May 2024 21:46:22 +0200 Subject: [PATCH] feat: can recursively load files Signed-off-by: kjuulh --- crates/cuddle-clusters/src/lib.rs | 2 + crates/cuddle-clusters/src/process.rs | 78 ++++++++++++++----- .../cuddle-clusters/tests/nested/cuddle.yaml | 2 + .../tests/nested/expected/dev/plan-raw | 1 + .../tests/nested/expected/dev/plan-template | 1 + .../tests/nested/expected/dev/service-raw | 1 + .../nested/expected/dev/service-template | 1 + .../tests/nested/expected/dev/some-file.yaml | 1 + .../tests/nested/expected/dev/some-other.yaml | 1 + .../tests/nested/expected/dev/some.yaml | 1 + .../nested/templates/clusters/raw/service-raw | 1 + .../templates/clusters/raw/some-file.yaml | 1 + .../clusters/service-template.jinja2 | 1 + .../templates/clusters/some-other.yaml.jinja2 | 1 + .../templates/clusters/some.yaml.jinja2 | 1 + crates/cuddle-clusters/tests/tests.rs | 7 ++ 16 files changed, 83 insertions(+), 18 deletions(-) create mode 100644 crates/cuddle-clusters/tests/nested/cuddle.yaml create mode 100644 crates/cuddle-clusters/tests/nested/expected/dev/plan-raw create mode 100644 crates/cuddle-clusters/tests/nested/expected/dev/plan-template create mode 100644 crates/cuddle-clusters/tests/nested/expected/dev/service-raw create mode 100644 crates/cuddle-clusters/tests/nested/expected/dev/service-template create mode 100644 crates/cuddle-clusters/tests/nested/expected/dev/some-file.yaml create mode 100644 crates/cuddle-clusters/tests/nested/expected/dev/some-other.yaml create mode 100644 crates/cuddle-clusters/tests/nested/expected/dev/some.yaml create mode 100644 crates/cuddle-clusters/tests/nested/templates/clusters/raw/service-raw create mode 100644 crates/cuddle-clusters/tests/nested/templates/clusters/raw/some-file.yaml create mode 100644 crates/cuddle-clusters/tests/nested/templates/clusters/service-template.jinja2 create mode 100644 crates/cuddle-clusters/tests/nested/templates/clusters/some-other.yaml.jinja2 create mode 100644 crates/cuddle-clusters/tests/nested/templates/clusters/some.yaml.jinja2 diff --git a/crates/cuddle-clusters/src/lib.rs b/crates/cuddle-clusters/src/lib.rs index 8c6b2a6..89aa60d 100644 --- a/crates/cuddle-clusters/src/lib.rs +++ b/crates/cuddle-clusters/src/lib.rs @@ -1,3 +1,5 @@ +#![feature(map_try_insert)] + pub mod components; pub use components::*; diff --git a/crates/cuddle-clusters/src/process.rs b/crates/cuddle-clusters/src/process.rs index f8001ec..3f7383b 100644 --- a/crates/cuddle-clusters/src/process.rs +++ b/crates/cuddle-clusters/src/process.rs @@ -5,6 +5,7 @@ use std::{ }; use anyhow::Context; +use futures::{future::BoxFuture, FutureExt}; use minijinja::context; use tokio::io::AsyncWriteExt; use tokio_stream::{wrappers::ReadDirStream, StreamExt}; @@ -32,6 +33,9 @@ impl Default for ProcessOpts { } } +const TEMPLATES_PATH_PREFIX: &str = "templates/clusters"; +const CUDDLE_PLAN_PATH_PREFIX: &str = ".cuddle/plan"; + pub async fn process_opts( components: impl IntoIterator, opts: ProcessOpts, @@ -44,18 +48,17 @@ pub async fn process_opts( let path = opts.path.canonicalize().context("failed to find folder")?; let cuddle_path = path.join("cuddle.yaml"); - let template = path.join("templates").join("clusters"); tracing::debug!( "searching for templates in: {} with cuddle: {}", - template.display(), + path.display(), cuddle_path.display() ); let clusters = read_cuddle_section(&cuddle_path).await?; tracing::debug!("found clusters: {:?}", clusters); - let template_files = load_template_files(&template).await?; + let template_files = load_template_files(&path).await?; tracing::debug!("found files: {:?}", template_files); tokio::fs::remove_dir_all(&opts.output).await?; @@ -100,31 +103,70 @@ async fn read_cuddle_section(path: &Path) -> anyhow::Result { #[derive(Debug, Clone)] struct TemplateFiles { - templates: Vec, - raw: Vec, + templates: HashMap, + raw: HashMap, } -async fn load_template_files(path: &Path) -> anyhow::Result { - Ok(TemplateFiles { - templates: read_dir(path) +impl TemplateFiles { + fn merge(&mut self, template_files: TemplateFiles) { + for (name, path) in template_files.templates { + // Ignored if the key is already present + let _ = self.templates.try_insert(name, path); + } + for (name, path) in template_files.raw { + // Ignored if the key is already present + let _ = self.raw.try_insert(name, path); + } + } +} + +fn load_template_files(path: &Path) -> BoxFuture<'static, anyhow::Result> { + let template_path = path.join(TEMPLATES_PATH_PREFIX); + let path = path.to_path_buf(); + + tracing::info!("reading folder: {}", path.display()); + + async move { + let templates = read_dir(&template_path) .await? .into_iter() - .filter(|i| i.extension().and_then(|e| e.to_str()) == Some("jinja2")) - .collect(), - raw: read_dir(&path.join("raw")).await.unwrap_or_default(), - }) + .filter(|(_, i)| i.extension().and_then(|e| e.to_str()) == Some("jinja2")) + .collect(); + let raw = read_dir(&template_path.join("raw")) + .await + .unwrap_or_default(); + let mut template_files = TemplateFiles { templates, raw }; + + let nested_path = path.join(CUDDLE_PLAN_PATH_PREFIX); + tracing::info!("trying to read: {}", nested_path.display()); + if nested_path.exists() { + let nested = load_template_files(&nested_path).await?; + template_files.merge(nested); + } + + Ok(template_files) + } + .boxed() } -async fn read_dir(path: &Path) -> anyhow::Result> { +async fn read_dir(path: &Path) -> anyhow::Result> { let template_dir = tokio::fs::read_dir(path).await?; let mut template_dir_stream = ReadDirStream::new(template_dir); - let mut paths = Vec::new(); + let mut paths = HashMap::new(); while let Some(entry) = template_dir_stream.next().await { let entry = entry?; if entry.metadata().await?.is_file() { - paths.push(entry.path()); + paths.insert( + entry + .path() + .file_name() + .expect("the file to have a filename") + .to_string_lossy() + .to_string(), + entry.path(), + ); } } @@ -139,7 +181,7 @@ async fn process_templates( ) -> anyhow::Result<()> { for (environment, value) in clusters.iter() { process_cluster( - &components, + components, value, environment, template_files, @@ -158,11 +200,11 @@ async fn process_cluster( template_files: &TemplateFiles, dest: &Path, ) -> anyhow::Result<()> { - for template_file in &template_files.templates { + for (_, template_file) in &template_files.templates { process_template_file(components, value, environment, template_file, dest).await?; } - for raw_file in &template_files.raw { + for (_, raw_file) in &template_files.raw { process_raw_file(environment, raw_file, dest).await?; } diff --git a/crates/cuddle-clusters/tests/nested/cuddle.yaml b/crates/cuddle-clusters/tests/nested/cuddle.yaml new file mode 100644 index 0000000..b81909a --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/cuddle.yaml @@ -0,0 +1,2 @@ +cuddle/clusters: + dev: diff --git a/crates/cuddle-clusters/tests/nested/expected/dev/plan-raw b/crates/cuddle-clusters/tests/nested/expected/dev/plan-raw new file mode 100644 index 0000000..79c5395 --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/expected/dev/plan-raw @@ -0,0 +1 @@ +nested diff --git a/crates/cuddle-clusters/tests/nested/expected/dev/plan-template b/crates/cuddle-clusters/tests/nested/expected/dev/plan-template new file mode 100644 index 0000000..bfe53d7 --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/expected/dev/plan-template @@ -0,0 +1 @@ +nested \ No newline at end of file diff --git a/crates/cuddle-clusters/tests/nested/expected/dev/service-raw b/crates/cuddle-clusters/tests/nested/expected/dev/service-raw new file mode 100644 index 0000000..24e1098 --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/expected/dev/service-raw @@ -0,0 +1 @@ +service diff --git a/crates/cuddle-clusters/tests/nested/expected/dev/service-template b/crates/cuddle-clusters/tests/nested/expected/dev/service-template new file mode 100644 index 0000000..ce93fac --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/expected/dev/service-template @@ -0,0 +1 @@ +service \ No newline at end of file diff --git a/crates/cuddle-clusters/tests/nested/expected/dev/some-file.yaml b/crates/cuddle-clusters/tests/nested/expected/dev/some-file.yaml new file mode 100644 index 0000000..24e1098 --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/expected/dev/some-file.yaml @@ -0,0 +1 @@ +service diff --git a/crates/cuddle-clusters/tests/nested/expected/dev/some-other.yaml b/crates/cuddle-clusters/tests/nested/expected/dev/some-other.yaml new file mode 100644 index 0000000..ce93fac --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/expected/dev/some-other.yaml @@ -0,0 +1 @@ +service \ No newline at end of file diff --git a/crates/cuddle-clusters/tests/nested/expected/dev/some.yaml b/crates/cuddle-clusters/tests/nested/expected/dev/some.yaml new file mode 100644 index 0000000..ce93fac --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/expected/dev/some.yaml @@ -0,0 +1 @@ +service \ No newline at end of file diff --git a/crates/cuddle-clusters/tests/nested/templates/clusters/raw/service-raw b/crates/cuddle-clusters/tests/nested/templates/clusters/raw/service-raw new file mode 100644 index 0000000..24e1098 --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/templates/clusters/raw/service-raw @@ -0,0 +1 @@ +service diff --git a/crates/cuddle-clusters/tests/nested/templates/clusters/raw/some-file.yaml b/crates/cuddle-clusters/tests/nested/templates/clusters/raw/some-file.yaml new file mode 100644 index 0000000..24e1098 --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/templates/clusters/raw/some-file.yaml @@ -0,0 +1 @@ +service diff --git a/crates/cuddle-clusters/tests/nested/templates/clusters/service-template.jinja2 b/crates/cuddle-clusters/tests/nested/templates/clusters/service-template.jinja2 new file mode 100644 index 0000000..24e1098 --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/templates/clusters/service-template.jinja2 @@ -0,0 +1 @@ +service diff --git a/crates/cuddle-clusters/tests/nested/templates/clusters/some-other.yaml.jinja2 b/crates/cuddle-clusters/tests/nested/templates/clusters/some-other.yaml.jinja2 new file mode 100644 index 0000000..24e1098 --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/templates/clusters/some-other.yaml.jinja2 @@ -0,0 +1 @@ +service diff --git a/crates/cuddle-clusters/tests/nested/templates/clusters/some.yaml.jinja2 b/crates/cuddle-clusters/tests/nested/templates/clusters/some.yaml.jinja2 new file mode 100644 index 0000000..24e1098 --- /dev/null +++ b/crates/cuddle-clusters/tests/nested/templates/clusters/some.yaml.jinja2 @@ -0,0 +1 @@ +service diff --git a/crates/cuddle-clusters/tests/tests.rs b/crates/cuddle-clusters/tests/tests.rs index d7abbaa..77c0aa5 100644 --- a/crates/cuddle-clusters/tests/tests.rs +++ b/crates/cuddle-clusters/tests/tests.rs @@ -27,6 +27,13 @@ async fn both() -> anyhow::Result<()> { Ok(()) } +#[tokio::test] +async fn nested() -> anyhow::Result<()> { + run_test("nested").await?; + + Ok(()) +} + #[tokio::test] async fn jinja() -> anyhow::Result<()> { run_test("jinja").await?;