diff --git a/Cargo.lock b/Cargo.lock index 24e6ef2..6054c58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -538,6 +538,7 @@ dependencies = [ "anyhow", "async-trait", "clap 4.4.4", + "crunch-codegen", "crunch-file", "thiserror", "tokio", @@ -554,11 +555,15 @@ dependencies = [ "bytes 0.4.12", "crunch-file", "crunch-traits", + "genco", + "pretty_assertions", "prost 0.12.1", "prost-build", "prost-types 0.12.1", + "tempfile", "tokio", "tracing", + "walkdir", ] [[package]] @@ -888,6 +893,28 @@ dependencies = [ "slab", ] +[[package]] +name = "genco" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6973ce8518068a71d404f428f6a5b563088545546a6bd8f9c0a7f2608149bc8a" +dependencies = [ + "genco-macros", + "relative-path", + "smallvec", +] + +[[package]] +name = "genco-macros" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c2c778cf01917d0fbed53900259d6604a421fab4916a2e738856ead9f1d926a" +dependencies = [ + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1604,6 +1631,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +[[package]] +name = "relative-path" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca" + [[package]] name = "ring" version = "0.16.20" @@ -1907,6 +1940,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2 1.0.67", + "quote 1.0.33", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.37" diff --git a/Cargo.toml b/Cargo.toml index 2f2911e..805df18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ crunch-envelope = { path = "crates/crunch-envelope" } crunch-in-memory = { path = "crates/crunch-in-memory" } crunch-nats = { path = "crates/crunch-nats" } crunch-file = {path = "crates/crunch-file"} +crunch-codegen = {path = "crates/crunch-codegen"} anyhow = { version = "1.0.71" } tokio = { version = "1", features = ["full"] } @@ -27,6 +28,9 @@ prost = {version = "0.12"} prost-types = {version = "0.12"} prost-build = "0.5" bytes = {version = "0.4"} +tempfile = {version = "3.8.0"} +genco = {version = "0.17.5"} +walkdir = {version = "2.4.0"} pretty_assertions = "1.4.0" \ No newline at end of file diff --git a/crates/crunch-cli/Cargo.toml b/crates/crunch-cli/Cargo.toml index d2a9053..427951f 100644 --- a/crates/crunch-cli/Cargo.toml +++ b/crates/crunch-cli/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] crunch-file.workspace = true +crunch-codegen.workspace = true anyhow.workspace = true tracing.workspace = true diff --git a/crates/crunch-cli/src/main.rs b/crates/crunch-cli/src/main.rs index 0ff08aa..93edc58 100644 --- a/crates/crunch-cli/src/main.rs +++ b/crates/crunch-cli/src/main.rs @@ -1,11 +1,10 @@ mod logging; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::anyhow; -use clap::{Args, Parser, Subcommand, ValueEnum}; +use clap::{Args, Parser, Subcommand}; use logging::LogArg; -use tracing::Level; #[derive(Parser, Clone)] #[command(author, version, about, long_about = None, subcommand_required = true)] @@ -43,13 +42,30 @@ async fn main() -> anyhow::Result<()> { match &cli.commands { Commands::Generate {} => { - let config_file = config::get_file(&cli.global_args.crunch_file) + let config = config::get_file(&cli.global_args.crunch_file) .await .map_err(|e| anyhow!("failed to load config: {}", e))? .get_config() .map_err(|e| anyhow!("invalid config: {}", e))?; - tracing::info!("generating crunch code") + tracing::info!("generating crunch code"); + let codegen = crunch_codegen::Codegen::new(); + + if let Some(publish) = config.publish { + for p in &publish { + let mut rel_schema_path = PathBuf::from(&p.schema_path); + let mut rel_output_path = PathBuf::from(&p.output_path); + + if let Some(dir_path) = cli.global_args.crunch_file.parent() { + rel_schema_path = dir_path.join(rel_schema_path); + rel_output_path = dir_path.join(rel_output_path); + } + + codegen + .generate_rust(&rel_schema_path, &rel_output_path) + .await?; + } + } } } diff --git a/crates/crunch-codegen/Cargo.toml b/crates/crunch-codegen/Cargo.toml index 16eb84d..e8d27c1 100644 --- a/crates/crunch-codegen/Cargo.toml +++ b/crates/crunch-codegen/Cargo.toml @@ -17,3 +17,9 @@ prost.workspace = true prost-types.workspace = true prost-build.workspace = true bytes.workspace = true +tempfile.workspace = true +genco.workspace = true +walkdir.workspace = true + +[dev-dependencies] +pretty_assertions.workspace = true diff --git a/crates/crunch-codegen/src/lib.rs b/crates/crunch-codegen/src/lib.rs index 192c2a8..1535560 100644 --- a/crates/crunch-codegen/src/lib.rs +++ b/crates/crunch-codegen/src/lib.rs @@ -1,15 +1,284 @@ +use anyhow::anyhow; +use genco::prelude::*; +use std::path::{Path, PathBuf}; +use tokio::io::AsyncWriteExt; +use walkdir::WalkDir; + +pub struct Codegen {} + +impl Codegen { + pub fn new() -> Self { + Self {} + } + + pub async fn generate_rust(&self, input_path: &Path, output_path: &Path) -> anyhow::Result<()> { + let input_protos = self.discover_files(input_path, "proto")?; + let (input_proto_paths, input_dir) = self.copy_protos(input_protos, input_path).await?; + let (output_proto_paths, output_dir) = self + .generate_rust_from_proto(input_proto_paths, output_path, input_dir.path()) + .await?; + + self.copy_rs(output_proto_paths, output_path, output_dir.path()) + .await?; + + Ok(()) + } + + fn discover_files(&self, input_path: &Path, extension: &str) -> anyhow::Result> { + let mut proto_files = Vec::new(); + for entry in WalkDir::new(input_path) { + let entry = entry?; + + if let Some(extension) = entry.path().extension().and_then(|e| e.to_str()) { + proto_files.push(entry.into_path()); + } + } + + if proto_files.is_empty() { + anyhow::anyhow!( + "failed to find any protobuf files in: {}", + input_path.display() + ); + } + + Ok(proto_files) + } + + async fn copy_protos( + &self, + input_protos: Vec, + root_path: &Path, + ) -> anyhow::Result<(Vec, tempfile::TempDir)> { + let in_tempdir = tempfile::TempDir::new()?; + let in_tempdir_path = in_tempdir.path(); + + let mut input_proto_paths = Vec::new(); + for input_proto in &input_protos { + let rel_proto_path = input_proto.strip_prefix(root_path)?; + let in_proto_path = in_tempdir_path.join(rel_proto_path); + if let Some(dir) = in_proto_path.parent() { + if !dir.exists() { + tokio::fs::create_dir_all(dir).await?; + } + } + + tokio::fs::copy(input_proto, &in_proto_path).await?; + input_proto_paths.push(in_proto_path); + } + + Ok((input_proto_paths, in_tempdir)) + } + + async fn generate_rust_from_proto( + &self, + input_proto_paths: Vec, + output_path: &Path, + in_root_path: &Path, + ) -> anyhow::Result<(Vec, tempfile::TempDir)> { + let out_tempdir = tempfile::TempDir::new()?; + let out_tempdir_path = out_tempdir.path(); + let handle = tokio::task::spawn_blocking({ + let out_tempdir_path = out_tempdir_path.to_path_buf(); + let in_root_path = in_root_path.to_path_buf(); + move || { + prost_build::Config::new() + .out_dir(out_tempdir_path) + .compile_protos(input_proto_paths.as_slice(), &[in_root_path])?; + + Ok(()) + } + }); + + let result: anyhow::Result<()> = handle.await?; + result?; + + let mut output_paths = self.discover_files(&out_tempdir_path, "rs")?; + + let mod_path = self + .generate_mod_file(&out_tempdir_path, &output_paths) + .await?; + output_paths.push(mod_path); + + Ok((output_paths, out_tempdir)) + } + + async fn generate_mod_file( + &self, + output_tempdir_path: &Path, + output_paths: &[PathBuf], + ) -> anyhow::Result { + let mod_path = output_tempdir_path.join("mod.rs"); + let mut mod_file = tokio::fs::File::create(&mod_path).await?; + + let mut includes: Vec = Vec::new(); + for generated_file in output_paths { + if let Some(name) = generated_file.file_name() { + let mod_name = generated_file + .file_stem() + .unwrap() + .to_ascii_lowercase() + .to_string_lossy() + .replace(".", "_") + .replace("-", "_"); + + let file_name = name.to_str().unwrap(); + + includes.push(genco::quote! { + pub mod $(mod_name) { + include!($(quoted(file_name))); + } + }); + } + } + + let mod_tokens: genco::lang::rust::Tokens = genco::quote! { + $(for tokens in includes join($['\n']) => $tokens) + }; + let mod_contents = mod_tokens.to_file_string()?; + mod_file.write_all(mod_contents.as_bytes()).await?; + + Ok(mod_path) + } + + async fn copy_rs( + &self, + output_proto_paths: Vec, + output_path: &Path, + root_path: &Path, + ) -> anyhow::Result<()> { + for output_rs in &output_proto_paths { + let rel_proto_path = output_rs.strip_prefix(root_path).map_err(|e| { + anyhow!( + "output: {} does not match root_path: {}", + output_rs.display(), + root_path.display() + ) + })?; + let in_proto_path = output_path.join(rel_proto_path); + if let Some(dir) = in_proto_path.parent() { + if !dir.exists() { + tokio::fs::create_dir_all(dir).await?; + } + } + + tokio::fs::copy(output_rs, &in_proto_path).await?; + } + + Ok(()) + } +} + +impl Default for Codegen { + fn default() -> Self { + Self::new() + } +} + #[cfg(test)] mod tests { + use genco::prelude::*; + use tokio::io::AsyncWriteExt; + #[tokio::test] - async fn test_can_generate_output_rust() { + async fn test_can_generate_output_rust() -> anyhow::Result<()> { + // Generate from protobuf let proto_spec = r#" syntax = "proto3"; +import "includes/test_include.proto"; + package test.can.generate.output.rust; message MyEvent { string name = 1; } "#; + + let proto_include_spec = r#" +syntax = "proto3"; + +package test.can.generate.output.rust.include.test_include; + +message MyInclude { + string name = 1; +} +"#; + + let out_tempdir = tempfile::TempDir::new()?; + let in_tempdir = tempfile::TempDir::new()?; + + let proto_path = in_tempdir.path().join("test.proto"); + let mut proto_file = tokio::fs::File::create(&proto_path).await?; + proto_file.write_all(proto_spec.as_bytes()).await?; + proto_file.sync_all().await?; + + tokio::fs::create_dir_all(in_tempdir.path().join("includes")).await?; + let proto_include_path = in_tempdir.path().join("includes/test_include.proto"); + let mut proto_file = tokio::fs::File::create(&proto_include_path).await?; + proto_file.write_all(proto_include_spec.as_bytes()).await?; + proto_file.sync_all().await?; + + let out_tempdir_path = out_tempdir.into_path(); + let handle = tokio::task::spawn_blocking({ + let out_tempdir_path = out_tempdir_path.clone(); + move || { + prost_build::Config::new() + .out_dir(out_tempdir_path) + .compile_protos(&[proto_path, proto_include_path], &[in_tempdir.into_path()])?; + + Ok(()) + } + }); + + let result: anyhow::Result<()> = handle.await?; + result?; + + let mut entries = tokio::fs::read_dir(&out_tempdir_path).await?; + let mut file_paths = Vec::new(); + while let Some(entry) = entries.next_entry().await? { + if let Some(ext) = entry.path().extension().and_then(|e| e.to_str()) { + if ext == "rs" { + file_paths.push(entry.path()); + } + } + } + + // Generate mod.rs + let mod_path = out_tempdir_path.join("mod.rs"); + let mut mod_file = tokio::fs::File::create(&mod_path).await?; + + let mut includes: Vec = Vec::new(); + for generated_file in &file_paths { + if let Some(name) = generated_file.file_name() { + let mod_name = generated_file + .file_stem() + .unwrap() + .to_ascii_lowercase() + .to_string_lossy() + .replace(".", "_") + .replace("-", "_"); + + let file_name = name.to_str().unwrap(); + + includes.push(genco::quote! { + pub mod $(mod_name) { + include!($(quoted(file_name))); + } + }); + } + } + + let mod_tokens: genco::lang::rust::Tokens = genco::quote! { + $(for tokens in includes join($['\n']) => $tokens) + }; + let mod_contents = mod_tokens.to_file_string()?; + + pretty_assertions::assert_eq!("", mod_contents); + + mod_file.write_all(mod_contents.as_bytes()).await?; + + assert_eq!(1, file_paths.len()); + + Ok(()) } } diff --git a/examples/basic-setup/schemas/crunch/my_event.proto b/examples/basic-setup/schemas/crunch/my_event.proto new file mode 100644 index 0000000..7ee13ad --- /dev/null +++ b/examples/basic-setup/schemas/crunch/my_event.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package test.can.generate.output.rust; + +message MyEvent { + string name = 1; +} diff --git a/examples/basic-setup/src/crunch/mod.rs b/examples/basic-setup/src/crunch/mod.rs new file mode 100644 index 0000000..b8b22b3 --- /dev/null +++ b/examples/basic-setup/src/crunch/mod.rs @@ -0,0 +1 @@ +pub mod test_can_generate_output_rust { include!("test.can.generate.output.rust.rs"); } diff --git a/examples/basic-setup/src/crunch/test.can.generate.output.rust.rs b/examples/basic-setup/src/crunch/test.can.generate.output.rust.rs new file mode 100644 index 0000000..cfd3b73 --- /dev/null +++ b/examples/basic-setup/src/crunch/test.can.generate.output.rust.rs @@ -0,0 +1,5 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MyEvent { + #[prost(string, tag="1")] + pub name: std::string::String, +}