From 6cf1a231696ae9723116baee10a8ef70940df643 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Sun, 18 Feb 2024 13:38:16 +0100 Subject: [PATCH] feat: with actual archive test Signed-off-by: kjuulh --- Cargo.lock | 319 ++++++++++++------ crates/flux-releaser/Cargo.toml | 3 +- .../schemas/proto/flux_releaser.proto | 2 +- crates/flux-releaser/src/api.rs | 11 +- crates/flux-releaser/src/app/infra/aws_s3.rs | 7 +- crates/flux-releaser/src/grpc.rs | 7 +- crates/flux-releaser/src/lib.rs | 15 + crates/flux-releaser/src/main.rs | 15 +- crates/flux-releaser/src/services.rs | 8 +- crates/flux-releaser/src/services/archive.rs | 17 +- .../src/services/domain_events.rs | 7 +- .../src/services/domain_events/extensions.rs | 1 - .../flux-releaser/src/services/file_reader.rs | 47 ++- .../src/services/file_reader/extensions.rs | 1 - .../flux-releaser/src/services/file_store.rs | 35 +- .../src/services/file_store/extensions.rs | 1 - .../src/services/release_manager.rs | 54 --- .../src/services/release_manager/models.rs | 10 + .../flux-releaser/tests/publish_artifacts.rs | 165 ++++++++- 19 files changed, 495 insertions(+), 230 deletions(-) create mode 100644 crates/flux-releaser/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 74c3632..d824b80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,9 +28,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" +checksum = "96b09b5178381e0874812a9b157f7fe84982617e48f71f4e3235482775e5b540" dependencies = [ "anstyle", "anstyle-parse", @@ -87,7 +87,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3" dependencies = [ "concurrent-queue", - "event-listener 5.0.0", + "event-listener 5.1.0", "event-listener-strategy 0.5.0", "futures-core", "pin-project-lite", @@ -184,7 +184,7 @@ dependencies = [ "hex", "http 0.2.11", "hyper 0.14.28", - "ring 0.17.7", + "ring 0.17.8", "time", "tokio", "tracing", @@ -344,7 +344,7 @@ dependencies = [ "once_cell", "p256", "percent-encoding", - "ring 0.17.7", + "ring 0.17.8", "sha2", "subtle", "time", @@ -994,12 +994,6 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" -[[package]] -name = "downcast" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" - [[package]] name = "ecdsa" version = "0.14.8" @@ -1060,6 +1054,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -1089,9 +1092,9 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b72557800024fabbaa2449dd4bf24e37b93702d457a4d4f2b0dd1f0f039f20c1" +checksum = "b7ad6fd685ce13acd6d9541a30f6db6567a7a24c9ffd4ba2955d29e3f22c8b27" dependencies = [ "concurrent-queue", "parking", @@ -1114,7 +1117,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "feedafcaa9b749175d5ac357452a9d41ea2911da598fde46ce1fe02c37751291" dependencies = [ - "event-listener 5.0.0", + "event-listener 5.1.0", "pin-project-lite", ] @@ -1179,10 +1182,9 @@ dependencies = [ "clap", "dotenv", "lazy_static", - "mockall", - "mockall_double", "nats", "prost", + "reqwest", "serde", "serde_json", "tar", @@ -1201,6 +1203,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1210,12 +1227,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fragile" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" - [[package]] name = "futures-channel" version = "0.3.30" @@ -1534,6 +1545,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.28", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "hyper-util" version = "0.1.3" @@ -1589,6 +1613,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "itertools" version = "0.11.0" @@ -1701,51 +1731,30 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "mockall" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43766c2b5203b10de348ffe19f7e54564b64f3d6018ff7648d1e2d6d3a0f0a48" -dependencies = [ - "cfg-if", - "downcast", - "fragile", - "lazy_static", - "mockall_derive", - "predicates", - "predicates-tree", -] - -[[package]] -name = "mockall_derive" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cbce79ec385a1d4f54baa90a76401eb15d9cab93685f62e7e9f942aa00ae2" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "mockall_double" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1ca96e5ac35256ae3e13536edd39b172b88f41615e1d7b653c8ad24524113e8" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "multimap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nats" version = "0.24.1" @@ -1867,12 +1876,50 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl" +version = "0.10.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" +dependencies = [ + "bitflags 2.4.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "outref" version = "0.5.1" @@ -2013,6 +2060,12 @@ dependencies = [ "spki 0.7.3", ] +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + [[package]] name = "platforms" version = "3.3.0" @@ -2031,32 +2084,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "predicates" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" -dependencies = [ - "anstyle", - "predicates-core", -] - -[[package]] -name = "predicates-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" - -[[package]] -name = "predicates-tree" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" -dependencies = [ - "predicates-core", - "termtree", -] - [[package]] name = "prettyplease" version = "0.2.16" @@ -2213,6 +2240,46 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "reqwest" +version = "0.11.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.24", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rfc6979" version = "0.3.1" @@ -2241,16 +2308,17 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", "getrandom", "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2288,7 +2356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.17.7", + "ring 0.17.8", "rustls-webpki 0.101.7", "sct", ] @@ -2330,7 +2398,7 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -2376,7 +2444,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -2654,6 +2722,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tar" version = "0.4.40" @@ -2677,12 +2766,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "termtree" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" - [[package]] name = "thread_local" version = "1.1.7" @@ -2779,6 +2862,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -3028,6 +3121,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -3090,6 +3189,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.91" @@ -3304,6 +3415,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "xattr" version = "1.3.1" diff --git a/crates/flux-releaser/Cargo.toml b/crates/flux-releaser/Cargo.toml index 1672848..2543236 100644 --- a/crates/flux-releaser/Cargo.toml +++ b/crates/flux-releaser/Cargo.toml @@ -15,7 +15,6 @@ prost = "0.12.3" tonic = "0.11.0" uuid = { version = "1.7.0", features = ["v7", "v4"] } async-trait = "0.1.77" -mockall_double = "0.3.1" aws-config = { version = "1.1.5", features = ["behavior-version-latest"] } aws-sdk-s3 = { version = "1.15.0", features = ["behavior-version-latest"] } serde = { version = "1.0.196", features = ["derive"] } @@ -29,4 +28,4 @@ tonic-build = "0.11.0" [dev-dependencies] lazy_static = "1.4.0" -mockall = "0.12.1" +reqwest = "0.11.24" diff --git a/crates/flux-releaser/schemas/proto/flux_releaser.proto b/crates/flux-releaser/schemas/proto/flux_releaser.proto index c79d931..570f596 100644 --- a/crates/flux-releaser/schemas/proto/flux_releaser.proto +++ b/crates/flux-releaser/schemas/proto/flux_releaser.proto @@ -13,5 +13,5 @@ message HelloRequest { } message HelloReply { - string message = 1; + string artifact_id = 1; } diff --git a/crates/flux-releaser/src/api.rs b/crates/flux-releaser/src/api.rs index 3991ac6..4f23d0e 100644 --- a/crates/flux-releaser/src/api.rs +++ b/crates/flux-releaser/src/api.rs @@ -1,11 +1,14 @@ use std::net::SocketAddr; -use axum::{routing::get, Router}; +use axum::{response::IntoResponse, routing::get, Router}; use crate::app::SharedApp; pub async fn axum_serve(host: SocketAddr, app: SharedApp) -> anyhow::Result<()> { - let app = Router::new().route("/", get(root)).with_state(app); + let app = Router::new() + .route("/ping", get(pong)) + .route("/", get(root)) + .with_state(app); tracing::info!("listening on {}", host); let listener = tokio::net::TcpListener::bind(host).await.unwrap(); @@ -15,6 +18,10 @@ pub async fn axum_serve(host: SocketAddr, app: SharedApp) -> anyhow::Result<()> Ok(()) } +async fn pong() -> impl IntoResponse { + "pong!" +} + async fn root() -> &'static str { "Hello, flux-releaser!" } diff --git a/crates/flux-releaser/src/app/infra/aws_s3.rs b/crates/flux-releaser/src/app/infra/aws_s3.rs index 2a865a1..e72be74 100644 --- a/crates/flux-releaser/src/app/infra/aws_s3.rs +++ b/crates/flux-releaser/src/app/infra/aws_s3.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use aws_config::{BehaviorVersion, Region}; use aws_sdk_s3::config::Credentials; @@ -5,15 +6,15 @@ pub async fn s3_client() -> anyhow::Result { let shared_config = aws_config::defaults(BehaviorVersion::latest()) .region(Region::new("eu-west-1")) .credentials_provider(Credentials::new( - std::env::var("AWS_ACCESS_KEY_ID")?, - std::env::var("AWS_SECRET_ACCESS_KEY")?, + std::env::var("AWS_ACCESS_KEY_ID").context("AWS_ACCESS_KEY_ID was not set")?, + std::env::var("AWS_SECRET_ACCESS_KEY").context("AWS_SECRET_ACCESS_KEY was not set")?, None, None, "flux_releaser", )); let config = aws_sdk_s3::config::Builder::from(&shared_config.load().await) - .endpoint_url(std::env::var("AWS_ENDPOINT_URL")?) + .endpoint_url(std::env::var("AWS_ENDPOINT_URL").context("AWS_ENDPOINT_URL was not set")?) .build(); let client = aws_sdk_s3::Client::from_conf(config); diff --git a/crates/flux-releaser/src/grpc.rs b/crates/flux-releaser/src/grpc.rs index 06ad02d..2fedad3 100644 --- a/crates/flux-releaser/src/grpc.rs +++ b/crates/flux-releaser/src/grpc.rs @@ -11,7 +11,7 @@ use crate::{ use self::gen::{greeter_server, HelloReply, HelloRequest}; -mod gen { +pub mod gen { tonic::include_proto!("flux_releaser"); } @@ -34,7 +34,8 @@ impl greeter_server::Greeter for FluxReleaserGrpc { request: tonic::Request, ) -> std::result::Result, tonic::Status> { let req = request.into_inner(); - self.release_manager + let artifact = self + .release_manager .commit_artifact( req.try_into() .map_err(|e: anyhow::Error| tonic::Status::invalid_argument(e.to_string()))?, @@ -43,7 +44,7 @@ impl greeter_server::Greeter for FluxReleaserGrpc { .unwrap(); Ok(tonic::Response::new(HelloReply { - message: "something".into(), + artifact_id: artifact.to_string(), })) } } diff --git a/crates/flux-releaser/src/lib.rs b/crates/flux-releaser/src/lib.rs new file mode 100644 index 0000000..c70334f --- /dev/null +++ b/crates/flux-releaser/src/lib.rs @@ -0,0 +1,15 @@ +use cli::Command; + +pub mod api; +pub mod app; +pub mod cli; +pub mod grpc; +pub mod services; + +pub async fn run() -> anyhow::Result<()> { + dotenv::dotenv().ok(); + + Command::run().await?; + + Ok(()) +} diff --git a/crates/flux-releaser/src/main.rs b/crates/flux-releaser/src/main.rs index fe619bd..b57db4b 100644 --- a/crates/flux-releaser/src/main.rs +++ b/crates/flux-releaser/src/main.rs @@ -1,19 +1,6 @@ -use cli::Command; - -mod cli; - -mod api; -mod grpc; - -mod app; - -mod services; - #[tokio::main] async fn main() -> anyhow::Result<()> { - dotenv::dotenv().ok(); - - Command::run().await?; + flux_releaser::run().await?; Ok(()) } diff --git a/crates/flux-releaser/src/services.rs b/crates/flux-releaser/src/services.rs index 267f293..ea1b36e 100644 --- a/crates/flux-releaser/src/services.rs +++ b/crates/flux-releaser/src/services.rs @@ -1,5 +1,5 @@ -mod archive; -mod domain_events; -mod file_reader; -mod file_store; +pub mod archive; +pub mod domain_events; +pub mod file_reader; +pub mod file_store; pub mod release_manager; diff --git a/crates/flux-releaser/src/services/archive.rs b/crates/flux-releaser/src/services/archive.rs index 30dd34e..cd50691 100644 --- a/crates/flux-releaser/src/services/archive.rs +++ b/crates/flux-releaser/src/services/archive.rs @@ -13,10 +13,8 @@ pub mod extensions { use crate::{app::SharedApp, services::release_manager::models::ArtifactID}; - #[mockall_double::double] use crate::services::file_store::FileStore; - #[mockall_double::double] use super::Archive; use super::ArchiveFile; @@ -92,19 +90,15 @@ pub mod extensions { } } -#[cfg(test)] -use mockall::{automock, mock, predicate::*}; - use super::file_reader::{File, Files}; -#[cfg_attr(test, automock)] impl Archive { pub fn new() -> Self { Self {} } pub async fn create_archive(&self, files: Files) -> anyhow::Result { - tracing::trace!("archiving files"); + tracing::trace!("archiving files: {}", files.len()); let buffer = Vec::new(); let cursor = Cursor::new(buffer); @@ -114,9 +108,14 @@ impl Archive { for file in files { let abs_file_path = file.path; - tracing::trace!("archiving file: {}", abs_file_path.display()); let mut fd = std::fs::File::open(&abs_file_path)?; - tar_builder.append_file(&abs_file_path, &mut fd)?; + if let Some(rel) = file.relative { + tracing::trace!("archiving rel file: {}", rel.display()); + tar_builder.append_file(&rel, &mut fd)?; + } else { + tracing::trace!("archiving file: {}", abs_file_path.display()); + tar_builder.append_file(&abs_file_path, &mut fd)?; + } } tar_builder.finish()?; diff --git a/crates/flux-releaser/src/services/domain_events.rs b/crates/flux-releaser/src/services/domain_events.rs index c264f1e..db19535 100644 --- a/crates/flux-releaser/src/services/domain_events.rs +++ b/crates/flux-releaser/src/services/domain_events.rs @@ -5,12 +5,7 @@ pub struct DomainEvents { nats: Nats, } -#[cfg(test)] -use mockall::{automock, mock, predicate::*}; - -use crate::app::infra::{nats::Nats}; - -#[cfg_attr(test, automock)] +use crate::app::infra::nats::Nats; impl DomainEvents { pub fn new(nats: Nats) -> Self { Self { nats } diff --git a/crates/flux-releaser/src/services/domain_events/extensions.rs b/crates/flux-releaser/src/services/domain_events/extensions.rs index 4041a03..473ff74 100644 --- a/crates/flux-releaser/src/services/domain_events/extensions.rs +++ b/crates/flux-releaser/src/services/domain_events/extensions.rs @@ -1,6 +1,5 @@ use crate::app::SharedApp; -#[mockall_double::double] use super::DomainEvents; pub trait DomainEventsExt { diff --git a/crates/flux-releaser/src/services/file_reader.rs b/crates/flux-releaser/src/services/file_reader.rs index 6f4bfff..0614896 100644 --- a/crates/flux-releaser/src/services/file_reader.rs +++ b/crates/flux-releaser/src/services/file_reader.rs @@ -1,15 +1,15 @@ #[derive(Clone)] pub struct FileReader {} -use std::{collections::BTreeMap, path::PathBuf}; +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, +}; pub mod extensions; use anyhow::anyhow; -#[cfg(test)] -use mockall::{automock, mock, predicate::*}; -#[cfg_attr(test, automock)] impl FileReader { pub fn new() -> Self { Self {} @@ -49,7 +49,11 @@ impl FileReader { cluster_name ); - files.push(file.into_path().into()) + if file.path().is_absolute() { + files.push((file.path(), file.path().strip_prefix(&location)?).into()) + } else { + files.push(file.into_path().into()) + } } } @@ -60,11 +64,32 @@ impl FileReader { #[derive(Debug, Clone)] pub struct File { pub path: PathBuf, + pub relative: Option, } impl From for File { fn from(value: PathBuf) -> Self { - Self { path: value } + Self { + path: value, + relative: None, + } + } +} + +impl From<(PathBuf, PathBuf)> for File { + fn from(value: (PathBuf, PathBuf)) -> Self { + Self { + path: value.0, + relative: Some(value.1), + } + } +} +impl From<(&Path, &Path)> for File { + fn from(value: (&Path, &Path)) -> Self { + Self { + path: value.0.to_path_buf(), + relative: Some(value.1.to_path_buf()), + } } } @@ -92,14 +117,8 @@ impl From for Vec { value .iter() .map(|(cluster_name, files)| (PathBuf::from(cluster_name), files)) - .flat_map(|(cluster_name, files)| { - files - .iter() - //.map(|file_path| cluster_name.join(&file_path.path)) - .map(|file_path| file_path.path.clone()) - .collect::>() - }) - .map(|f| f.into()) + .flat_map(|(_cluster_name, files)| files.to_vec()) + // .map(|f| f.into()) .collect::>() } } diff --git a/crates/flux-releaser/src/services/file_reader/extensions.rs b/crates/flux-releaser/src/services/file_reader/extensions.rs index d3a1192..7052b6a 100644 --- a/crates/flux-releaser/src/services/file_reader/extensions.rs +++ b/crates/flux-releaser/src/services/file_reader/extensions.rs @@ -1,6 +1,5 @@ use crate::app::SharedApp; -#[mockall_double::double] use super::FileReader; pub trait FileReaderExt { diff --git a/crates/flux-releaser/src/services/file_store.rs b/crates/flux-releaser/src/services/file_store.rs index 68ece0f..b9e6e2b 100644 --- a/crates/flux-releaser/src/services/file_store.rs +++ b/crates/flux-releaser/src/services/file_store.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::{env::temp_dir, path::PathBuf}; use super::release_manager::models::ArtifactID; @@ -10,11 +10,8 @@ pub struct FileStore { } use aws_sdk_s3::primitives::ByteStream; -#[cfg(test)] -use mockall::{automock, mock, predicate::*}; -use tokio::io::AsyncReadExt; +use tokio::io::BufReader; -#[cfg_attr(test, automock)] impl FileStore { pub fn new(client: aws_sdk_s3::Client) -> Self { Self { client } @@ -33,4 +30,32 @@ impl FileStore { Ok(()) } + + pub async fn get_archive(&self, artifact_id: ArtifactID) -> anyhow::Result { + tracing::trace!("getting archive: {}", artifact_id.to_string()); + + let archive_name = format!("archives/{}.tar", &artifact_id.to_string()); + + let obj = self + .client + .get_object() + .bucket("mybucket") + .key(&archive_name) + .send() + .await?; + + let archive_path = temp_dir() + .join("flux_releaser") + .join("cache") + .join(&archive_name); + tokio::fs::create_dir_all(archive_path.parent().unwrap()).await?; + let mut archive_file = tokio::fs::File::create(&archive_path).await?; + let mut buf_reader = BufReader::new(obj.body.into_async_read()); + + tokio::io::copy(&mut buf_reader, &mut archive_file).await?; + + tracing::debug!("created archive: {}", archive_path.display()); + + Ok(archive_path) + } } diff --git a/crates/flux-releaser/src/services/file_store/extensions.rs b/crates/flux-releaser/src/services/file_store/extensions.rs index a2ced2f..c32be14 100644 --- a/crates/flux-releaser/src/services/file_store/extensions.rs +++ b/crates/flux-releaser/src/services/file_store/extensions.rs @@ -1,6 +1,5 @@ use crate::app::SharedApp; -#[mockall_double::double] use super::FileStore; pub trait FileStoreExt { diff --git a/crates/flux-releaser/src/services/release_manager.rs b/crates/flux-releaser/src/services/release_manager.rs index ec33773..1a3894b 100644 --- a/crates/flux-releaser/src/services/release_manager.rs +++ b/crates/flux-releaser/src/services/release_manager.rs @@ -1,14 +1,10 @@ use serde::Serialize; use crate::services::archive::extensions::ArchiveUploadExt; -#[mockall_double::double] use crate::services::file_store::FileStore; -#[mockall_double::double] use super::archive::Archive; -#[mockall_double::double] use super::domain_events::DomainEvents; -#[mockall_double::double] use super::file_reader::FileReader; use self::models::{ArtifactID, CommitArtifact}; @@ -64,53 +60,3 @@ pub struct CommittedArtifactEvent { pub mod extensions; pub mod models; - -#[cfg(test)] -mod test { - use crate::services::archive::{ArchiveFile, MockArchive}; - use crate::services::domain_events::MockDomainEvents; - use crate::services::file_reader::{Files, MockFileReader}; - use crate::services::file_store::MockFileStore; - - use super::*; - - #[tokio::test] - async fn generated_artifact_id() -> anyhow::Result<()> { - let mut file_store = MockFileStore::default(); - file_store - .expect_upload_file() - .times(1) - .returning(|_, _| Ok(())); - - let mut domain_events = MockDomainEvents::default(); - domain_events - .expect_publish_event() - .times(1) - .returning(|_| Ok(())); - - let mut file_reader = MockFileReader::default(); - file_reader - .expect_read_files() - .times(1) - .returning(|_| Ok(Files::default())); - - let mut archive = MockArchive::default(); - archive.expect_create_archive().times(1).returning(|_| { - Ok(ArchiveFile { - content: Vec::new(), - }) - }); - - let releaser_manager = ReleaseManager::new(file_reader, file_store, archive, domain_events); - - releaser_manager - .commit_artifact(CommitArtifact { - app: "app".into(), - branch: "branch".into(), - folder: "someFolder".into(), - }) - .await?; - - Ok(()) - } -} diff --git a/crates/flux-releaser/src/services/release_manager/models.rs b/crates/flux-releaser/src/services/release_manager/models.rs index 83c25fa..689355e 100644 --- a/crates/flux-releaser/src/services/release_manager/models.rs +++ b/crates/flux-releaser/src/services/release_manager/models.rs @@ -23,3 +23,13 @@ impl std::ops::Deref for ArtifactID { &self.0 } } + +impl TryFrom for ArtifactID { + type Error = anyhow::Error; + + fn try_from(value: String) -> Result { + let uuid = uuid::Uuid::parse_str(&value)?; + + Ok(ArtifactID(uuid)) + } +} diff --git a/crates/flux-releaser/tests/publish_artifacts.rs b/crates/flux-releaser/tests/publish_artifacts.rs index 51fdf81..3f119ab 100644 --- a/crates/flux-releaser/tests/publish_artifacts.rs +++ b/crates/flux-releaser/tests/publish_artifacts.rs @@ -1,48 +1,191 @@ -struct Server {} +use std::{ + env::temp_dir, + net::{Ipv4Addr, SocketAddr}, + time::Duration, +}; + +use anyhow::Context; +use flux_releaser::{ + app::SharedApp, grpc::gen::HelloRequest, services::file_store::extensions::FileStoreExt, +}; +use tokio::{net::TcpListener, runtime::Runtime, time::sleep}; +use tonic::transport::Channel; +use uuid::Uuid; + +struct Server { + endpoints: Endpoints, + app: SharedApp, +} + +#[derive(Clone, Debug)] +struct Endpoints { + http: SocketAddr, + grpc: SocketAddr, +} impl Server { - pub async fn new() -> Self { - Self {} + pub async fn new() -> anyhow::Result { + let http_socket = Self::find_free_port().await?; + let grpc_socket = Self::find_free_port().await?; + + Ok(Self { + endpoints: Endpoints { + http: http_socket, + grpc: grpc_socket, + }, + app: SharedApp::new(flux_releaser::app::App::new().await?), + }) } pub async fn start(&self) -> anyhow::Result<()> { + flux_releaser::cli::server::run_server(self.endpoints.http, self.endpoints.grpc).await?; + Ok(()) } + + pub async fn find_free_port() -> anyhow::Result { + let socket = SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 0); + + let listener = TcpListener::bind(socket).await?; + + listener.local_addr().context("failed to get local addr") + } } static INIT: std::sync::Once = std::sync::Once::new(); +async fn perform_task_with_backoff( + mut task: F, + max_retries: u32, + base_delay_ms: u64, +) -> Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut retries = 0; + let mut delay = base_delay_ms; + + loop { + match task().await { + Ok(result) => return Ok(result), + Err(e) if retries < max_retries => { + sleep(Duration::from_millis(delay)).await; + delay *= 2; // Exponential backoff + retries += 1; + } + Err(e) => return Err(e), + } + } +} + // Makes sure the setup is ready for execution async fn is_ready() -> anyhow::Result<()> { tokio::time::sleep(std::time::Duration::from_secs(1)).await; + perform_task_with_backoff( + || async { + let endpoints = unsafe { + if ENDPOINTS.is_none() { + anyhow::bail!("endpoints not set yet"); + } + + ENDPOINTS.clone().unwrap() + }; + + let resp = reqwest::get(format!("http://{}/ping", endpoints.http)).await?; + + if !resp.status().is_success() { + anyhow::bail!("failed with status: {}", resp.status()); + } + + Ok::<(), anyhow::Error>(()) + }, + 5, + 500, + ) + .await?; + Ok(()) } -async fn setup() -> anyhow::Result<()> { +static mut ENDPOINTS: Option = None; +static mut APP: Option = None; + +async fn setup() -> anyhow::Result<(Endpoints, SharedApp)> { INIT.call_once(|| { - tokio::spawn(async move { - println!("once was created once"); - Server::new().await.start().await.unwrap(); + std::thread::spawn(|| { + let rt = Runtime::new().unwrap(); + rt.block_on(async move { + println!("once was created once"); + let server = Server::new().await.unwrap(); + + unsafe { + ENDPOINTS = Some(server.endpoints.clone()); + APP = Some(server.app.clone()); + } + + server.start().await.unwrap(); + }) }); }); is_ready().await?; - Ok(()) + Ok(unsafe { (ENDPOINTS.clone().unwrap(), APP.clone().unwrap()) }) } #[tokio::test] async fn can_create_artifact() -> anyhow::Result<()> { setup().await?; - anyhow::bail!("failed one"); + Ok(()) } #[tokio::test] async fn can_more_create_artifact() -> anyhow::Result<()> { - setup().await?; + std::env::set_var("RUST_LOG", "flux_releaser=trace"); + + let (endpoints, app) = setup().await?; + + let mut client = flux_releaser::grpc::gen::greeter_client::GreeterClient::connect(format!( + "http://{}", + endpoints.grpc + )) + .await?; + + let test_id = Uuid::new_v4(); + + let temp = temp_dir() + .join("flux_releaser") + .join("tests") + .join(test_id.to_string()); + + let file_path = temp + .join("clusters") + .join(Uuid::new_v4().to_string()) + .join("some-file.yaml"); + tokio::fs::create_dir_all(file_path.parent().unwrap()).await?; + let _ = tokio::fs::File::create(file_path).await?; + + let resp = client + .say_hello(HelloRequest { + app: "some-app".into(), + branch: "some-branch".into(), + folder: temp.to_string_lossy().to_string(), + }) + .await?; + + let artifact = resp.into_inner(); + + let archive = app + .file_store() + .get_archive(artifact.artifact_id.try_into()?) + .await?; + + assert!(archive.exists()); - anyhow::bail!("failed two"); Ok(()) } + +pub struct TestGreeter {}