From 41f36c09beaa520882794ecc38fca4040ce30042 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Tue, 19 Sep 2023 08:31:13 +0200 Subject: [PATCH] feat: with protobuf Signed-off-by: kjuulh --- Cargo.lock | 337 ++++++++++++++++-- crates/crunch-envelope/Cargo.toml | 11 +- .../benches/envelope_benchmark.rs | 9 + crates/crunch-envelope/build.rs | 2 + crates/crunch-envelope/src/envelope.proto | 15 + crates/crunch-envelope/src/lib.rs | 13 + crates/crunch-envelope/src/proto_envelope.rs | 30 ++ crates/crunch/src/lib.rs | 1 + 8 files changed, 387 insertions(+), 31 deletions(-) create mode 100644 crates/crunch-envelope/src/envelope.proto create mode 100644 crates/crunch-envelope/src/proto_envelope.rs diff --git a/Cargo.lock b/Cargo.lock index 3a5b15b..95d3c06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,9 +44,9 @@ version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 2.0.35", ] [[package]] @@ -93,12 +93,34 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + [[package]] name = "bumpalo" version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + [[package]] name = "bytes" version = "1.5.0" @@ -174,7 +196,7 @@ version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ - "bitflags", + "bitflags 1.3.2", "clap_lex", "indexmap", "textwrap", @@ -201,7 +223,7 @@ dependencies = [ "ciborium", "clap", "criterion-plot", - "itertools", + "itertools 0.10.5", "lazy_static", "num-traits", "oorandom", @@ -222,7 +244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -285,9 +307,13 @@ name = "crunch-envelope" version = "0.1.0" dependencies = [ "base64", + "bytes 0.4.12", "capnp", "capnpc", "criterion", + "prost 0.12.1", + "prost-build", + "prost-types 0.12.1", "serde", "serde_json", "thiserror", @@ -299,6 +325,48 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "errno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", +] + +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + +[[package]] +name = "fixedbitset" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" + [[package]] name = "gimli" version = "0.28.0" @@ -317,6 +385,15 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -342,6 +419,24 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.10.5" @@ -378,6 +473,12 @@ version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +[[package]] +name = "linux-raw-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" + [[package]] name = "lock_api" version = "0.4.10" @@ -429,6 +530,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "multimap" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04b9f127583ed176e163fb9ec6f3e793b87e21deedd5734a69386a18a0151" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -514,6 +621,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "petgraph" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" +dependencies = [ + "fixedbitset", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -548,6 +664,15 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid", +] + [[package]] name = "proc-macro2" version = "1.0.67" @@ -557,13 +682,106 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d14b1c185652833d24aaad41c5832b0be5616a590227c1fbff57c616754b23" +dependencies = [ + "byteorder", + "bytes 0.4.12", + "prost-derive 0.5.0", +] + +[[package]] +name = "prost" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +dependencies = [ + "bytes 1.5.0", + "prost-derive 0.12.1", +] + +[[package]] +name = "prost-build" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb788126ea840817128183f8f603dce02cb7aea25c2a0b764359d8e20010702e" +dependencies = [ + "bytes 0.4.12", + "heck", + "itertools 0.8.2", + "log", + "multimap", + "petgraph", + "prost 0.5.0", + "prost-types 0.5.0", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7dc378b94ac374644181a2247cebf59a6ec1c88b49ac77f3a94b86b79d0e11" +dependencies = [ + "failure", + "itertools 0.8.2", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + +[[package]] +name = "prost-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 2.0.35", +] + +[[package]] +name = "prost-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de482a366941c8d56d19b650fac09ca08508f2a696119ee7513ad590c8bac6f" +dependencies = [ + "bytes 0.4.12", + "prost 0.5.0", +] + +[[package]] +name = "prost-types" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" +dependencies = [ + "prost 0.12.1", +] + +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.67", ] [[package]] @@ -594,7 +812,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -632,6 +850,19 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustix" +version = "0.38.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" +dependencies = [ + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "ryu" version = "1.0.15" @@ -668,9 +899,9 @@ version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 2.0.35", ] [[package]] @@ -718,17 +949,41 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid", +] + [[package]] name = "syn" version = "2.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59bf04c28bee9043ed9ea1e41afc0552288d3aba9c6efdd78903b802926f4879" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.67", + "quote 1.0.33", "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + [[package]] name = "test-bin" version = "0.1.0" @@ -757,9 +1012,9 @@ version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 2.0.35", ] [[package]] @@ -789,7 +1044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", - "bytes", + "bytes 1.5.0", "libc", "mio", "num_cpus", @@ -807,9 +1062,9 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 2.0.35", ] [[package]] @@ -831,9 +1086,9 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 2.0.35", ] [[package]] @@ -877,6 +1132,18 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "valuable" version = "0.1.0" @@ -918,9 +1185,9 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 2.0.35", "wasm-bindgen-shared", ] @@ -930,7 +1197,7 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ - "quote", + "quote 1.0.33", "wasm-bindgen-macro-support", ] @@ -940,9 +1207,9 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 2.0.35", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -963,6 +1230,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164" +dependencies = [ + "failure", + "libc", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/crates/crunch-envelope/Cargo.toml b/crates/crunch-envelope/Cargo.toml index 4ba9c6f..4b7efcc 100644 --- a/crates/crunch-envelope/Cargo.toml +++ b/crates/crunch-envelope/Cargo.toml @@ -13,18 +13,27 @@ harness = false debug = true [features] -default = ["json"] +default = ["json", "proto"] json = ["dep:serde", "dep:serde_json", "dep:base64"] +proto = ["dep:prost", "dep:prost-types", "dep:bytes"] [dependencies] capnp = "0.17.2" thiserror.workspace = true + +# Json serde = { version = "1.0.188" ,optional = true, features = ["derive"] } serde_json = {version = "1.0.107",optional = true} base64 = {version = "0.21.4",optional = true} +# Proto +prost = {version = "0.12", optional = true} +prost-types = {version = "0.12", optional = true} +bytes = {version = "0.4", optional = true} + [build-dependencies] capnpc = "0.17.2" +prost-build = "0.5" [dev-dependencies] criterion = { version = "0.4" } \ No newline at end of file diff --git a/crates/crunch-envelope/benches/envelope_benchmark.rs b/crates/crunch-envelope/benches/envelope_benchmark.rs index 04297f2..ea77e30 100644 --- a/crates/crunch-envelope/benches/envelope_benchmark.rs +++ b/crates/crunch-envelope/benches/envelope_benchmark.rs @@ -13,6 +13,12 @@ fn envelope_json_benchmark(content: &[u8]) -> () { let _ = crunch_envelope::json::unwrap(&out).expect("to be able to unwrap capnp message"); } +fn envelope_proto_benchmark(content: &[u8]) -> () { + let out = crunch_envelope::proto::wrap("some-domain", "some-entity", content); + + let _ = crunch_envelope::proto::unwrap(&out).expect("to be able to unwrap capnp message"); +} + fn criterion_benchmark(c: &mut Criterion) { let large_content: [u8; 10000] = [0; 10000]; @@ -22,6 +28,9 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("envelope::json", |b| { b.iter(|| envelope_json_benchmark(&large_content)) }); + c.bench_function("envelope::proto", |b| { + b.iter(|| envelope_proto_benchmark(&large_content)) + }); } criterion_group!(benches, criterion_benchmark); diff --git a/crates/crunch-envelope/build.rs b/crates/crunch-envelope/build.rs index 412f854..985d540 100644 --- a/crates/crunch-envelope/build.rs +++ b/crates/crunch-envelope/build.rs @@ -7,4 +7,6 @@ fn main() { .file("schemas/envelope.capnp") .run() .unwrap(); + + prost_build::compile_protos(&["src/envelope.proto"], &["src/"]).unwrap(); } diff --git a/crates/crunch-envelope/src/envelope.proto b/crates/crunch-envelope/src/envelope.proto new file mode 100644 index 0000000..f7d01cf --- /dev/null +++ b/crates/crunch-envelope/src/envelope.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package crunch.envelope; + +message Envelope { + Metadata metadata = 1; + bytes content = 2; +} + +message Metadata { + string domain = 1; + string entity = 2; + uint64 timestamp = 3; + uint64 sequence = 4; +} diff --git a/crates/crunch-envelope/src/lib.rs b/crates/crunch-envelope/src/lib.rs index 77cae58..9d51c75 100644 --- a/crates/crunch-envelope/src/lib.rs +++ b/crates/crunch-envelope/src/lib.rs @@ -3,11 +3,19 @@ mod envelope_capnp; #[cfg(feature = "json")] mod json_envelope; +#[cfg(feature = "proto")] +mod proto_envelope; + #[cfg(feature = "json")] pub mod json { pub use crate::json_envelope::*; } +#[cfg(feature = "proto")] +pub mod proto { + pub use crate::proto_envelope::*; +} + use capnp::message::{Builder, ReaderOptions}; use capnp::serialize; use thiserror::Error; @@ -22,6 +30,11 @@ pub enum EnvelopeError { #[cfg(feature = "json")] #[error("base64 failed to serialize or deserialize code")] Base64Error(#[source] base64::DecodeError), + #[cfg(feature = "proto")] + #[error("prost failed to serialize or deserialize code")] + ProtoError(#[source] prost::DecodeError), + #[error("metadata is missing from field")] + MetadataError(), } pub fn wrap<'a>(domain: &'a str, entity: &'a str, content: &'a [u8]) -> Vec { diff --git a/crates/crunch-envelope/src/proto_envelope.rs b/crates/crunch-envelope/src/proto_envelope.rs new file mode 100644 index 0000000..c49a610 --- /dev/null +++ b/crates/crunch-envelope/src/proto_envelope.rs @@ -0,0 +1,30 @@ +pub mod envelope { + include!(concat!(env!("OUT_DIR"), "/crunch.envelope.rs")); +} + +use prost::Message; + +use crate::EnvelopeError; + +pub fn wrap<'a>(domain: &'a str, entity: &'a str, content: &'a [u8]) -> Vec { + let out = envelope::Envelope { + metadata: Some(envelope::Metadata { + domain: domain.to_string(), + entity: entity.to_string(), + timestamp: 0, + sequence: 0, + }), + content: content.to_vec(), + }; + + out.encode_to_vec() +} + +pub fn unwrap<'a>(message: &'a [u8]) -> Result<(Vec, envelope::Metadata), EnvelopeError> { + let out = envelope::Envelope::decode(message).map_err(EnvelopeError::ProtoError)?; + + Ok(( + out.content, + out.metadata.ok_or(EnvelopeError::MetadataError())?, + )) +} diff --git a/crates/crunch/src/lib.rs b/crates/crunch/src/lib.rs index f748220..37bf3b6 100644 --- a/crates/crunch/src/lib.rs +++ b/crates/crunch/src/lib.rs @@ -48,6 +48,7 @@ mod impls { use crate::{traits, EventInfo}; pub struct InMemoryPersistence {} + #[async_trait] impl traits::Persistence for InMemoryPersistence { async fn insert(&self, event_info: &EventInfo, content: Vec) -> anyhow::Result<()> {