feat: with protobuf

Signed-off-by: kjuulh <contact@kjuulh.io>

Cargo.lock

@@ -44,9 +44,9 @@ version = "0.1.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.67",
- "quote",
+ "quote 1.0.33",
- "syn",
+ "syn 2.0.35",
 ]
 
 [[package]]
@@ -93,12 +93,34 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitflags"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
+
 [[package]]
 name = "bumpalo"
 version = "3.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
 
+[[package]]
+name = "byteorder"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+
+[[package]]
+name = "bytes"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
+dependencies = [
+ "byteorder",
+ "iovec",
+]
+
 [[package]]
 name = "bytes"
 version = "1.5.0"
@@ -174,7 +196,7 @@ version = "3.2.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "clap_lex",
  "indexmap",
  "textwrap",
@@ -201,7 +223,7 @@ dependencies = [
  "ciborium",
  "clap",
  "criterion-plot",
- "itertools",
+ "itertools 0.10.5",
  "lazy_static",
  "num-traits",
  "oorandom",
@@ -222,7 +244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
 dependencies = [
  "cast",
- "itertools",
+ "itertools 0.10.5",
 ]
 
 [[package]]
@@ -285,9 +307,13 @@ name = "crunch-envelope"
 version = "0.1.0"
 dependencies = [
  "base64",
+ "bytes 0.4.12",
  "capnp",
  "capnpc",
  "criterion",
+ "prost 0.12.1",
+ "prost-build",
+ "prost-types 0.12.1",
  "serde",
  "serde_json",
  "thiserror",
@@ -299,6 +325,48 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
+[[package]]
+name = "errno"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "failure"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
+dependencies = [
+ "backtrace",
+]
+
+[[package]]
+name = "fastrand"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
+
+[[package]]
+name = "fixedbitset"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33"
+
 [[package]]
 name = "gimli"
 version = "0.28.0"
@@ -317,6 +385,15 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
+[[package]]
+name = "heck"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
+dependencies = [
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.19"
@@ -342,6 +419,24 @@ dependencies = [
  "hashbrown",
 ]
 
+[[package]]
+name = "iovec"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "itertools"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -378,6 +473,12 @@ version = "0.2.148"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128"
+
 [[package]]
 name = "lock_api"
 version = "0.4.10"
@@ -429,6 +530,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "multimap"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2eb04b9f127583ed176e163fb9ec6f3e793b87e21deedd5734a69386a18a0151"
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -514,6 +621,15 @@ dependencies = [
  "windows-targets",
 ]
 
+[[package]]
+name = "petgraph"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f"
+dependencies = [
+ "fixedbitset",
+]
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.13"
@@ -548,6 +664,15 @@ dependencies = [
  "plotters-backend",
 ]
 
+[[package]]
+name = "proc-macro2"
+version = "0.4.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
+dependencies = [
+ "unicode-xid",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.67"
@@ -557,13 +682,106 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "prost"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96d14b1c185652833d24aaad41c5832b0be5616a590227c1fbff57c616754b23"
+dependencies = [
+ "byteorder",
+ "bytes 0.4.12",
+ "prost-derive 0.5.0",
+]
+
+[[package]]
+name = "prost"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d"
+dependencies = [
+ "bytes 1.5.0",
+ "prost-derive 0.12.1",
+]
+
+[[package]]
+name = "prost-build"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb788126ea840817128183f8f603dce02cb7aea25c2a0b764359d8e20010702e"
+dependencies = [
+ "bytes 0.4.12",
+ "heck",
+ "itertools 0.8.2",
+ "log",
+ "multimap",
+ "petgraph",
+ "prost 0.5.0",
+ "prost-types 0.5.0",
+ "tempfile",
+ "which",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e7dc378b94ac374644181a2247cebf59a6ec1c88b49ac77f3a94b86b79d0e11"
+dependencies = [
+ "failure",
+ "itertools 0.8.2",
+ "proc-macro2 0.4.30",
+ "quote 0.6.13",
+ "syn 0.15.44",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32"
+dependencies = [
+ "anyhow",
+ "itertools 0.10.5",
+ "proc-macro2 1.0.67",
+ "quote 1.0.33",
+ "syn 2.0.35",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1de482a366941c8d56d19b650fac09ca08508f2a696119ee7513ad590c8bac6f"
+dependencies = [
+ "bytes 0.4.12",
+ "prost 0.5.0",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf"
+dependencies = [
+ "prost 0.12.1",
+]
+
+[[package]]
+name = "quote"
+version = "0.6.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
+dependencies = [
+ "proc-macro2 0.4.30",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.67",
 ]
 
 [[package]]
@@ -594,7 +812,7 @@ version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
@@ -632,6 +850,19 @@ version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
+[[package]]
+name = "rustix"
+version = "0.38.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662"
+dependencies = [
+ "bitflags 2.4.0",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.15"
@@ -668,9 +899,9 @@ version = "1.0.188"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.67",
- "quote",
+ "quote 1.0.33",
- "syn",
+ "syn 2.0.35",
 ]
 
 [[package]]
@@ -718,17 +949,41 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "syn"
+version = "0.15.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5"
+dependencies = [
+ "proc-macro2 0.4.30",
+ "quote 0.6.13",
+ "unicode-xid",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "59bf04c28bee9043ed9ea1e41afc0552288d3aba9c6efdd78903b802926f4879"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.67",
- "quote",
+ "quote 1.0.33",
  "unicode-ident",
 ]
 
+[[package]]
+name = "tempfile"
+version = "3.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "redox_syscall",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "test-bin"
 version = "0.1.0"
@@ -757,9 +1012,9 @@ version = "1.0.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.67",
- "quote",
+ "quote 1.0.33",
- "syn",
+ "syn 2.0.35",
 ]
 
 [[package]]
@@ -789,7 +1044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9"
 dependencies = [
  "backtrace",
- "bytes",
+ "bytes 1.5.0",
  "libc",
  "mio",
  "num_cpus",
@@ -807,9 +1062,9 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.67",
- "quote",
+ "quote 1.0.33",
- "syn",
+ "syn 2.0.35",
 ]
 
 [[package]]
@@ -831,9 +1086,9 @@ version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.67",
- "quote",
+ "quote 1.0.33",
- "syn",
+ "syn 2.0.35",
 ]
 
 [[package]]
@@ -877,6 +1132,18 @@ version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
+
+[[package]]
+name = "unicode-xid"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
+
 [[package]]
 name = "valuable"
 version = "0.1.0"
@@ -918,9 +1185,9 @@ dependencies = [
  "bumpalo",
  "log",
  "once_cell",
- "proc-macro2",
+ "proc-macro2 1.0.67",
- "quote",
+ "quote 1.0.33",
- "syn",
+ "syn 2.0.35",
  "wasm-bindgen-shared",
 ]
 
@@ -930,7 +1197,7 @@ version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
 dependencies = [
- "quote",
+ "quote 1.0.33",
  "wasm-bindgen-macro-support",
 ]
 
@@ -940,9 +1207,9 @@ version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.67",
- "quote",
+ "quote 1.0.33",
- "syn",
+ "syn 2.0.35",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -963,6 +1230,16 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "which"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164"
+dependencies = [
+ "failure",
+ "libc",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"

crates/crunch-envelope/Cargo.toml

@@ -13,18 +13,27 @@ harness = false
 debug = true
 
 [features]
-default = ["json"]
+default = ["json", "proto"]
 json = ["dep:serde", "dep:serde_json", "dep:base64"]
+proto = ["dep:prost", "dep:prost-types", "dep:bytes"]
 
 [dependencies]
 capnp = "0.17.2"
 thiserror.workspace = true
+
+# Json
 serde = { version = "1.0.188" ,optional = true, features = ["derive"] }
 serde_json = {version = "1.0.107",optional = true}
 base64 = {version = "0.21.4",optional = true}
 
+# Proto
+prost = {version  = "0.12", optional = true}
+prost-types = {version = "0.12", optional = true}
+bytes = {version = "0.4", optional = true}
+
 [build-dependencies]
 capnpc = "0.17.2"
+prost-build = "0.5"
 
 [dev-dependencies]
 criterion = { version = "0.4" }

crates/crunch-envelope/benches/envelope_benchmark.rs

@@ -13,6 +13,12 @@ fn envelope_json_benchmark(content: &[u8]) -> () {
     let _ = crunch_envelope::json::unwrap(&out).expect("to be able to unwrap capnp message");
 }
 
+fn envelope_proto_benchmark(content: &[u8]) -> () {
+    let out = crunch_envelope::proto::wrap("some-domain", "some-entity", content);
+
+    let _ = crunch_envelope::proto::unwrap(&out).expect("to be able to unwrap capnp message");
+}
+
 fn criterion_benchmark(c: &mut Criterion) {
     let large_content: [u8; 10000] = [0; 10000];
 
@@ -22,6 +28,9 @@ fn criterion_benchmark(c: &mut Criterion) {
     c.bench_function("envelope::json", |b| {
         b.iter(|| envelope_json_benchmark(&large_content))
     });
+    c.bench_function("envelope::proto", |b| {
+        b.iter(|| envelope_proto_benchmark(&large_content))
+    });
 }
 
 criterion_group!(benches, criterion_benchmark);

crates/crunch-envelope/build.rs

@@ -7,4 +7,6 @@ fn main() {
         .file("schemas/envelope.capnp")
         .run()
         .unwrap();
+
+    prost_build::compile_protos(&["src/envelope.proto"], &["src/"]).unwrap();
 }

crates/crunch-envelope/src/envelope.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+package crunch.envelope;
+
+message Envelope {
+    Metadata metadata = 1;
+    bytes content = 2;
+}
+
+message Metadata {
+    string domain = 1;
+    string entity = 2;
+    uint64 timestamp = 3;
+    uint64 sequence = 4;
+}

crates/crunch-envelope/src/lib.rs

@@ -3,11 +3,19 @@ mod envelope_capnp;
 #[cfg(feature = "json")]
 mod json_envelope;
 
+#[cfg(feature = "proto")]
+mod proto_envelope;
+
 #[cfg(feature = "json")]
 pub mod json {
     pub use crate::json_envelope::*;
 }
 
+#[cfg(feature = "proto")]
+pub mod proto {
+    pub use crate::proto_envelope::*;
+}
+
 use capnp::message::{Builder, ReaderOptions};
 use capnp::serialize;
 use thiserror::Error;
@@ -22,6 +30,11 @@ pub enum EnvelopeError {
     #[cfg(feature = "json")]
     #[error("base64 failed to serialize or deserialize code")]
     Base64Error(#[source] base64::DecodeError),
+    #[cfg(feature = "proto")]
+    #[error("prost failed to serialize or deserialize code")]
+    ProtoError(#[source] prost::DecodeError),
+    #[error("metadata is missing from field")]
+    MetadataError(),
 }
 
 pub fn wrap<'a>(domain: &'a str, entity: &'a str, content: &'a [u8]) -> Vec<u8> {

crates/crunch-envelope/src/proto_envelope.rs

@@ -0,0 +1,30 @@
+pub mod envelope {
+    include!(concat!(env!("OUT_DIR"), "/crunch.envelope.rs"));
+}
+
+use prost::Message;
+
+use crate::EnvelopeError;
+
+pub fn wrap<'a>(domain: &'a str, entity: &'a str, content: &'a [u8]) -> Vec<u8> {
+    let out = envelope::Envelope {
+        metadata: Some(envelope::Metadata {
+            domain: domain.to_string(),
+            entity: entity.to_string(),
+            timestamp: 0,
+            sequence: 0,
+        }),
+        content: content.to_vec(),
+    };
+
+    out.encode_to_vec()
+}
+
+pub fn unwrap<'a>(message: &'a [u8]) -> Result<(Vec<u8>, envelope::Metadata), EnvelopeError> {
+    let out = envelope::Envelope::decode(message).map_err(EnvelopeError::ProtoError)?;
+
+    Ok((
+        out.content,
+        out.metadata.ok_or(EnvelopeError::MetadataError())?,
+    ))
+}

crates/crunch/src/lib.rs

@@ -48,6 +48,7 @@ mod impls {
     use crate::{traits, EventInfo};
 
     pub struct InMemoryPersistence {}
+
     #[async_trait]
     impl traits::Persistence for InMemoryPersistence {
         async fn insert(&self, event_info: &EventInfo, content: Vec<u8>) -> anyhow::Result<()> {