kjuulh/crunch
1
0
Fork 0
Code Issues 1 Pull Requests 5 Packages Projects Releases Wiki Activity

feat: with protobuf

Browse Source 
Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
Kasper Juul Hermansen 2023-09-19 08:31:13 +02:00
parent d4d5374392
commit 41f36c09be
Signed by: kjuulh
GPG Key ID: 57B6E1465221F912
8 changed files with 387 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

337
Cargo.lock generated
View File

@ -44,9 +44,9 @@ version = "0.1.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
dependencies = [
"proc-macro2",
"quote",
"syn",
"proc-macro2 1.0.67",
"quote 1.0.33",
"syn 2.0.35",
]
[[package]]
@ -93,12 +93,34 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
[[package]]
name = "bumpalo"
version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
[[package]]
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
dependencies = [
"byteorder",
"iovec",
]
[[package]]
name = "bytes"
version = "1.5.0"
@ -174,7 +196,7 @@ version = "3.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"clap_lex",
"indexmap",
"textwrap",
@ -201,7 +223,7 @@ dependencies = [
"ciborium",
"clap",
"criterion-plot",
"itertools",
"itertools 0.10.5",
"lazy_static",
"num-traits",
"oorandom",
@ -222,7 +244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
dependencies = [
"cast",
"itertools",
"itertools 0.10.5",
]
[[package]]
@ -285,9 +307,13 @@ name = "crunch-envelope"
version = "0.1.0"
dependencies = [
"base64",
"bytes 0.4.12",
"capnp",
"capnpc",
"criterion",
"prost 0.12.1",
"prost-build",
"prost-types 0.12.1",
"serde",
"serde_json",
"thiserror",
@ -299,6 +325,48 @@ version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
[[package]]
name = "errno"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd"
dependencies = [
"errno-dragonfly",
"libc",
"windows-sys",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "failure"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
dependencies = [
"backtrace",
]
[[package]]
name = "fastrand"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
[[package]]
name = "fixedbitset"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33"
[[package]]
name = "gimli"
version = "0.28.0"
@ -317,6 +385,15 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "heck"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
dependencies = [
"unicode-segmentation",
]
[[package]]
name = "hermit-abi"
version = "0.1.19"
@ -342,6 +419,24 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "iovec"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
dependencies = [
"libc",
]
[[package]]
name = "itertools"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484"
dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.10.5"
@ -378,6 +473,12 @@ version = "0.2.148"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
[[package]]
name = "linux-raw-sys"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128"
[[package]]
name = "lock_api"
version = "0.4.10"
@ -429,6 +530,12 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "multimap"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2eb04b9f127583ed176e163fb9ec6f3e793b87e21deedd5734a69386a18a0151"
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
@ -514,6 +621,15 @@ dependencies = [
"windows-targets",
]
[[package]]
name = "petgraph"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f"
dependencies = [
"fixedbitset",
]
[[package]]
name = "pin-project-lite"
version = "0.2.13"
@ -548,6 +664,15 @@ dependencies = [
"plotters-backend",
]
[[package]]
name = "proc-macro2"
version = "0.4.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
dependencies = [
"unicode-xid",
]
[[package]]
name = "proc-macro2"
version = "1.0.67"
@ -557,13 +682,106 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "prost"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96d14b1c185652833d24aaad41c5832b0be5616a590227c1fbff57c616754b23"
dependencies = [
"byteorder",
"bytes 0.4.12",
"prost-derive 0.5.0",
]
[[package]]
name = "prost"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d"
dependencies = [
"bytes 1.5.0",
"prost-derive 0.12.1",
]
[[package]]
name = "prost-build"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb788126ea840817128183f8f603dce02cb7aea25c2a0b764359d8e20010702e"
dependencies = [
"bytes 0.4.12",
"heck",
"itertools 0.8.2",
"log",
"multimap",
"petgraph",
"prost 0.5.0",
"prost-types 0.5.0",
"tempfile",
"which",
]
[[package]]
name = "prost-derive"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e7dc378b94ac374644181a2247cebf59a6ec1c88b49ac77f3a94b86b79d0e11"
dependencies = [
"failure",
"itertools 0.8.2",
"proc-macro2 0.4.30",
"quote 0.6.13",
"syn 0.15.44",
]
[[package]]
name = "prost-derive"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32"
dependencies = [
"anyhow",
"itertools 0.10.5",
"proc-macro2 1.0.67",
"quote 1.0.33",
"syn 2.0.35",
]
[[package]]
name = "prost-types"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1de482a366941c8d56d19b650fac09ca08508f2a696119ee7513ad590c8bac6f"
dependencies = [
"bytes 0.4.12",
"prost 0.5.0",
]
[[package]]
name = "prost-types"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf"
dependencies = [
"prost 0.12.1",
]
[[package]]
name = "quote"
version = "0.6.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
dependencies = [
"proc-macro2 0.4.30",
]
[[package]]
name = "quote"
version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
dependencies = [
"proc-macro2",
"proc-macro2 1.0.67",
]
[[package]]
@ -594,7 +812,7 @@ version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
dependencies = [
"bitflags",
"bitflags 1.3.2",
]
[[package]]
@ -632,6 +850,19 @@ version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "rustix"
version = "0.38.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662"
dependencies = [
"bitflags 2.4.0",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "ryu"
version = "1.0.15"
@ -668,9 +899,9 @@ version = "1.0.188"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
dependencies = [
"proc-macro2",
"quote",
"syn",
"proc-macro2 1.0.67",
"quote 1.0.33",
"syn 2.0.35",
]
[[package]]
@ -718,17 +949,41 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "syn"
version = "0.15.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5"
dependencies = [
"proc-macro2 0.4.30",
"quote 0.6.13",
"unicode-xid",
]
[[package]]
name = "syn"
version = "2.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59bf04c28bee9043ed9ea1e41afc0552288d3aba9c6efdd78903b802926f4879"
dependencies = [
"proc-macro2",
"quote",
"proc-macro2 1.0.67",
"quote 1.0.33",
"unicode-ident",
]
[[package]]
name = "tempfile"
version = "3.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
dependencies = [
"cfg-if",
"fastrand",
"redox_syscall",
"rustix",
"windows-sys",
]
[[package]]
name = "test-bin"
version = "0.1.0"
@ -757,9 +1012,9 @@ version = "1.0.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
dependencies = [
"proc-macro2",
"quote",
"syn",
"proc-macro2 1.0.67",
"quote 1.0.33",
"syn 2.0.35",
]
[[package]]
@ -789,7 +1044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9"
dependencies = [
"backtrace",
"bytes",
"bytes 1.5.0",
"libc",
"mio",
"num_cpus",
@ -807,9 +1062,9 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2",
"quote",
"syn",
"proc-macro2 1.0.67",
"quote 1.0.33",
"syn 2.0.35",
]
[[package]]
@ -831,9 +1086,9 @@ version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
dependencies = [
"proc-macro2",
"quote",
"syn",
"proc-macro2 1.0.67",
"quote 1.0.33",
"syn 2.0.35",
]
[[package]]
@ -877,6 +1132,18 @@ version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "unicode-segmentation"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
[[package]]
name = "unicode-xid"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
[[package]]
name = "valuable"
version = "0.1.0"
@ -918,9 +1185,9 @@ dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn",
"proc-macro2 1.0.67",
"quote 1.0.33",
"syn 2.0.35",
"wasm-bindgen-shared",
]
@ -930,7 +1197,7 @@ version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
dependencies = [
"quote",
"quote 1.0.33",
"wasm-bindgen-macro-support",
]
@ -940,9 +1207,9 @@ version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2",
"quote",
"syn",
"proc-macro2 1.0.67",
"quote 1.0.33",
"syn 2.0.35",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@ -963,6 +1230,16 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "which"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164"
dependencies = [
"failure",
"libc",
]
[[package]]
name = "winapi"
version = "0.3.9"

11
crates/crunch-envelope/Cargo.toml
View File

@ -13,18 +13,27 @@ harness = false
debug = true
[features]
default = ["json"]
default = ["json", "proto"]
json = ["dep:serde", "dep:serde_json", "dep:base64"]
proto = ["dep:prost", "dep:prost-types", "dep:bytes"]
[dependencies]
capnp = "0.17.2"
thiserror.workspace = true
# Json
serde = { version = "1.0.188" ,optional = true, features = ["derive"] }
serde_json = {version = "1.0.107",optional = true}
base64 = {version = "0.21.4",optional = true}
# Proto
prost = {version = "0.12", optional = true}
prost-types = {version = "0.12", optional = true}
bytes = {version = "0.4", optional = true}
[build-dependencies]
capnpc = "0.17.2"
prost-build = "0.5"
[dev-dependencies]
criterion = { version = "0.4" }

9
crates/crunch-envelope/benches/envelope_benchmark.rs
View File

@ -13,6 +13,12 @@ fn envelope_json_benchmark(content: &[u8]) -> () {
let _ = crunch_envelope::json::unwrap(&out).expect("to be able to unwrap capnp message");
}
fn envelope_proto_benchmark(content: &[u8]) -> () {
let out = crunch_envelope::proto::wrap("some-domain", "some-entity", content);
let _ = crunch_envelope::proto::unwrap(&out).expect("to be able to unwrap capnp message");
}
fn criterion_benchmark(c: &mut Criterion) {
let large_content: [u8; 10000] = [0; 10000];
@ -22,6 +28,9 @@ fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("envelope::json", |b| {
b.iter(|| envelope_json_benchmark(&large_content))
});
c.bench_function("envelope::proto", |b| {
b.iter(|| envelope_proto_benchmark(&large_content))
});
}
criterion_group!(benches, criterion_benchmark);

2
crates/crunch-envelope/build.rs
View File

@ -7,4 +7,6 @@ fn main() {
.file("schemas/envelope.capnp")
.run()
.unwrap();
prost_build::compile_protos(&["src/envelope.proto"], &["src/"]).unwrap();
}

15
crates/crunch-envelope/src/envelope.proto Normal file
View File

@ -0,0 +1,15 @@
syntax = "proto3";
package crunch.envelope;
message Envelope {
Metadata metadata = 1;
bytes content = 2;
}
message Metadata {
string domain = 1;
string entity = 2;
uint64 timestamp = 3;
uint64 sequence = 4;
}

13
crates/crunch-envelope/src/lib.rs
View File

@ -3,11 +3,19 @@ mod envelope_capnp;
#[cfg(feature = "json")]
mod json_envelope;
#[cfg(feature = "proto")]
mod proto_envelope;
#[cfg(feature = "json")]
pub mod json {
pub use crate::json_envelope::*;
}
#[cfg(feature = "proto")]
pub mod proto {
pub use crate::proto_envelope::*;
}
use capnp::message::{Builder, ReaderOptions};
use capnp::serialize;
use thiserror::Error;
@ -22,6 +30,11 @@ pub enum EnvelopeError {
#[cfg(feature = "json")]
#[error("base64 failed to serialize or deserialize code")]
Base64Error(#[source] base64::DecodeError),
#[cfg(feature = "proto")]
#[error("prost failed to serialize or deserialize code")]
ProtoError(#[source] prost::DecodeError),
#[error("metadata is missing from field")]
MetadataError(),
}
pub fn wrap<'a>(domain: &'a str, entity: &'a str, content: &'a [u8]) -> Vec<u8> {

30
crates/crunch-envelope/src/proto_envelope.rs Normal file
View File

@ -0,0 +1,30 @@
pub mod envelope {
include!(concat!(env!("OUT_DIR"), "/crunch.envelope.rs"));
}
use prost::Message;
use crate::EnvelopeError;
pub fn wrap<'a>(domain: &'a str, entity: &'a str, content: &'a [u8]) -> Vec<u8> {
let out = envelope::Envelope {
metadata: Some(envelope::Metadata {
domain: domain.to_string(),
entity: entity.to_string(),
timestamp: 0,
sequence: 0,
}),
content: content.to_vec(),
};
out.encode_to_vec()
}
pub fn unwrap<'a>(message: &'a [u8]) -> Result<(Vec<u8>, envelope::Metadata), EnvelopeError> {
let out = envelope::Envelope::decode(message).map_err(EnvelopeError::ProtoError)?;
Ok((
out.content,
out.metadata.ok_or(EnvelopeError::MetadataError())?,
))
}

1
crates/crunch/src/lib.rs
View File

@ -48,6 +48,7 @@ mod impls {
use crate::{traits, EventInfo};
pub struct InMemoryPersistence {}
#[async_trait]
impl traits::Persistence for InMemoryPersistence {
async fn insert(&self, event_info: &EventInfo, content: Vec<u8>) -> anyhow::Result<()> {