feat: add broker setup

this is mainly to decouple the actual sending of events, from the ingest.

we now ingest the data, and update consumer groups with the new offset. Consumer groups now in the background continously send out data from the update. they tick 1 second between checks, but if something takes long than a second, the next run just continues from where we left off

Signed-off-by: kjuulh <contact@kjuulh.io>

Cargo.lock

@@ -480,6 +480,19 @@ version = "0.15.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
 
+[[package]]
+name = "drift"
+version = "0.2.0"
+source = "git+https://github.com/kjuulh/drift?branch=main#fb4bd51d7106bf2ebffa851dbce172c06ed12ff7"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "thiserror",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
 [[package]]
 name = "either"
 version = "1.13.0"
@@ -973,9 +986,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.155"
+version = "0.2.156"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "a5f43f184355eefb8d17fc948dbecf6c13be3c141f20d834ae842193a448c72a"
 
 [[package]]
 name = "libm"
@@ -1106,6 +1119,7 @@ dependencies = [
  "chrono",
  "clap",
  "dotenv",
+ "drift",
  "mad",
  "prost",
  "prost-types",