From fc27206708986938a776d1fd98e72c72b130dd0a Mon Sep 17 00:00:00 2001
From: kjuulh <contact@kjuulh.io>
Date: Mon, 11 Dec 2023 22:31:52 +0100
Subject: [PATCH] feat: with real time streaming

Signed-off-by: kjuulh <contact@kjuulh.io>
---
 .../clickhouse/docker/00_create_nats_table.sh | 23 +++++++
 config/clickhouse/server/config.d/config.xml  | 48 +++++++++++++++
 config/clickhouse/server/users.d/users.xml    | 37 +++++++++++
 docker-compose.yml                            | 61 +++++++++++++++++++
 4 files changed, 169 insertions(+)
 create mode 100755 config/clickhouse/docker/00_create_nats_table.sh
 create mode 100644 config/clickhouse/server/config.d/config.xml
 create mode 100644 config/clickhouse/server/users.d/users.xml
 create mode 100644 docker-compose.yml

diff --git a/config/clickhouse/docker/00_create_nats_table.sh b/config/clickhouse/docker/00_create_nats_table.sh
new file mode 100755
index 0000000..86193e5
--- /dev/null
+++ b/config/clickhouse/docker/00_create_nats_table.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e 
+
+clickhouse client -n <<-EOSQL
+CREATE TABLE nats (
+  key String,
+) ENGINE = NATS
+  SETTINGS nats_url = 'nats:4222',
+           nats_username = 'natsadmin',
+           nats_password = 'natsadmin',
+           nats_subjects = 'cli.demo',
+           nats_format = 'JSONEachRow',
+           date_time_input_format = 'best_effort';
+
+CREATE TABLE daily (key String)
+  ENGINE = MergeTree() ORDER BY key;
+
+CREATE MATERIALIZED VIEW consumer TO daily
+  AS SELECT key FROM nats;
+
+SELECT key FROM daily ORDER BY key;
+EOSQL
diff --git a/config/clickhouse/server/config.d/config.xml b/config/clickhouse/server/config.d/config.xml
new file mode 100644
index 0000000..d87aa3a
--- /dev/null
+++ b/config/clickhouse/server/config.d/config.xml
@@ -0,0 +1,48 @@
+<clickhouse replace="true">
+    <logger>
+        <level>debug</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <display_name>ch_minio_s3</display_name>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <user_directories>
+        <users_xml>
+            <path>users.xml</path>
+        </users_xml>
+        <local_directory>
+            <path>/var/lib/clickhouse/access/</path>
+        </local_directory>
+    </user_directories>
+    <storage_configuration>
+        <disks>
+            <s3>
+                <type>s3</type>
+                <endpoint>http://minio:10000/clickhouse//</endpoint>
+                <access_key_id>minioadmin</access_key_id>
+                <secret_access_key>minioadminpassword</secret_access_key>
+                <region></region>
+                <metadata_path>/var/lib/clickhouse/disks/s3/</metadata_path>
+            </s3>
+            <s3_cache>
+                <type>cache</type>
+                <disk>s3</disk>
+                <path>/var/lib/clickhouse/disks/s3_cache/</path>
+                <max_size>10Gi</max_size>
+            </s3_cache>
+        </disks>
+        <policies>
+            <s3_main>
+                <volumes>
+                    <s3>
+                        <disk>s3</disk>
+                    </s3>
+                </volumes>
+            </s3_main>
+        </policies>
+    </storage_configuration>
+</clickhouse>
diff --git a/config/clickhouse/server/users.d/users.xml b/config/clickhouse/server/users.d/users.xml
new file mode 100644
index 0000000..0f32c64
--- /dev/null
+++ b/config/clickhouse/server/users.d/users.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<clickhouse replace="true">
+    <profiles>
+        <default>
+            <max_memory_usage>10000000000</max_memory_usage>
+            <use_uncompressed_cache>0</use_uncompressed_cache>
+            <load_balancing>in_order</load_balancing>
+            <log_queries>1</log_queries>
+        </default>
+    </profiles>
+    <users>
+        <default>
+            <access_management>1</access_management>
+            <profile>default</profile>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <quota>default</quota>
+            <access_management>1</access_management>
+            <named_collection_control>1</named_collection_control>
+            <show_named_collections>1</show_named_collections>
+            <show_named_collections_secrets>1</show_named_collections_secrets>
+        </default>
+    </users>
+    <quotas>
+        <default>
+            <interval>
+                <duration>3600</duration>
+                <queries>0</queries>
+                <errors>0</errors>
+                <result_rows>0</result_rows>
+                <read_rows>0</read_rows>
+                <execution_time>0</execution_time>
+            </interval>
+        </default>
+    </quotas>
+</clickhouse>
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..a4d785a
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,61 @@
+version: '3.8'
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: '101:101'
+    container_name: clickhouse
+    hostname: clickhouse
+    volumes:
+      - ${PWD}/config/clickhouse/server/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml
+      - ${PWD}/config/clickhouse/server/users.d/users.xml:/etc/clickhouse-server/users.d/users.xml
+      - ${PWD}/config/clickhouse/docker/:/docker-entrypoint-initdb.d
+    ports:
+      - '127.0.0.1:8123:8123'
+      - '127.0.0.1:9000:9000'
+    depends_on:
+      - minio
+      - createbuckets
+      - nats
+  minio:
+    image: quay.io/minio/minio
+    container_name: minio
+    hostname: minio
+    command: server --address 0.0.0.0:10000 --console-address 0.0.0.0:10001 /data
+    ports:
+      - '127.0.0.1:10000:10000'
+      - '127.0.0.1:10001:10001'
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadminpassword
+
+  createbuckets:
+    image: minio/mc
+    depends_on:
+      - minio
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set myminio http://minio:10000 minioadmin minioadminpassword;
+      /usr/bin/mc admin info myminio;
+      /usr/bin/mc mb myminio/clickhouse;
+      /usr/bin/mc policy set public myminio/clickhouse;
+      exit 0;
+      "
+  nats:
+    container_name: nats
+    hostname: nats
+    image: 'bitnami/nats:latest'
+    ports:
+      - 127.0.0.1:4222:4222
+      - 127.0.0.1:6222:6222
+      - 127.0.0.1:8222:8222
+    environment:
+      - NATS_ENABLE_AUTH=yes
+      - NATS_USERNAME=natsadmin
+      - NATS_PASSWORD=natsadmin
+      #- NATS_TOKEN=natsadmin
+    # healthcheck:
+    #   test: ["CMD", "curl", "-f", "http://localhost:4222"]
+    #   interval: 1m30s
+    #   timeout: 10s
+    #   retries: 3
+    #   start_period: 2m