feat: with real time streaming
Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
commit
fc27206708
GPG Key ID:
57B6E1465221F912
23
config/clickhouse/docker/00_create_nats_table.sh
Executable file
23
config/clickhouse/docker/00_create_nats_table.sh
Executable file
@ -0,0 +1,23 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
clickhouse client -n <<-EOSQL
|
||||||
|
CREATE TABLE nats (
|
||||||
|
key String,
|
||||||
|
) ENGINE = NATS
|
||||||
|
SETTINGS nats_url = 'nats:4222',
|
||||||
|
nats_username = 'natsadmin',
|
||||||
|
nats_password = 'natsadmin',
|
||||||
|
nats_subjects = 'cli.demo',
|
||||||
|
nats_format = 'JSONEachRow',
|
||||||
|
date_time_input_format = 'best_effort';
|
||||||
|
|
||||||
|
CREATE TABLE daily (key String)
|
||||||
|
ENGINE = MergeTree() ORDER BY key;
|
||||||
|
|
||||||
|
CREATE MATERIALIZED VIEW consumer TO daily
|
||||||
|
AS SELECT key FROM nats;
|
||||||
|
|
||||||
|
SELECT key FROM daily ORDER BY key;
|
||||||
|
EOSQL
|
||||||
48
config/clickhouse/server/config.d/config.xml
Normal file
48
config/clickhouse/server/config.d/config.xml
Normal file
@ -0,0 +1,48 @@
|
|||||||
|
<clickhouse replace="true">
|
||||||
|
<logger>
|
||||||
|
<level>debug</level>
|
||||||
|
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
||||||
|
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
||||||
|
<size>1000M</size>
|
||||||
|
<count>3</count>
|
||||||
|
</logger>
|
||||||
|
<display_name>ch_minio_s3</display_name>
|
||||||
|
<listen_host>0.0.0.0</listen_host>
|
||||||
|
<http_port>8123</http_port>
|
||||||
|
<tcp_port>9000</tcp_port>
|
||||||
|
<user_directories>
|
||||||
|
<users_xml>
|
||||||
|
<path>users.xml</path>
|
||||||
|
</users_xml>
|
||||||
|
<local_directory>
|
||||||
|
<path>/var/lib/clickhouse/access/</path>
|
||||||
|
</local_directory>
|
||||||
|
</user_directories>
|
||||||
|
<storage_configuration>
|
||||||
|
<disks>
|
||||||
|
<s3>
|
||||||
|
<type>s3</type>
|
||||||
|
<endpoint>http://minio:10000/clickhouse//</endpoint>
|
||||||
|
<access_key_id>minioadmin</access_key_id>
|
||||||
|
<secret_access_key>minioadminpassword</secret_access_key>
|
||||||
|
<region></region>
|
||||||
|
<metadata_path>/var/lib/clickhouse/disks/s3/</metadata_path>
|
||||||
|
</s3>
|
||||||
|
<s3_cache>
|
||||||
|
<type>cache</type>
|
||||||
|
<disk>s3</disk>
|
||||||
|
<path>/var/lib/clickhouse/disks/s3_cache/</path>
|
||||||
|
<max_size>10Gi</max_size>
|
||||||
|
</s3_cache>
|
||||||
|
</disks>
|
||||||
|
<policies>
|
||||||
|
<s3_main>
|
||||||
|
<volumes>
|
||||||
|
<s3>
|
||||||
|
<disk>s3</disk>
|
||||||
|
</s3>
|
||||||
|
</volumes>
|
||||||
|
</s3_main>
|
||||||
|
</policies>
|
||||||
|
</storage_configuration>
|
||||||
|
</clickhouse>
|
||||||
37
config/clickhouse/server/users.d/users.xml
Normal file
37
config/clickhouse/server/users.d/users.xml
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
<?xml version="1.0"?>
|
||||||
|
<clickhouse replace="true">
|
||||||
|
<profiles>
|
||||||
|
<default>
|
||||||
|
<max_memory_usage>10000000000</max_memory_usage>
|
||||||
|
<use_uncompressed_cache>0</use_uncompressed_cache>
|
||||||
|
<load_balancing>in_order</load_balancing>
|
||||||
|
<log_queries>1</log_queries>
|
||||||
|
</default>
|
||||||
|
</profiles>
|
||||||
|
<users>
|
||||||
|
<default>
|
||||||
|
<access_management>1</access_management>
|
||||||
|
<profile>default</profile>
|
||||||
|
<networks>
|
||||||
|
<ip>::/0</ip>
|
||||||
|
</networks>
|
||||||
|
<quota>default</quota>
|
||||||
|
<access_management>1</access_management>
|
||||||
|
<named_collection_control>1</named_collection_control>
|
||||||
|
<show_named_collections>1</show_named_collections>
|
||||||
|
<show_named_collections_secrets>1</show_named_collections_secrets>
|
||||||
|
</default>
|
||||||
|
</users>
|
||||||
|
<quotas>
|
||||||
|
<default>
|
||||||
|
<interval>
|
||||||
|
<duration>3600</duration>
|
||||||
|
<queries>0</queries>
|
||||||
|
<errors>0</errors>
|
||||||
|
<result_rows>0</result_rows>
|
||||||
|
<read_rows>0</read_rows>
|
||||||
|
<execution_time>0</execution_time>
|
||||||
|
</interval>
|
||||||
|
</default>
|
||||||
|
</quotas>
|
||||||
|
</clickhouse>
|
||||||
61
docker-compose.yml
Normal file
61
docker-compose.yml
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
version: '3.8'
|
||||||
|
services:
|
||||||
|
clickhouse:
|
||||||
|
image: clickhouse/clickhouse-server
|
||||||
|
user: '101:101'
|
||||||
|
container_name: clickhouse
|
||||||
|
hostname: clickhouse
|
||||||
|
volumes:
|
||||||
|
- ${PWD}/config/clickhouse/server/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml
|
||||||
|
- ${PWD}/config/clickhouse/server/users.d/users.xml:/etc/clickhouse-server/users.d/users.xml
|
||||||
|
- ${PWD}/config/clickhouse/docker/:/docker-entrypoint-initdb.d
|
||||||
|
ports:
|
||||||
|
- '127.0.0.1:8123:8123'
|
||||||
|
- '127.0.0.1:9000:9000'
|
||||||
|
depends_on:
|
||||||
|
- minio
|
||||||
|
- createbuckets
|
||||||
|
- nats
|
||||||
|
minio:
|
||||||
|
image: quay.io/minio/minio
|
||||||
|
container_name: minio
|
||||||
|
hostname: minio
|
||||||
|
command: server --address 0.0.0.0:10000 --console-address 0.0.0.0:10001 /data
|
||||||
|
ports:
|
||||||
|
- '127.0.0.1:10000:10000'
|
||||||
|
- '127.0.0.1:10001:10001'
|
||||||
|
environment:
|
||||||
|
- MINIO_ROOT_USER=minioadmin
|
||||||
|
- MINIO_ROOT_PASSWORD=minioadminpassword
|
||||||
|
|
||||||
|
createbuckets:
|
||||||
|
image: minio/mc
|
||||||
|
depends_on:
|
||||||
|
- minio
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
/usr/bin/mc alias set myminio http://minio:10000 minioadmin minioadminpassword;
|
||||||
|
/usr/bin/mc admin info myminio;
|
||||||
|
/usr/bin/mc mb myminio/clickhouse;
|
||||||
|
/usr/bin/mc policy set public myminio/clickhouse;
|
||||||
|
exit 0;
|
||||||
|
"
|
||||||
|
nats:
|
||||||
|
container_name: nats
|
||||||
|
hostname: nats
|
||||||
|
image: 'bitnami/nats:latest'
|
||||||
|
ports:
|
||||||
|
- 127.0.0.1:4222:4222
|
||||||
|
- 127.0.0.1:6222:6222
|
||||||
|
- 127.0.0.1:8222:8222
|
||||||
|
environment:
|
||||||
|
- NATS_ENABLE_AUTH=yes
|
||||||
|
- NATS_USERNAME=natsadmin
|
||||||
|
- NATS_PASSWORD=natsadmin
|
||||||
|
#- NATS_TOKEN=natsadmin
|
||||||
|
# healthcheck:
|
||||||
|
# test: ["CMD", "curl", "-f", "http://localhost:4222"]
|
||||||
|
# interval: 1m30s
|
||||||
|
# timeout: 10s
|
||||||
|
# retries: 3
|
||||||
|
# start_period: 2m
|
||||||
Loading…
Reference in New Issue
Block a user