feat: with real time streaming
Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
commit
fc27206708
GPG Key ID:
57B6E1465221F912
23
config/clickhouse/docker/00_create_nats_table.sh
Executable file
23
config/clickhouse/docker/00_create_nats_table.sh
Executable file
@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
clickhouse client -n <<-EOSQL
|
||||
CREATE TABLE nats (
|
||||
key String,
|
||||
) ENGINE = NATS
|
||||
SETTINGS nats_url = 'nats:4222',
|
||||
nats_username = 'natsadmin',
|
||||
nats_password = 'natsadmin',
|
||||
nats_subjects = 'cli.demo',
|
||||
nats_format = 'JSONEachRow',
|
||||
date_time_input_format = 'best_effort';
|
||||
|
||||
CREATE TABLE daily (key String)
|
||||
ENGINE = MergeTree() ORDER BY key;
|
||||
|
||||
CREATE MATERIALIZED VIEW consumer TO daily
|
||||
AS SELECT key FROM nats;
|
||||
|
||||
SELECT key FROM daily ORDER BY key;
|
||||
EOSQL
|
||||
48
config/clickhouse/server/config.d/config.xml
Normal file
48
config/clickhouse/server/config.d/config.xml
Normal file
@ -0,0 +1,48 @@
|
||||
<clickhouse replace="true">
|
||||
<logger>
|
||||
<level>debug</level>
|
||||
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
||||
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
||||
<size>1000M</size>
|
||||
<count>3</count>
|
||||
</logger>
|
||||
<display_name>ch_minio_s3</display_name>
|
||||
<listen_host>0.0.0.0</listen_host>
|
||||
<http_port>8123</http_port>
|
||||
<tcp_port>9000</tcp_port>
|
||||
<user_directories>
|
||||
<users_xml>
|
||||
<path>users.xml</path>
|
||||
</users_xml>
|
||||
<local_directory>
|
||||
<path>/var/lib/clickhouse/access/</path>
|
||||
</local_directory>
|
||||
</user_directories>
|
||||
<storage_configuration>
|
||||
<disks>
|
||||
<s3>
|
||||
<type>s3</type>
|
||||
<endpoint>http://minio:10000/clickhouse//</endpoint>
|
||||
<access_key_id>minioadmin</access_key_id>
|
||||
<secret_access_key>minioadminpassword</secret_access_key>
|
||||
<region></region>
|
||||
<metadata_path>/var/lib/clickhouse/disks/s3/</metadata_path>
|
||||
</s3>
|
||||
<s3_cache>
|
||||
<type>cache</type>
|
||||
<disk>s3</disk>
|
||||
<path>/var/lib/clickhouse/disks/s3_cache/</path>
|
||||
<max_size>10Gi</max_size>
|
||||
</s3_cache>
|
||||
</disks>
|
||||
<policies>
|
||||
<s3_main>
|
||||
<volumes>
|
||||
<s3>
|
||||
<disk>s3</disk>
|
||||
</s3>
|
||||
</volumes>
|
||||
</s3_main>
|
||||
</policies>
|
||||
</storage_configuration>
|
||||
</clickhouse>
|
||||
37
config/clickhouse/server/users.d/users.xml
Normal file
37
config/clickhouse/server/users.d/users.xml
Normal file
@ -0,0 +1,37 @@
|
||||
<?xml version="1.0"?>
|
||||
<clickhouse replace="true">
|
||||
<profiles>
|
||||
<default>
|
||||
<max_memory_usage>10000000000</max_memory_usage>
|
||||
<use_uncompressed_cache>0</use_uncompressed_cache>
|
||||
<load_balancing>in_order</load_balancing>
|
||||
<log_queries>1</log_queries>
|
||||
</default>
|
||||
</profiles>
|
||||
<users>
|
||||
<default>
|
||||
<access_management>1</access_management>
|
||||
<profile>default</profile>
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
<quota>default</quota>
|
||||
<access_management>1</access_management>
|
||||
<named_collection_control>1</named_collection_control>
|
||||
<show_named_collections>1</show_named_collections>
|
||||
<show_named_collections_secrets>1</show_named_collections_secrets>
|
||||
</default>
|
||||
</users>
|
||||
<quotas>
|
||||
<default>
|
||||
<interval>
|
||||
<duration>3600</duration>
|
||||
<queries>0</queries>
|
||||
<errors>0</errors>
|
||||
<result_rows>0</result_rows>
|
||||
<read_rows>0</read_rows>
|
||||
<execution_time>0</execution_time>
|
||||
</interval>
|
||||
</default>
|
||||
</quotas>
|
||||
</clickhouse>
|
||||
61
docker-compose.yml
Normal file
61
docker-compose.yml
Normal file
@ -0,0 +1,61 @@
|
||||
version: '3.8'
|
||||
services:
|
||||
clickhouse:
|
||||
image: clickhouse/clickhouse-server
|
||||
user: '101:101'
|
||||
container_name: clickhouse
|
||||
hostname: clickhouse
|
||||
volumes:
|
||||
- ${PWD}/config/clickhouse/server/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml
|
||||
- ${PWD}/config/clickhouse/server/users.d/users.xml:/etc/clickhouse-server/users.d/users.xml
|
||||
- ${PWD}/config/clickhouse/docker/:/docker-entrypoint-initdb.d
|
||||
ports:
|
||||
- '127.0.0.1:8123:8123'
|
||||
- '127.0.0.1:9000:9000'
|
||||
depends_on:
|
||||
- minio
|
||||
- createbuckets
|
||||
- nats
|
||||
minio:
|
||||
image: quay.io/minio/minio
|
||||
container_name: minio
|
||||
hostname: minio
|
||||
command: server --address 0.0.0.0:10000 --console-address 0.0.0.0:10001 /data
|
||||
ports:
|
||||
- '127.0.0.1:10000:10000'
|
||||
- '127.0.0.1:10001:10001'
|
||||
environment:
|
||||
- MINIO_ROOT_USER=minioadmin
|
||||
- MINIO_ROOT_PASSWORD=minioadminpassword
|
||||
|
||||
createbuckets:
|
||||
image: minio/mc
|
||||
depends_on:
|
||||
- minio
|
||||
entrypoint: >
|
||||
/bin/sh -c "
|
||||
/usr/bin/mc alias set myminio http://minio:10000 minioadmin minioadminpassword;
|
||||
/usr/bin/mc admin info myminio;
|
||||
/usr/bin/mc mb myminio/clickhouse;
|
||||
/usr/bin/mc policy set public myminio/clickhouse;
|
||||
exit 0;
|
||||
"
|
||||
nats:
|
||||
container_name: nats
|
||||
hostname: nats
|
||||
image: 'bitnami/nats:latest'
|
||||
ports:
|
||||
- 127.0.0.1:4222:4222
|
||||
- 127.0.0.1:6222:6222
|
||||
- 127.0.0.1:8222:8222
|
||||
environment:
|
||||
- NATS_ENABLE_AUTH=yes
|
||||
- NATS_USERNAME=natsadmin
|
||||
- NATS_PASSWORD=natsadmin
|
||||
#- NATS_TOKEN=natsadmin
|
||||
# healthcheck:
|
||||
# test: ["CMD", "curl", "-f", "http://localhost:4222"]
|
||||
# interval: 1m30s
|
||||
# timeout: 10s
|
||||
# retries: 3
|
||||
# start_period: 2m
|
||||
Loading…
Reference in New Issue
Block a user