From 026a95790e7ad0ab6dc5ab60e4a238ffef4f2878 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Tue, 6 Aug 2024 00:02:20 +0200 Subject: [PATCH] feat: add presto Signed-off-by: kjuulh --- connect-to-presto.sh | 5 +++++ docker-exec.sh | 1 + prepare.sh | 13 +++++++++++++ templates/docker-compose.iceberg.yaml | 1 + templates/docker-compose.trino.yaml | 20 ++++++++++++++++++++ templates/hadoop-catalog.xml | 22 ++++++++++++++++++++++ templates/iceberg.properties | 5 +++++ templates/log.properties | 2 ++ templates/presto.Dockerfile | 5 +++++ 9 files changed, 74 insertions(+) create mode 100755 connect-to-presto.sh create mode 100755 prepare.sh create mode 100644 templates/docker-compose.trino.yaml create mode 100644 templates/hadoop-catalog.xml create mode 100644 templates/iceberg.properties create mode 100644 templates/log.properties create mode 100644 templates/presto.Dockerfile diff --git a/connect-to-presto.sh b/connect-to-presto.sh new file mode 100755 index 0000000..c33f7df --- /dev/null +++ b/connect-to-presto.sh @@ -0,0 +1,5 @@ +#!/bin/zsh + +set -eu + +./docker-exec.sh exec presto presto-cli --server localhost:8080 --execute 'select count(*) from iceberg.demo_db.demo_table;' diff --git a/docker-exec.sh b/docker-exec.sh index 4d83cf5..55f377f 100755 --- a/docker-exec.sh +++ b/docker-exec.sh @@ -4,5 +4,6 @@ docker compose \ -f templates/docker-compose.rising-wave.yaml \ -f templates/docker-compose.iceberg.yaml \ -f templates/docker-compose.clickhouse.yaml \ + -f templates/docker-compose.trino.yaml \ -f templates/docker-compose.yaml \ $@ diff --git a/prepare.sh b/prepare.sh new file mode 100755 index 0000000..2031947 --- /dev/null +++ b/prepare.sh @@ -0,0 +1,13 @@ +#!/bin/zsh + +set -eu + + +psql -h 0.0.0.0 -p 4566 -d dev -U root -f templates/create_sources.sql || true +psql -h 0.0.0.0 -p 4566 -d dev -U root -f templates/create_mvs.sql || true +psql -h 0.0.0.0 -p 4566 -d dev -U root -f templates/create_create_iceberg_sink.sql || true +psql -h 0.0.0.0 -p 4566 -d dev -U root -f templates/create_create_clickhouse_sink.sql || true + +./docker-exec.sh exec spark bash /spark-script/run_sql.sh create-table +./docker-exec.sh exec spark bash /spark-script/run_sql.sh query-table + diff --git a/templates/docker-compose.iceberg.yaml b/templates/docker-compose.iceberg.yaml index e151b28..637edd0 100644 --- a/templates/docker-compose.iceberg.yaml +++ b/templates/docker-compose.iceberg.yaml @@ -29,6 +29,7 @@ services: - SPARK_MASTER_URL=spark://spark:7077 - SPARK_WORKER_MEMORY=1G - SPARK_WORKER_CORES=1 + configs: run_sql: content: | diff --git a/templates/docker-compose.trino.yaml b/templates/docker-compose.trino.yaml new file mode 100644 index 0000000..714aa85 --- /dev/null +++ b/templates/docker-compose.trino.yaml @@ -0,0 +1,20 @@ +services: + presto: + build: + context: . + dockerfile: presto.Dockerfile + container_name: presto + +# trino: +# image: trinodb/trino +# ports: +# - 18080:8080 +# configs: +# - source: iceberg_properties +# target: /etc/catalog/iceberg.properties +# configs: +# iceberg_properties: +# content: | +# connector.name=iceberg +# hive.metastore.uri=thrift://192.0.2.3:9083 + diff --git a/templates/hadoop-catalog.xml b/templates/hadoop-catalog.xml new file mode 100644 index 0000000..669c6d0 --- /dev/null +++ b/templates/hadoop-catalog.xml @@ -0,0 +1,22 @@ + + + presto.s3.endpoint + http://minio-0:9301 + + + presto.s3.path-style-access + true + + + presto.s3.access-key + hummockadmin + + + presto.s3.secret-key + hummockadmin + + + fs.s3a.impl + com.facebook.presto.hive.s3.PrestoS3FileSystem + + diff --git a/templates/iceberg.properties b/templates/iceberg.properties new file mode 100644 index 0000000..ce4307d --- /dev/null +++ b/templates/iceberg.properties @@ -0,0 +1,5 @@ +connector.name=iceberg +iceberg.catalog.type=hadoop +iceberg.catalog.warehouse=s3a://hummock001/iceberg-data +iceberg.hadoop.config.resources=/etc/iceberg/conf/hadoop-catalog.xml +hive.config.resources=/etc/iceberg/conf/hadoop-catalog.xml diff --git a/templates/log.properties b/templates/log.properties new file mode 100644 index 0000000..ca2af4c --- /dev/null +++ b/templates/log.properties @@ -0,0 +1,2 @@ +com.facebook.presto=DEBUG +org.apache.hadoop=DEBUG diff --git a/templates/presto.Dockerfile b/templates/presto.Dockerfile new file mode 100644 index 0000000..f083216 --- /dev/null +++ b/templates/presto.Dockerfile @@ -0,0 +1,5 @@ +FROM prestodb/presto + +COPY ./iceberg.properties /opt/presto-server/etc/catalog +COPY ./log.properties /opt/presto-server/etc +COPY ./hadoop-catalog.xml /etc/iceberg/conf/hadoop-catalog.xml