feat: add basic ingest from spark iceberg example
This commit is contained in:
commit
d27096896b
SSH Key Fingerprint:
SHA256:RjXh0p7U6opxnfd3ga/Y9TCo18FYlHFdSpRIV72S/QM
2
.drone.yml
Normal file
2
.drone.yml
Normal file
@ -0,0 +1,2 @@
|
||||
kind: template
|
||||
load: cuddle-empty-plan.yaml
|
||||
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
.cuddle/
|
||||
16
cuddle.yaml
Normal file
16
cuddle.yaml
Normal file
@ -0,0 +1,16 @@
|
||||
# yaml-language-server: $schema=https://git.front.kjuulh.io/kjuulh/cuddle/raw/branch/main/schemas/base.json
|
||||
|
||||
base: "git@git.front.kjuulh.io:kjuulh/cuddle-empty-plan.git"
|
||||
|
||||
vars:
|
||||
service: "data-platform-in-a-box"
|
||||
registry: kasperhermansen
|
||||
|
||||
please:
|
||||
project:
|
||||
owner: kjuulh
|
||||
repository: "data-platform-in-a-box"
|
||||
branch: main
|
||||
settings:
|
||||
api_url: https://git.front.kjuulh.io
|
||||
|
||||
212
docker-compose.yml
Normal file
212
docker-compose.yml
Normal file
@ -0,0 +1,212 @@
|
||||
services:
|
||||
redpanda:
|
||||
image: redpandadata/redpanda:v24.3.4
|
||||
container_name: redpanda
|
||||
ports:
|
||||
- "9092:9092"
|
||||
- "8081:8081"
|
||||
- "8082:8082"
|
||||
- "29092:29092"
|
||||
command:
|
||||
- redpanda
|
||||
- start
|
||||
- --overprovisioned
|
||||
- --smp
|
||||
- "1"
|
||||
- --memory
|
||||
- "1G"
|
||||
- --reserve-memory
|
||||
- "0M"
|
||||
- --node-id
|
||||
- "0"
|
||||
- --kafka-addr
|
||||
- PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
|
||||
- --advertise-kafka-addr
|
||||
- PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092
|
||||
- --check=false
|
||||
|
||||
connect:
|
||||
image: confluentinc/cp-kafka-connect-base:7.8.0
|
||||
depends_on:
|
||||
- redpanda
|
||||
hostname: connect
|
||||
container_name: connect
|
||||
ports:
|
||||
- 8083:8083
|
||||
environment:
|
||||
CONNECT_BOOTSTRAP_SERVERS: 'redpanda:29092'
|
||||
CONNECT_REST_ADVERTISED_HOST_NAME: "connect"
|
||||
CONNECT_REST_PORT: 8083
|
||||
CONNECT_GROUP_ID: connect-cluster-group
|
||||
CONNECT_CONFIG_STORAGE_TOPIC: _kafka-connect-configs
|
||||
CONNECT_OFFSET_STORAGE_TOPIC: _kafka-connect-offsets
|
||||
CONNECT_STATUS_STORAGE_TOPIC: _kafka-connect-status
|
||||
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
|
||||
CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false"
|
||||
CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
|
||||
CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
|
||||
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.StringConverter"
|
||||
CONNECT_INTERNAL_KEY_CONVERTER_SCHEMAS_ENABLE: "false"
|
||||
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
|
||||
CONNECT_INTERNAL_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
|
||||
CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
|
||||
CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
|
||||
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
|
||||
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
|
||||
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
|
||||
CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/,/connectors/'
|
||||
AWS_ACCESS_KEY_ID: "minioadmin"
|
||||
AWS_SECRET_ACCESS_KEY: "minioadmin"
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- |
|
||||
#
|
||||
echo "Installing connector plugins"
|
||||
confluent-hub install --no-prompt tabular/iceberg-kafka-connect:0.4.11
|
||||
#
|
||||
echo "Launching Kafka Connect worker"
|
||||
/etc/confluent/docker/run &
|
||||
#
|
||||
echo "Waiting for Kafka Connect to start listening on localhost ⏳"
|
||||
while : ; do
|
||||
curl_status=$$(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors)
|
||||
echo -e $$(date) " Kafka Connect listener HTTP state: " $$curl_status " (waiting for 200)"
|
||||
if [ $$curl_status -eq 200 ] ; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo -e "\n--\n+> Creating connector"
|
||||
curl -X PUT \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Accept: application/json' http://localhost:8083/connectors/IcebergSinkConnector/config \
|
||||
-d '{
|
||||
"tasks.max": "1",
|
||||
"topics": "payments",
|
||||
"connector.class": "io.tabular.iceberg.connect.IcebergSinkConnector",
|
||||
"iceberg.catalog.s3.endpoint": "http://minio:9000",
|
||||
"iceberg.catalog.s3.secret-access-key": "minioadmin",
|
||||
"iceberg.catalog.s3.access-key-id": "minioadmin",
|
||||
"iceberg.catalog.s3.path-style-access": "true",
|
||||
"iceberg.catalog.uri": "http://rest:8181",
|
||||
"iceberg.catalog.warehouse": "s3://warehouse/",
|
||||
"iceberg.catalog.client.region": "eu-west-1",
|
||||
"iceberg.catalog.type": "rest",
|
||||
"iceberg.control.commitIntervalMs": "1000",
|
||||
"iceberg.tables": "orders.payments",
|
||||
"value.converter.schemas.enable": "false",
|
||||
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
|
||||
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
|
||||
"schemas.enable": "false"
|
||||
}'
|
||||
sleep infinity
|
||||
|
||||
console:
|
||||
image: redpandadata/console:v2.1.1
|
||||
entrypoint: /bin/sh
|
||||
command: -c "echo \"$$CONSOLE_CONFIG_FILE\" > /tmp/config.yml; /app/console"
|
||||
environment:
|
||||
CONFIG_FILEPATH: /tmp/config.yml
|
||||
CONSOLE_CONFIG_FILE: |
|
||||
kafka:
|
||||
brokers: ["redpanda:29092"]
|
||||
schemaRegistry:
|
||||
enabled: true
|
||||
urls: ["http://redpanda:8081"]
|
||||
connect:
|
||||
enabled: true
|
||||
clusters:
|
||||
- name: local-connect-cluster
|
||||
url: http://connect:8083
|
||||
redpanda:
|
||||
adminApi:
|
||||
enabled: true
|
||||
urls: ["http://redpanda:9644"]
|
||||
ports:
|
||||
- 18080:8080
|
||||
depends_on:
|
||||
- redpanda
|
||||
- connect
|
||||
|
||||
minio:
|
||||
image: minio/minio
|
||||
hostname: minio
|
||||
container_name: minio
|
||||
environment:
|
||||
- MINIO_ROOT_USER=minioadmin
|
||||
- MINIO_ROOT_PASSWORD=minioadmin
|
||||
- MINIO_DOMAIN=minio
|
||||
networks:
|
||||
default:
|
||||
aliases:
|
||||
- warehouse.minio
|
||||
ports:
|
||||
- 9001:9001
|
||||
- 9000:9000
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
|
||||
aws:
|
||||
image: amazon/aws-cli
|
||||
container_name: aws-cli
|
||||
command: |
|
||||
-c "sleep 2 && \
|
||||
aws --endpoint-url http://minio:9000 s3 mb s3://warehouse --region eu-west-1 || exit 0"
|
||||
entrypoint: [/bin/bash]
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID: "minioadmin"
|
||||
AWS_SECRET_ACCESS_KEY: "minioadmin"
|
||||
depends_on:
|
||||
- minio
|
||||
|
||||
spark-iceberg:
|
||||
image: tabulario/spark-iceberg
|
||||
hostname: spark-iceberg
|
||||
container_name: spark-iceberg
|
||||
build: spark/
|
||||
depends_on:
|
||||
- rest
|
||||
- minio
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID: minioadmin
|
||||
AWS_SECRET_ACCESS_KEY: minioadmin
|
||||
AWS_REGION: eu-west-1
|
||||
SPARK_DEFAULTS: |
|
||||
spark.sql.extensions org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
|
||||
spark.sql.catalog.iceberg org.apache.iceberg.spark.SparkCatalog
|
||||
spark.sql.catalog.iceberg.catalog-impl org.apache.iceberg.rest.RESTCatalog
|
||||
spark.sql.catalog.iceberg.uri http://rest:8181
|
||||
spark.sql.catalog.iceberg.io-impl org.apache.iceberg.aws.s3.S3FileIO
|
||||
spark.sql.catalog.iceberg.warehouse s3://warehouse/wh/
|
||||
spark.sql.catalog.iceberg.s3.endpoint http://minio:9000
|
||||
spark.sql.catalog.iceberg.s3.path-style-access true
|
||||
spark.sql.defaultCatalog iceberg
|
||||
spark.sql.catalogImplementation in-memory
|
||||
spark.eventLog.enabled true
|
||||
spark.eventLog.dir /home/iceberg/spark-events
|
||||
spark.history.fs.logDirectory /home/iceberg/spark-events
|
||||
spark.jars.packages org.apache.hadoop:hadoop-aws:3.2.0
|
||||
ports:
|
||||
- 8888:8888
|
||||
- 8080:8080
|
||||
- 10000:10000
|
||||
- 10001:10001
|
||||
volumes:
|
||||
- ./spark:/home/iceberg/scripts
|
||||
- ./notebooks:/home/iceberg/notebooks/notebooks
|
||||
command: ["echo \"$$SPARK_DEFAULTS\" > /opt/spark/conf/spark-defaults.conf && spark-submit /home/iceberg/scripts/create_table.py && notebook"]
|
||||
|
||||
rest:
|
||||
image: tabulario/iceberg-rest
|
||||
hostname: rest
|
||||
container_name: rest
|
||||
ports:
|
||||
- 8181:8181
|
||||
environment:
|
||||
- AWS_ACCESS_KEY_ID=minioadmin
|
||||
- AWS_SECRET_ACCESS_KEY=minioadmin
|
||||
- AWS_REGION=eu-west-1
|
||||
- CATALOG_WAREHOUSE=s3://warehouse/
|
||||
- CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
|
||||
- CATALOG_S3_ENDPOINT=http://minio:9000
|
||||
- CATALOG_S3_PATH__STYLE__ACCESS=True
|
||||
103
notebooks/.ipynb_checkpoints/iceberg-checkpoint.ipynb
Normal file
103
notebooks/.ipynb_checkpoints/iceberg-checkpoint.ipynb
Normal file
@ -0,0 +1,103 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "29f8d24e-e4bf-484d-afd4-cb82ff6cd50d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SHOW DATABASES"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "70349765-e5f1-43a5-a141-cc2d54c69a58",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SHOW TABLES FROM orders"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "fabaed9c-9049-4996-9d26-b20f66303911",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SHOW TBLPROPERTIES orders.payments"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6317d9c6-140e-4a63-890e-2173fbb9503e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SELECT COUNT(*)\n",
|
||||
"FROM orders.payments"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2a1ff132-dc65-4943-a9be-416ba5a13c26",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SELECT *\n",
|
||||
"FROM orders.payments\n",
|
||||
"LIMIT 10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a2688a95-594c-45ad-9d49-70a1bcd59a1b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SELECT * \n",
|
||||
"FROM orders.payments.partitions\n",
|
||||
"ORDER BY record_count DESC\n",
|
||||
"LIMIT 10"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.17"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
103
notebooks/iceberg.ipynb
Normal file
103
notebooks/iceberg.ipynb
Normal file
@ -0,0 +1,103 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "29f8d24e-e4bf-484d-afd4-cb82ff6cd50d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SHOW DATABASES"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "70349765-e5f1-43a5-a141-cc2d54c69a58",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SHOW TABLES FROM orders"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "fabaed9c-9049-4996-9d26-b20f66303911",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SHOW TBLPROPERTIES orders.payments"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6317d9c6-140e-4a63-890e-2173fbb9503e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SELECT COUNT(*)\n",
|
||||
"FROM orders.payments"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2a1ff132-dc65-4943-a9be-416ba5a13c26",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SELECT *\n",
|
||||
"FROM orders.payments\n",
|
||||
"LIMIT 10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a2688a95-594c-45ad-9d49-70a1bcd59a1b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"\n",
|
||||
"SELECT * \n",
|
||||
"FROM orders.payments.partitions\n",
|
||||
"ORDER BY record_count DESC\n",
|
||||
"LIMIT 10"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.17"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
3
renovate.json
Normal file
3
renovate.json
Normal file
@ -0,0 +1,3 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json"
|
||||
}
|
||||
20
spark/create_table.py
Normal file
20
spark/create_table.py
Normal file
@ -0,0 +1,20 @@
|
||||
from pyspark.sql import SparkSession
|
||||
|
||||
spark = SparkSession.builder.appName("").getOrCreate()
|
||||
|
||||
print("creating database")
|
||||
spark.sql('CREATE DATABASE IF NOT EXISTS orders')
|
||||
|
||||
print("creating table")
|
||||
spark.sql('''
|
||||
CREATE TABLE IF NOT EXISTS orders.payments (
|
||||
id STRING,
|
||||
type STRING,
|
||||
created_at TIMESTAMP,
|
||||
document STRING,
|
||||
payer STRING,
|
||||
amount INT
|
||||
)
|
||||
USING iceberg
|
||||
PARTITIONED BY (document)
|
||||
''')
|
||||
Loading…
x
Reference in New Issue
Block a user