From e69073ecad3f675f52405a85ba70f1e049118dbd Mon Sep 17 00:00:00 2001 From: kjuulh Date: Mon, 14 Feb 2022 20:03:53 +0100 Subject: [PATCH] Add docker --- .idea/sqldialects.xml | 6 - docker-compose.yml | 100 ++++++++++- services/db/README.md | 7 + .../db/migrations}/001_create_users.sql | 0 .../002_emails_are_unique_for_user.sql | 0 .../db/migrations}/003_add_projects.sql | 0 services/db/migrations/Dockerfile | 9 + .../db/migrations}/tern.conf | 0 services/db/migrations/tern.docker.conf | 34 ++++ services/db/migrations/wait-for-database.sh | 13 ++ services/entry/.dockerignore | 1 + services/entry/Dockerfile | 12 ++ go.mod => services/entry/go.mod | 0 go.sum => services/entry/go.sum | 0 main.go => services/entry/main.go | 6 +- .../entry/pkg}/application/projects/model.go | 0 .../pkg}/application/projects/repository.go | 0 .../pkg}/application/projects/service.go | 0 .../entry/pkg}/application/users/model.go | 0 .../pkg}/application/users/repository.go | 0 .../entry/pkg}/application/users/service.go | 0 .../entry/pkg}/application/users/user.go | 0 {pkg => services/entry/pkg}/db/db.go | 0 .../pkg}/db/postgres/projectsRepository.go | 0 .../entry/pkg}/db/postgres/usersRepository.go | 0 services/logs/loki/config.yaml | 56 ++++++ services/logs/promtail/config.yaml | 52 ++++++ services/metrics/grafana/config.monitoring | 2 + .../provisioning/dashboards/Default.json | 167 ++++++++++++++++++ .../provisioning/dashboards/dashboard.yml | 11 ++ .../provisioning/datasources/datasource.yaml | 30 ++++ services/metrics/prometheus/prometheus.yml | 12 ++ 32 files changed, 508 insertions(+), 10 deletions(-) delete mode 100644 .idea/sqldialects.xml create mode 100644 services/db/README.md rename {migrations => services/db/migrations}/001_create_users.sql (100%) rename {migrations => services/db/migrations}/002_emails_are_unique_for_user.sql (100%) rename {migrations => services/db/migrations}/003_add_projects.sql (100%) create mode 100644 services/db/migrations/Dockerfile rename {migrations => services/db/migrations}/tern.conf (100%) create mode 100644 services/db/migrations/tern.docker.conf create mode 100755 services/db/migrations/wait-for-database.sh create mode 100644 services/entry/.dockerignore create mode 100644 services/entry/Dockerfile rename go.mod => services/entry/go.mod (100%) rename go.sum => services/entry/go.sum (100%) rename main.go => services/entry/main.go (97%) rename {pkg => services/entry/pkg}/application/projects/model.go (100%) rename {pkg => services/entry/pkg}/application/projects/repository.go (100%) rename {pkg => services/entry/pkg}/application/projects/service.go (100%) rename {pkg => services/entry/pkg}/application/users/model.go (100%) rename {pkg => services/entry/pkg}/application/users/repository.go (100%) rename {pkg => services/entry/pkg}/application/users/service.go (100%) rename {pkg => services/entry/pkg}/application/users/user.go (100%) rename {pkg => services/entry/pkg}/db/db.go (100%) rename {pkg => services/entry/pkg}/db/postgres/projectsRepository.go (100%) rename {pkg => services/entry/pkg}/db/postgres/usersRepository.go (100%) create mode 100644 services/logs/loki/config.yaml create mode 100644 services/logs/promtail/config.yaml create mode 100644 services/metrics/grafana/config.monitoring create mode 100644 services/metrics/grafana/provisioning/dashboards/Default.json create mode 100644 services/metrics/grafana/provisioning/dashboards/dashboard.yml create mode 100644 services/metrics/grafana/provisioning/datasources/datasource.yaml create mode 100644 services/metrics/prometheus/prometheus.yml diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml deleted file mode 100644 index b980cf6..0000000 --- a/.idea/sqldialects.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index aba46d7..2d19d29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,116 @@ version: "3" +networks: + back-tier: + front-tier: + +x-logging: &loki-logging + driver: json-file + options: + tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}" + + services: + +# Database db: image: postgres restart: always volumes: - db_data:/var/lib/postgresql/data/pgdata ports: - - 5432:5432 + - "5432:5432" environment: PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_USER: serverctl POSTGRES_PASSWORD: serverctlsecret POSTGRES_DB: serverctl + networks: + - back-tier + + db_migrator: + build: + context: services/db/migrations + networks: + - back-tier + depends_on: + - "db" + + app: + build: + context: services/entry/ + networks: + - back-tier + - front-tier + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./services/entry/:/app/ + environment: + DATABASE_URL: "postgresql://serverctl:serverctlsecret@db/serverctl" + ports: + - "8080:8080" + logging: *loki-logging + depends_on: + - db_migrator + +# Logging + loki: + image: grafana/loki:2.4.2 + ports: + - 3100 + networks: + - back-tier + volumes: + - './services/logs/loki/config.yaml:/mnt/config/loki-config.yaml' + command: -config.file=/mnt/config/loki-config.yaml + logging: *loki-logging + + promtail: + image: grafana/promtail:2.4.2 + volumes: + - ./services/logs/promtail/config.yaml:/mnt/config/promtail-config.yaml + - /var/lib/docker/containers:/host/containers + command: -config.file /mnt/config/promtail-config.yaml + networks: + - back-tier + logging: *loki-logging + depends_on: + - loki + +#Metrics + prometheus: + image: prom/prometheus + volumes: + - ./services/metrics/prometheus/:/etc/prometheus + - prometheus_data:/prometheus + networks: + - back-tier + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + restart: always + + grafana: + image: grafana/grafana + user: "472" + depends_on: + - prometheus + ports: + - "3000:3000" + volumes: + - grafana_data:/var/lib/grafana + - ./services/metrics/grafana/provisioning:/etc/grafana/provisioning + env_file: + - ./services/metrics/grafana/config.monitoring + networks: + - back-tier + - front-tier + restart: always + volumes: db_data: {} + prometheus_data: {} + grafana_data: {} diff --git a/services/db/README.md b/services/db/README.md new file mode 100644 index 0000000..f2a87db --- /dev/null +++ b/services/db/README.md @@ -0,0 +1,7 @@ +# ServerCtl Database + +## Setup + +```bash +tern migrate +``` diff --git a/migrations/001_create_users.sql b/services/db/migrations/001_create_users.sql similarity index 100% rename from migrations/001_create_users.sql rename to services/db/migrations/001_create_users.sql diff --git a/migrations/002_emails_are_unique_for_user.sql b/services/db/migrations/002_emails_are_unique_for_user.sql similarity index 100% rename from migrations/002_emails_are_unique_for_user.sql rename to services/db/migrations/002_emails_are_unique_for_user.sql diff --git a/migrations/003_add_projects.sql b/services/db/migrations/003_add_projects.sql similarity index 100% rename from migrations/003_add_projects.sql rename to services/db/migrations/003_add_projects.sql diff --git a/services/db/migrations/Dockerfile b/services/db/migrations/Dockerfile new file mode 100644 index 0000000..abe3e50 --- /dev/null +++ b/services/db/migrations/Dockerfile @@ -0,0 +1,9 @@ +FROM golang:1.17-bullseye + +RUN go install github.com/jackc/tern@latest + +COPY . /app/migration/ + +WORKDIR /app/migration + +CMD ./wait-for-database.sh diff --git a/migrations/tern.conf b/services/db/migrations/tern.conf similarity index 100% rename from migrations/tern.conf rename to services/db/migrations/tern.conf diff --git a/services/db/migrations/tern.docker.conf b/services/db/migrations/tern.docker.conf new file mode 100644 index 0000000..8d628db --- /dev/null +++ b/services/db/migrations/tern.docker.conf @@ -0,0 +1,34 @@ +[database] +# host is required (network host or path to Unix domain socket) +host = db +port = 5432 +# database is required +database = serverctl +# user defaults to OS user +user = serverctl +password = serverctlsecret +version_table = public.schema_version +# +# sslmode generally matches the behavior described in: +# http://www.postgresql.org/docs/9.4/static/libpq-ssl.html#LIBPQ-SSL-PROTECTION +# +# There are only two modes that most users should use: +# prefer - on trusted networks where security is not required +# verify-full - require SSL connection +sslmode = prefer +# +# sslrootcert is generally used with sslmode=verify-full +# sslrootcert = /path/to/root/ca + +# Proxy the above database connection via SSH +# [ssh-tunnel] +# host = +# port = 22 +# user defaults to OS user +# user = +# password is not required if using SSH agent authentication +# password = + +[data] +# Any fields in the data section are available in migration templates +prefix = serverctl diff --git a/services/db/migrations/wait-for-database.sh b/services/db/migrations/wait-for-database.sh new file mode 100755 index 0000000..dac1285 --- /dev/null +++ b/services/db/migrations/wait-for-database.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# wait-for-postgres.sh + +set -e + +until tern status -c tern.docker.conf; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 +done + +>&2 echo "Postgres is up - executing command" + +tern migrate -c tern.docker.conf diff --git a/services/entry/.dockerignore b/services/entry/.dockerignore new file mode 100644 index 0000000..3fec32c --- /dev/null +++ b/services/entry/.dockerignore @@ -0,0 +1 @@ +tmp/ diff --git a/services/entry/Dockerfile b/services/entry/Dockerfile new file mode 100644 index 0000000..4e4a14b --- /dev/null +++ b/services/entry/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.17-bullseye + +RUN go install github.com/cosmtrek/air@latest +# Development don't need this +# COPY . /app/ +WORKDIR /app/ + +ARG DATABASE_URL +ENV DATABASE_URL DATABASE_URL + +#CMD go run main.go +CMD air diff --git a/go.mod b/services/entry/go.mod similarity index 100% rename from go.mod rename to services/entry/go.mod diff --git a/go.sum b/services/entry/go.sum similarity index 100% rename from go.sum rename to services/entry/go.sum diff --git a/main.go b/services/entry/main.go similarity index 97% rename from main.go rename to services/entry/main.go index 87fe703..c7ef98a 100644 --- a/main.go +++ b/services/entry/main.go @@ -37,13 +37,13 @@ func setupLogger() *zap.Logger { consoleErrors := zapcore.Lock(os.Stderr) fileEncoder := zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()) - consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) + _ = zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) core := zapcore.NewTee( zapcore.NewCore(fileEncoder, fileErrors, highPriority), - zapcore.NewCore(consoleEncoder, consoleErrors, highPriority), + zapcore.NewCore(fileEncoder, consoleErrors, highPriority), zapcore.NewCore(fileEncoder, fileDebugging, lowPriority), - zapcore.NewCore(consoleEncoder, consoleDebugging, lowPriority), + zapcore.NewCore(fileEncoder, consoleDebugging, lowPriority), ) logger := zap.New(core) diff --git a/pkg/application/projects/model.go b/services/entry/pkg/application/projects/model.go similarity index 100% rename from pkg/application/projects/model.go rename to services/entry/pkg/application/projects/model.go diff --git a/pkg/application/projects/repository.go b/services/entry/pkg/application/projects/repository.go similarity index 100% rename from pkg/application/projects/repository.go rename to services/entry/pkg/application/projects/repository.go diff --git a/pkg/application/projects/service.go b/services/entry/pkg/application/projects/service.go similarity index 100% rename from pkg/application/projects/service.go rename to services/entry/pkg/application/projects/service.go diff --git a/pkg/application/users/model.go b/services/entry/pkg/application/users/model.go similarity index 100% rename from pkg/application/users/model.go rename to services/entry/pkg/application/users/model.go diff --git a/pkg/application/users/repository.go b/services/entry/pkg/application/users/repository.go similarity index 100% rename from pkg/application/users/repository.go rename to services/entry/pkg/application/users/repository.go diff --git a/pkg/application/users/service.go b/services/entry/pkg/application/users/service.go similarity index 100% rename from pkg/application/users/service.go rename to services/entry/pkg/application/users/service.go diff --git a/pkg/application/users/user.go b/services/entry/pkg/application/users/user.go similarity index 100% rename from pkg/application/users/user.go rename to services/entry/pkg/application/users/user.go diff --git a/pkg/db/db.go b/services/entry/pkg/db/db.go similarity index 100% rename from pkg/db/db.go rename to services/entry/pkg/db/db.go diff --git a/pkg/db/postgres/projectsRepository.go b/services/entry/pkg/db/postgres/projectsRepository.go similarity index 100% rename from pkg/db/postgres/projectsRepository.go rename to services/entry/pkg/db/postgres/projectsRepository.go diff --git a/pkg/db/postgres/usersRepository.go b/services/entry/pkg/db/postgres/usersRepository.go similarity index 100% rename from pkg/db/postgres/usersRepository.go rename to services/entry/pkg/db/postgres/usersRepository.go diff --git a/services/logs/loki/config.yaml b/services/logs/loki/config.yaml new file mode 100644 index 0000000..5bbae68 --- /dev/null +++ b/services/logs/loki/config.yaml @@ -0,0 +1,56 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +ingester: + wal: + enabled: true + dir: /tmp/wal + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed + max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h + chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first + chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) + max_transfer_retries: 0 # Chunk transfers disabled + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: /tmp/loki/boltdb-shipper-active + cache_location: /tmp/loki/boltdb-shipper-cache + cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space + shared_store: filesystem + filesystem: + directory: /tmp/loki/chunks + +compactor: + working_directory: /tmp/loki/boltdb-shipper-compactor + shared_store: filesystem + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: true + retention_period: 24h diff --git a/services/logs/promtail/config.yaml b/services/logs/promtail/config.yaml new file mode 100644 index 0000000..254b8bf --- /dev/null +++ b/services/logs/promtail/config.yaml @@ -0,0 +1,52 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + # - job_name: system + # static_configs: + # - targets: + # - localhost + # labels: + # job: varlogs + # __path__: /var/log/*log + - job_name: docker + static_configs: + - targets: + - localhost + labels: + job: dockerlogs + __path__: /host/containers/*/*log + pipeline_stages: + - json: + expressions: + output: log + stream: stream + attrs: + - json: + expressions: + tag: + source: attrs + - regex: + expression: (?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])) + source: tag + - timestamp: + format: RFC3339Nano + source: time + - labels: + tag: + stream: + image_name: + container_name: + image_id: + container_id: + - output: + source: output + + diff --git a/services/metrics/grafana/config.monitoring b/services/metrics/grafana/config.monitoring new file mode 100644 index 0000000..70cea8d --- /dev/null +++ b/services/metrics/grafana/config.monitoring @@ -0,0 +1,2 @@ +GF_SECURITY_ADMIN_PASSWORD=serverctlsecret +GF_USERS_ALLOW_SIGN_UP=false diff --git a/services/metrics/grafana/provisioning/dashboards/Default.json b/services/metrics/grafana/provisioning/dashboards/Default.json new file mode 100644 index 0000000..5782336 --- /dev/null +++ b/services/metrics/grafana/provisioning/dashboards/Default.json @@ -0,0 +1,167 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "PEA2100DC89AE9FE2" + }, + "gridPos": { + "h": 19, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "dedupStrategy": "numbers", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "PEA2100DC89AE9FE2" + }, + "expr": "{container_name=\"serverctl-app-1\"}", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "app", + "type": "logs" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PA58DA793C7250F1B" + }, + "exemplar": true, + "expr": "scrape_duration_seconds{}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Panel Title", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 34, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Default", + "uid": "U8c_qq-7k", + "version": 1, + "weekStart": "" +} diff --git a/services/metrics/grafana/provisioning/dashboards/dashboard.yml b/services/metrics/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..51547d4 --- /dev/null +++ b/services/metrics/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: "Metrics" + orgId: 1 + folder: '' + type: file + disableDeletions: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/services/metrics/grafana/provisioning/datasources/datasource.yaml b/services/metrics/grafana/provisioning/datasources/datasource.yaml new file mode 100644 index 0000000..d03e6ba --- /dev/null +++ b/services/metrics/grafana/provisioning/datasources/datasource.yaml @@ -0,0 +1,30 @@ +apiVersion: 1 + +deleteDatasources: + - name: Metrics + orgId: 1 + - name: Logs + orgId: 1 + +datasources: + - name: Metrics + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: true + jsonData: + graphiteVersion: "1.1" + tlsAuth: false + tlsAuthWithCACert: false + version: 1 + editable: false + + - name: Logs + type: loki + access: proxy + url: http://loki:3100 + jsonData: + maxLines: 1000 + orgId: 1 diff --git a/services/metrics/prometheus/prometheus.yml b/services/metrics/prometheus/prometheus.yml new file mode 100644 index 0000000..11e42cb --- /dev/null +++ b/services/metrics/prometheus/prometheus.yml @@ -0,0 +1,12 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + monitor: "serverctl" + +scrape_configs: + - job_name: 'prometheus' + scrape_interval: 5s + static_configs: + - targets: ["localhost:9090"] +