diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml
deleted file mode 100644
index b980cf6..0000000
--- a/.idea/sqldialects.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index aba46d7..2d19d29 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,18 +1,116 @@
version: "3"
+networks:
+ back-tier:
+ front-tier:
+
+x-logging: &loki-logging
+ driver: json-file
+ options:
+ tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
+
+
services:
+
+# Database
db:
image: postgres
restart: always
volumes:
- db_data:/var/lib/postgresql/data/pgdata
ports:
- - 5432:5432
+ - "5432:5432"
environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_USER: serverctl
POSTGRES_PASSWORD: serverctlsecret
POSTGRES_DB: serverctl
+ networks:
+ - back-tier
+
+ db_migrator:
+ build:
+ context: services/db/migrations
+ networks:
+ - back-tier
+ depends_on:
+ - "db"
+
+ app:
+ build:
+ context: services/entry/
+ networks:
+ - back-tier
+ - front-tier
+ volumes:
+ - /var/run/docker.sock:/var/run/docker.sock
+ - ./services/entry/:/app/
+ environment:
+ DATABASE_URL: "postgresql://serverctl:serverctlsecret@db/serverctl"
+ ports:
+ - "8080:8080"
+ logging: *loki-logging
+ depends_on:
+ - db_migrator
+
+# Logging
+ loki:
+ image: grafana/loki:2.4.2
+ ports:
+ - 3100
+ networks:
+ - back-tier
+ volumes:
+ - './services/logs/loki/config.yaml:/mnt/config/loki-config.yaml'
+ command: -config.file=/mnt/config/loki-config.yaml
+ logging: *loki-logging
+
+ promtail:
+ image: grafana/promtail:2.4.2
+ volumes:
+ - ./services/logs/promtail/config.yaml:/mnt/config/promtail-config.yaml
+ - /var/lib/docker/containers:/host/containers
+ command: -config.file /mnt/config/promtail-config.yaml
+ networks:
+ - back-tier
+ logging: *loki-logging
+ depends_on:
+ - loki
+
+#Metrics
+ prometheus:
+ image: prom/prometheus
+ volumes:
+ - ./services/metrics/prometheus/:/etc/prometheus
+ - prometheus_data:/prometheus
+ networks:
+ - back-tier
+ command:
+ - '--config.file=/etc/prometheus/prometheus.yml'
+ - '--storage.tsdb.path=/prometheus'
+ - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+ - '--web.console.templates=/usr/share/prometheus/consoles'
+ restart: always
+
+ grafana:
+ image: grafana/grafana
+ user: "472"
+ depends_on:
+ - prometheus
+ ports:
+ - "3000:3000"
+ volumes:
+ - grafana_data:/var/lib/grafana
+ - ./services/metrics/grafana/provisioning:/etc/grafana/provisioning
+ env_file:
+ - ./services/metrics/grafana/config.monitoring
+ networks:
+ - back-tier
+ - front-tier
+ restart: always
+
volumes:
db_data: {}
+ prometheus_data: {}
+ grafana_data: {}
diff --git a/services/db/README.md b/services/db/README.md
new file mode 100644
index 0000000..f2a87db
--- /dev/null
+++ b/services/db/README.md
@@ -0,0 +1,7 @@
+# ServerCtl Database
+
+## Setup
+
+```bash
+tern migrate
+```
diff --git a/migrations/001_create_users.sql b/services/db/migrations/001_create_users.sql
similarity index 100%
rename from migrations/001_create_users.sql
rename to services/db/migrations/001_create_users.sql
diff --git a/migrations/002_emails_are_unique_for_user.sql b/services/db/migrations/002_emails_are_unique_for_user.sql
similarity index 100%
rename from migrations/002_emails_are_unique_for_user.sql
rename to services/db/migrations/002_emails_are_unique_for_user.sql
diff --git a/migrations/003_add_projects.sql b/services/db/migrations/003_add_projects.sql
similarity index 100%
rename from migrations/003_add_projects.sql
rename to services/db/migrations/003_add_projects.sql
diff --git a/services/db/migrations/Dockerfile b/services/db/migrations/Dockerfile
new file mode 100644
index 0000000..abe3e50
--- /dev/null
+++ b/services/db/migrations/Dockerfile
@@ -0,0 +1,9 @@
+FROM golang:1.17-bullseye
+
+RUN go install github.com/jackc/tern@latest
+
+COPY . /app/migration/
+
+WORKDIR /app/migration
+
+CMD ./wait-for-database.sh
diff --git a/migrations/tern.conf b/services/db/migrations/tern.conf
similarity index 100%
rename from migrations/tern.conf
rename to services/db/migrations/tern.conf
diff --git a/services/db/migrations/tern.docker.conf b/services/db/migrations/tern.docker.conf
new file mode 100644
index 0000000..8d628db
--- /dev/null
+++ b/services/db/migrations/tern.docker.conf
@@ -0,0 +1,34 @@
+[database]
+# host is required (network host or path to Unix domain socket)
+host = db
+port = 5432
+# database is required
+database = serverctl
+# user defaults to OS user
+user = serverctl
+password = serverctlsecret
+version_table = public.schema_version
+#
+# sslmode generally matches the behavior described in:
+# http://www.postgresql.org/docs/9.4/static/libpq-ssl.html#LIBPQ-SSL-PROTECTION
+#
+# There are only two modes that most users should use:
+# prefer - on trusted networks where security is not required
+# verify-full - require SSL connection
+sslmode = prefer
+#
+# sslrootcert is generally used with sslmode=verify-full
+# sslrootcert = /path/to/root/ca
+
+# Proxy the above database connection via SSH
+# [ssh-tunnel]
+# host =
+# port = 22
+# user defaults to OS user
+# user =
+# password is not required if using SSH agent authentication
+# password =
+
+[data]
+# Any fields in the data section are available in migration templates
+prefix = serverctl
diff --git a/services/db/migrations/wait-for-database.sh b/services/db/migrations/wait-for-database.sh
new file mode 100755
index 0000000..dac1285
--- /dev/null
+++ b/services/db/migrations/wait-for-database.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+# wait-for-postgres.sh
+
+set -e
+
+until tern status -c tern.docker.conf; do
+ >&2 echo "Postgres is unavailable - sleeping"
+ sleep 1
+done
+
+>&2 echo "Postgres is up - executing command"
+
+tern migrate -c tern.docker.conf
diff --git a/services/entry/.dockerignore b/services/entry/.dockerignore
new file mode 100644
index 0000000..3fec32c
--- /dev/null
+++ b/services/entry/.dockerignore
@@ -0,0 +1 @@
+tmp/
diff --git a/services/entry/Dockerfile b/services/entry/Dockerfile
new file mode 100644
index 0000000..4e4a14b
--- /dev/null
+++ b/services/entry/Dockerfile
@@ -0,0 +1,12 @@
+FROM golang:1.17-bullseye
+
+RUN go install github.com/cosmtrek/air@latest
+# Development don't need this
+# COPY . /app/
+WORKDIR /app/
+
+ARG DATABASE_URL
+ENV DATABASE_URL DATABASE_URL
+
+#CMD go run main.go
+CMD air
diff --git a/go.mod b/services/entry/go.mod
similarity index 100%
rename from go.mod
rename to services/entry/go.mod
diff --git a/go.sum b/services/entry/go.sum
similarity index 100%
rename from go.sum
rename to services/entry/go.sum
diff --git a/main.go b/services/entry/main.go
similarity index 97%
rename from main.go
rename to services/entry/main.go
index 87fe703..c7ef98a 100644
--- a/main.go
+++ b/services/entry/main.go
@@ -37,13 +37,13 @@ func setupLogger() *zap.Logger {
consoleErrors := zapcore.Lock(os.Stderr)
fileEncoder := zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig())
- consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig())
+ _ = zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig())
core := zapcore.NewTee(
zapcore.NewCore(fileEncoder, fileErrors, highPriority),
- zapcore.NewCore(consoleEncoder, consoleErrors, highPriority),
+ zapcore.NewCore(fileEncoder, consoleErrors, highPriority),
zapcore.NewCore(fileEncoder, fileDebugging, lowPriority),
- zapcore.NewCore(consoleEncoder, consoleDebugging, lowPriority),
+ zapcore.NewCore(fileEncoder, consoleDebugging, lowPriority),
)
logger := zap.New(core)
diff --git a/pkg/application/projects/model.go b/services/entry/pkg/application/projects/model.go
similarity index 100%
rename from pkg/application/projects/model.go
rename to services/entry/pkg/application/projects/model.go
diff --git a/pkg/application/projects/repository.go b/services/entry/pkg/application/projects/repository.go
similarity index 100%
rename from pkg/application/projects/repository.go
rename to services/entry/pkg/application/projects/repository.go
diff --git a/pkg/application/projects/service.go b/services/entry/pkg/application/projects/service.go
similarity index 100%
rename from pkg/application/projects/service.go
rename to services/entry/pkg/application/projects/service.go
diff --git a/pkg/application/users/model.go b/services/entry/pkg/application/users/model.go
similarity index 100%
rename from pkg/application/users/model.go
rename to services/entry/pkg/application/users/model.go
diff --git a/pkg/application/users/repository.go b/services/entry/pkg/application/users/repository.go
similarity index 100%
rename from pkg/application/users/repository.go
rename to services/entry/pkg/application/users/repository.go
diff --git a/pkg/application/users/service.go b/services/entry/pkg/application/users/service.go
similarity index 100%
rename from pkg/application/users/service.go
rename to services/entry/pkg/application/users/service.go
diff --git a/pkg/application/users/user.go b/services/entry/pkg/application/users/user.go
similarity index 100%
rename from pkg/application/users/user.go
rename to services/entry/pkg/application/users/user.go
diff --git a/pkg/db/db.go b/services/entry/pkg/db/db.go
similarity index 100%
rename from pkg/db/db.go
rename to services/entry/pkg/db/db.go
diff --git a/pkg/db/postgres/projectsRepository.go b/services/entry/pkg/db/postgres/projectsRepository.go
similarity index 100%
rename from pkg/db/postgres/projectsRepository.go
rename to services/entry/pkg/db/postgres/projectsRepository.go
diff --git a/pkg/db/postgres/usersRepository.go b/services/entry/pkg/db/postgres/usersRepository.go
similarity index 100%
rename from pkg/db/postgres/usersRepository.go
rename to services/entry/pkg/db/postgres/usersRepository.go
diff --git a/services/logs/loki/config.yaml b/services/logs/loki/config.yaml
new file mode 100644
index 0000000..5bbae68
--- /dev/null
+++ b/services/logs/loki/config.yaml
@@ -0,0 +1,56 @@
+auth_enabled: false
+
+server:
+ http_listen_port: 3100
+ grpc_listen_port: 9096
+
+ingester:
+ wal:
+ enabled: true
+ dir: /tmp/wal
+ lifecycler:
+ address: 127.0.0.1
+ ring:
+ kvstore:
+ store: inmemory
+ replication_factor: 1
+ final_sleep: 0s
+ chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed
+ max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h
+ chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
+ chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+ max_transfer_retries: 0 # Chunk transfers disabled
+
+schema_config:
+ configs:
+ - from: 2020-10-24
+ store: boltdb-shipper
+ object_store: filesystem
+ schema: v11
+ index:
+ prefix: index_
+ period: 24h
+
+storage_config:
+ boltdb_shipper:
+ active_index_directory: /tmp/loki/boltdb-shipper-active
+ cache_location: /tmp/loki/boltdb-shipper-cache
+ cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
+ shared_store: filesystem
+ filesystem:
+ directory: /tmp/loki/chunks
+
+compactor:
+ working_directory: /tmp/loki/boltdb-shipper-compactor
+ shared_store: filesystem
+
+limits_config:
+ reject_old_samples: true
+ reject_old_samples_max_age: 168h
+
+chunk_store_config:
+ max_look_back_period: 0s
+
+table_manager:
+ retention_deletes_enabled: true
+ retention_period: 24h
diff --git a/services/logs/promtail/config.yaml b/services/logs/promtail/config.yaml
new file mode 100644
index 0000000..254b8bf
--- /dev/null
+++ b/services/logs/promtail/config.yaml
@@ -0,0 +1,52 @@
+server:
+ http_listen_port: 9080
+ grpc_listen_port: 0
+
+positions:
+ filename: /tmp/positions.yaml
+
+clients:
+ - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+ # - job_name: system
+ # static_configs:
+ # - targets:
+ # - localhost
+ # labels:
+ # job: varlogs
+ # __path__: /var/log/*log
+ - job_name: docker
+ static_configs:
+ - targets:
+ - localhost
+ labels:
+ job: dockerlogs
+ __path__: /host/containers/*/*log
+ pipeline_stages:
+ - json:
+ expressions:
+ output: log
+ stream: stream
+ attrs:
+ - json:
+ expressions:
+ tag:
+ source: attrs
+ - regex:
+ expression: (?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|]))
+ source: tag
+ - timestamp:
+ format: RFC3339Nano
+ source: time
+ - labels:
+ tag:
+ stream:
+ image_name:
+ container_name:
+ image_id:
+ container_id:
+ - output:
+ source: output
+
+
diff --git a/services/metrics/grafana/config.monitoring b/services/metrics/grafana/config.monitoring
new file mode 100644
index 0000000..70cea8d
--- /dev/null
+++ b/services/metrics/grafana/config.monitoring
@@ -0,0 +1,2 @@
+GF_SECURITY_ADMIN_PASSWORD=serverctlsecret
+GF_USERS_ALLOW_SIGN_UP=false
diff --git a/services/metrics/grafana/provisioning/dashboards/Default.json b/services/metrics/grafana/provisioning/dashboards/Default.json
new file mode 100644
index 0000000..5782336
--- /dev/null
+++ b/services/metrics/grafana/provisioning/dashboards/Default.json
@@ -0,0 +1,167 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "PEA2100DC89AE9FE2"
+ },
+ "gridPos": {
+ "h": 19,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 4,
+ "options": {
+ "dedupStrategy": "numbers",
+ "enableLogDetails": true,
+ "prettifyLogMessage": false,
+ "showCommonLabels": false,
+ "showLabels": false,
+ "showTime": false,
+ "sortOrder": "Descending",
+ "wrapLogMessage": false
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "PEA2100DC89AE9FE2"
+ },
+ "expr": "{container_name=\"serverctl-app-1\"}",
+ "instant": false,
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "app",
+ "type": "logs"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 19
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "multi"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PA58DA793C7250F1B"
+ },
+ "exemplar": true,
+ "expr": "scrape_duration_seconds{}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Panel Title",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": false,
+ "schemaVersion": 34,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "Default",
+ "uid": "U8c_qq-7k",
+ "version": 1,
+ "weekStart": ""
+}
diff --git a/services/metrics/grafana/provisioning/dashboards/dashboard.yml b/services/metrics/grafana/provisioning/dashboards/dashboard.yml
new file mode 100644
index 0000000..51547d4
--- /dev/null
+++ b/services/metrics/grafana/provisioning/dashboards/dashboard.yml
@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+ - name: "Metrics"
+ orgId: 1
+ folder: ''
+ type: file
+ disableDeletions: false
+ editable: true
+ options:
+ path: /etc/grafana/provisioning/dashboards
diff --git a/services/metrics/grafana/provisioning/datasources/datasource.yaml b/services/metrics/grafana/provisioning/datasources/datasource.yaml
new file mode 100644
index 0000000..d03e6ba
--- /dev/null
+++ b/services/metrics/grafana/provisioning/datasources/datasource.yaml
@@ -0,0 +1,30 @@
+apiVersion: 1
+
+deleteDatasources:
+ - name: Metrics
+ orgId: 1
+ - name: Logs
+ orgId: 1
+
+datasources:
+ - name: Metrics
+ type: prometheus
+ access: proxy
+ orgId: 1
+ url: http://prometheus:9090
+ basicAuth: false
+ isDefault: true
+ jsonData:
+ graphiteVersion: "1.1"
+ tlsAuth: false
+ tlsAuthWithCACert: false
+ version: 1
+ editable: false
+
+ - name: Logs
+ type: loki
+ access: proxy
+ url: http://loki:3100
+ jsonData:
+ maxLines: 1000
+ orgId: 1
diff --git a/services/metrics/prometheus/prometheus.yml b/services/metrics/prometheus/prometheus.yml
new file mode 100644
index 0000000..11e42cb
--- /dev/null
+++ b/services/metrics/prometheus/prometheus.yml
@@ -0,0 +1,12 @@
+global:
+ scrape_interval: 15s
+ evaluation_interval: 15s
+ external_labels:
+ monitor: "serverctl"
+
+scrape_configs:
+ - job_name: 'prometheus'
+ scrape_interval: 5s
+ static_configs:
+ - targets: ["localhost:9090"]
+