Add docker

.idea/sqldialects.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="SqlDialectMappings">
-    <file url="file://$PROJECT_DIR$/migrations/002_emails_are_unique_for_user.sql" dialect="PostgreSQL" />
-  </component>
-</project>

docker-compose.yml

@@ -1,18 +1,116 @@
 version: "3"
 
+networks:
+  back-tier:
+  front-tier:
+
+x-logging: &loki-logging
+  driver: json-file
+  options:
+    tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
+
+
 services:
+
+# Database
   db:
     image: postgres
     restart: always
     volumes:
       - db_data:/var/lib/postgresql/data/pgdata
     ports:
-      - 5432:5432
+      - "5432:5432"
     environment:
       PGDATA: /var/lib/postgresql/data/pgdata
       POSTGRES_USER: serverctl
       POSTGRES_PASSWORD: serverctlsecret
       POSTGRES_DB: serverctl 
+    networks:
+      - back-tier
+
+  db_migrator:
+    build:
+      context: services/db/migrations
+    networks:
+      - back-tier
+    depends_on:
+      - "db"
+
+  app:
+    build:
+      context: services/entry/
+    networks:
+      - back-tier
+      - front-tier
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./services/entry/:/app/
+    environment:
+      DATABASE_URL: "postgresql://serverctl:serverctlsecret@db/serverctl"
+    ports:
+      - "8080:8080"
+    logging: *loki-logging
+    depends_on:
+      - db_migrator
+
+# Logging
+  loki:
+    image: grafana/loki:2.4.2
+    ports:
+      - 3100
+    networks:
+      - back-tier
+    volumes:
+      - './services/logs/loki/config.yaml:/mnt/config/loki-config.yaml'
+    command: -config.file=/mnt/config/loki-config.yaml
+    logging: *loki-logging
+
+  promtail:
+    image: grafana/promtail:2.4.2
+    volumes:
+      - ./services/logs/promtail/config.yaml:/mnt/config/promtail-config.yaml
+      - /var/lib/docker/containers:/host/containers
+    command: -config.file /mnt/config/promtail-config.yaml
+    networks:
+      - back-tier
+    logging: *loki-logging
+    depends_on:
+      - loki
+
+#Metrics
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - ./services/metrics/prometheus/:/etc/prometheus
+      - prometheus_data:/prometheus
+    networks:
+      - back-tier
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+    restart: always
+
+  grafana:
+    image: grafana/grafana
+    user: "472"
+    depends_on:
+      - prometheus
+    ports:
+      - "3000:3000"
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./services/metrics/grafana/provisioning:/etc/grafana/provisioning
+    env_file:
+      - ./services/metrics/grafana/config.monitoring
+    networks:
+      - back-tier
+      - front-tier
+    restart: always
+
 
 volumes:
   db_data: {}
+  prometheus_data: {}
+  grafana_data: {}

services/db/README.md

@@ -0,0 +1,7 @@
+# ServerCtl Database
+
+## Setup
+
+```bash
+tern migrate
+```

services/db/migrations/Dockerfile

@@ -0,0 +1,9 @@
+FROM golang:1.17-bullseye
+
+RUN go install github.com/jackc/tern@latest
+
+COPY . /app/migration/
+
+WORKDIR /app/migration
+
+CMD ./wait-for-database.sh

services/db/migrations/tern.docker.conf

@@ -0,0 +1,34 @@
+[database]
+# host is required (network host or path to Unix domain socket)
+host = db
+port = 5432
+# database is required
+database = serverctl
+# user defaults to OS user
+user = serverctl
+password = serverctlsecret
+version_table = public.schema_version
+#
+# sslmode generally matches the behavior described in:
+# http://www.postgresql.org/docs/9.4/static/libpq-ssl.html#LIBPQ-SSL-PROTECTION
+#
+# There are only two modes that most users should use:
+# prefer - on trusted networks where security is not required
+# verify-full - require SSL connection
+sslmode = prefer
+#
+# sslrootcert is generally used with sslmode=verify-full
+# sslrootcert = /path/to/root/ca
+
+# Proxy the above database connection via SSH
+# [ssh-tunnel]
+# host =
+# port = 22
+# user defaults to OS user
+# user =
+# password is not required if using SSH agent authentication
+# password =
+
+[data]
+# Any fields in the data section are available in migration templates
+prefix = serverctl

services/db/migrations/wait-for-database.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+# wait-for-postgres.sh
+
+set -e
+
+until tern status -c tern.docker.conf; do
+  >&2 echo "Postgres is unavailable - sleeping"
+  sleep 1
+done
+
+>&2 echo "Postgres is up - executing command"
+
+tern migrate -c tern.docker.conf

services/entry/.dockerignore

@@ -0,0 +1 @@
+tmp/

services/entry/Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.17-bullseye
+
+RUN go install github.com/cosmtrek/air@latest
+# Development don't need this
+# COPY . /app/
+WORKDIR /app/
+
+ARG DATABASE_URL
+ENV DATABASE_URL DATABASE_URL
+
+#CMD go run main.go
+CMD air

services/entry/main.go

@@ -37,13 +37,13 @@ func setupLogger() *zap.Logger {
 	consoleErrors := zapcore.Lock(os.Stderr)
 
 	fileEncoder := zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig())
-	consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig())
+	_ = zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig())
 
 	core := zapcore.NewTee(
 		zapcore.NewCore(fileEncoder, fileErrors, highPriority),
-		zapcore.NewCore(consoleEncoder, consoleErrors, highPriority),
+		zapcore.NewCore(fileEncoder, consoleErrors, highPriority),
 		zapcore.NewCore(fileEncoder, fileDebugging, lowPriority),
-		zapcore.NewCore(consoleEncoder, consoleDebugging, lowPriority),
+		zapcore.NewCore(fileEncoder, consoleDebugging, lowPriority),
 	)
 
 	logger := zap.New(core)

services/logs/loki/config.yaml

@@ -0,0 +1,56 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+ingester:
+  wal:
+    enabled: true
+    dir: /tmp/wal
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed
+  max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h
+  chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
+  chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+  max_transfer_retries: 0 # Chunk transfers disabled
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /tmp/loki/boltdb-shipper-active
+    cache_location: /tmp/loki/boltdb-shipper-cache
+    cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
+    shared_store: filesystem
+  filesystem:
+    directory: /tmp/loki/chunks
+
+compactor:
+  working_directory: /tmp/loki/boltdb-shipper-compactor
+  shared_store: filesystem
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: true
+  retention_period: 24h

services/logs/promtail/config.yaml

@@ -0,0 +1,52 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  # - job_name: system
+  #   static_configs:
+  #     - targets:
+  #         - localhost
+  #       labels:
+  #         job: varlogs
+  #         __path__: /var/log/*log
+  - job_name: docker
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: dockerlogs
+          __path__: /host/containers/*/*log
+    pipeline_stages:
+      - json:
+          expressions:
+            output: log
+            stream: stream
+            attrs:
+      - json:
+          expressions:
+            tag:
+          source: attrs
+      - regex:
+          expression: (?P<image_name>(?:[^|]*[^|])).(?P<container_name>(?:[^|]*[^|])).(?P<image_id>(?:[^|]*[^|])).(?P<container_id>(?:[^|]*[^|]))
+          source: tag
+      - timestamp:
+          format: RFC3339Nano
+          source: time
+      - labels:
+          tag:
+          stream:
+          image_name:
+          container_name:
+          image_id:
+          container_id:
+      - output:
+          source: output
+
+

services/metrics/grafana/config.monitoring

@@ -0,0 +1,2 @@
+GF_SECURITY_ADMIN_PASSWORD=serverctlsecret
+GF_USERS_ALLOW_SIGN_UP=false

services/metrics/grafana/provisioning/dashboards/Default.json

@@ -0,0 +1,167 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "PEA2100DC89AE9FE2"
+      },
+      "gridPos": {
+        "h": 19,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "dedupStrategy": "numbers",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "PEA2100DC89AE9FE2"
+          },
+          "expr": "{container_name=\"serverctl-app-1\"}",
+          "instant": false,
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "app",
+      "type": "logs"
+    },
+    {
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 19
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PA58DA793C7250F1B"
+          },
+          "exemplar": true,
+          "expr": "scrape_duration_seconds{}",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Panel Title",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 34,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Default",
+  "uid": "U8c_qq-7k",
+  "version": 1,
+  "weekStart": ""
+}

services/metrics/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: "Metrics"
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletions: false
+    editable: true
+    options:
+      path: /etc/grafana/provisioning/dashboards

services/metrics/grafana/provisioning/datasources/datasource.yaml

@@ -0,0 +1,30 @@
+apiVersion: 1
+
+deleteDatasources:
+  - name: Metrics
+    orgId: 1
+  - name: Logs
+    orgId: 1
+
+datasources:
+  - name: Metrics
+    type: prometheus
+    access: proxy
+    orgId: 1
+    url: http://prometheus:9090
+    basicAuth: false
+    isDefault: true
+    jsonData:
+      graphiteVersion: "1.1"
+      tlsAuth: false
+      tlsAuthWithCACert: false
+    version: 1
+    editable: false
+
+  - name: Logs
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    jsonData:
+      maxLines: 1000
+    orgId: 1

services/metrics/prometheus/prometheus.yml

@@ -0,0 +1,12 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    monitor: "serverctl"
+
+scrape_configs:
+  - job_name: 'prometheus'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ["localhost:9090"]
+