cron, db, cache and migrations

This commit is contained in:
Kasper Juul Hermansen 2022-02-13 18:09:26 +01:00
parent 5f36bdb8a6
commit e03efbf09c
Signed by: kjuulh
GPG Key ID: 0F95C140730F2F23
8 changed files with 1638 additions and 2 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
tmp/

12
.idea/dataSources.xml Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
<data-source source="LOCAL" name="serverctl@localhost" uuid="372f3f01-6672-42bd-a724-099a867af00b">
<driver-ref>postgresql</driver-ref>
<synchronize>true</synchronize>
<jdbc-driver>org.postgresql.Driver</jdbc-driver>
<jdbc-url>jdbc:postgresql://localhost:5432/serverctl</jdbc-url>
<working-dir>$ProjectFileDir$</working-dir>
</data-source>
</component>
</project>

18
docker-compose.yml Normal file
View File

@ -0,0 +1,18 @@
version: "3"
services:
db:
image: postgres
restart: always
volumes:
- db_data:/var/lib/postgresql/data/pgdata
ports:
- 5432:5432
environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_USER: serverctl
POSTGRES_PASSWORD: serverctlsecret
POSTGRES_DB: serverctl
volumes:
db_data: {}

74
go.mod
View File

@ -1,3 +1,77 @@
module serverctl
go 1.17
require (
github.com/Microsoft/go-winio v0.4.17 // indirect
github.com/XiaoMi/pegasus-go-client v0.0.0-20210427083443-f3b6b08bc4c2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect
github.com/cenkalti/backoff/v4 v4.1.1 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/containerd/containerd v1.5.9 // indirect
github.com/dgraph-io/ristretto v0.1.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/docker/distribution v2.8.0+incompatible // indirect
github.com/docker/docker v20.10.12+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/eko/gocache v1.2.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/gin-gonic/gin v1.7.7 // indirect
github.com/go-co-op/gocron v1.12.0 // indirect
github.com/go-playground/locales v0.13.0 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/go-playground/validator/v10 v10.4.1 // indirect
github.com/go-redis/redis/v8 v8.8.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
github.com/golang/protobuf v1.5.0 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.11.0 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.2.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/jackc/pgtype v1.10.0 // indirect
github.com/jackc/pgx v3.6.2+incompatible // indirect
github.com/jackc/pgx/v4 v4.15.0 // indirect
github.com/jackc/puddle v1.2.1 // indirect
github.com/json-iterator/go v1.1.10 // indirect
github.com/leodido/go-urn v1.2.0 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/pegasus-kv/thrift v0.13.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.10.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.18.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/ugorji/go/codec v1.1.7 // indirect
go.opentelemetry.io/otel v0.19.0 // indirect
go.opentelemetry.io/otel/metric v0.19.0 // indirect
go.opentelemetry.io/otel/trace v0.19.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
golang.org/x/text v0.3.6 // indirect
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
google.golang.org/grpc v1.44.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/apimachinery v0.20.6 // indirect
)

1332
go.sum Normal file

File diff suppressed because it is too large Load Diff

160
main.go
View File

@ -1,5 +1,161 @@
package main
func main() {
println("hello world")
import (
"context"
"errors"
"github.com/dgraph-io/ristretto"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/eko/gocache/cache"
"github.com/eko/gocache/store"
"github.com/gin-gonic/gin"
"github.com/go-co-op/gocron"
"github.com/jackc/pgx/v4/pgxpool"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"io/ioutil"
"net/http"
"os"
"time"
)
func setupLogger() *zap.Logger {
highPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
return lvl >= zapcore.ErrorLevel
})
lowPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
return lvl < zapcore.ErrorLevel
})
fileDebugging := zapcore.AddSync(ioutil.Discard)
fileErrors := zapcore.AddSync(ioutil.Discard)
consoleDebugging := zapcore.Lock(os.Stdout)
consoleErrors := zapcore.Lock(os.Stderr)
fileEncoder := zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig())
consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig())
core := zapcore.NewTee(
zapcore.NewCore(fileEncoder, fileErrors, highPriority),
zapcore.NewCore(consoleEncoder, consoleErrors, highPriority),
zapcore.NewCore(fileEncoder, fileDebugging, lowPriority),
zapcore.NewCore(consoleEncoder, consoleDebugging, lowPriority),
)
logger := zap.New(core)
defer logger.Sync()
return logger
}
func setupApi(l *zap.Logger, cc *cache.Cache) {
l.Info("Setting up serverctl setupApi (using gin)")
r := gin.Default()
r.GET("/containers", func(c *gin.Context) {
type container struct {
Name string `json:"name"`
}
var msg struct {
Containers []container `json:"containers"`
}
get, err := cc.Get("docker-containers")
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "could not get containers from container runtime"})
return
}
msg.Containers = []container{}
for _, cont := range get.([]types.Container) {
msg.Containers = append(msg.Containers, container{
Name: cont.Names[0],
})
}
c.JSON(http.StatusOK, msg)
})
r.Run(":8080")
}
func setupDocker(l *zap.Logger) *client.Client {
l.Info("Setting up Docker")
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
panic(err)
}
return cli
}
func setupCache(l *zap.Logger) *cache.Cache {
l.Info("Setting up cache")
ristrettoCache, err := ristretto.NewCache(&ristretto.Config{
NumCounters: 1000,
MaxCost: 100,
BufferItems: 64,
})
if err != nil {
panic(err)
}
ristrettoStore := store.NewRistretto(ristrettoCache, nil)
cacheManager := cache.New(ristrettoStore)
return cacheManager
}
func setupCron(l *zap.Logger, cm *cache.Cache, cc *client.Client) {
l.Info("Setting up job scheduler (cron)")
s := gocron.NewScheduler(time.UTC)
s.Every(10).Second().Do(func() {
l.Debug("getting container list")
list, err := cc.ContainerList(context.Background(), types.ContainerListOptions{})
if err != nil {
l.Warn(err.Error())
return
}
err = cm.Set("docker-containers", list, &store.Options{
Cost: 2,
})
if err != nil {
l.Warn(err.Error())
return
}
})
s.StartAsync()
}
func setupDatabase(l *zap.Logger) *pgxpool.Pool {
l.Info("Setting up database connection")
dbUrl := os.Getenv("DATABASE_URL")
if dbUrl == "" {
panic(errors.New("DATABASE_URL is not set"))
}
dbpool, err := pgxpool.Connect(context.Background(), dbUrl)
if err != nil {
panic(err)
}
var greeting string
err = dbpool.QueryRow(context.Background(), "select 'Hello, world!'").Scan(&greeting)
if err != nil {
panic(err)
}
l.Info("Database successfully connected")
return dbpool
}
func main() {
logger := setupLogger()
logger.Info("Starting serverctl")
cacheM := setupCache(logger)
containerClient := setupDocker(logger)
setupCron(logger, cacheM, containerClient)
dbpool := setupDatabase(logger)
setupApi(logger, cacheM)
}

View File

@ -0,0 +1,9 @@
create table users(
id serial primary key,
email varchar(320) not null,
password_hash varchar(256) not null
);
---- create above / drop below ----
drop table users;

34
migrations/tern.conf Normal file
View File

@ -0,0 +1,34 @@
[database]
# host is required (network host or path to Unix domain socket)
host = localhost
port = 5432
# database is required
database = serverctl
# user defaults to OS user
user = serverctl
password = serverctlsecret
version_table = public.schema_version
#
# sslmode generally matches the behavior described in:
# http://www.postgresql.org/docs/9.4/static/libpq-ssl.html#LIBPQ-SSL-PROTECTION
#
# There are only two modes that most users should use:
# prefer - on trusted networks where security is not required
# verify-full - require SSL connection
sslmode = prefer
#
# sslrootcert is generally used with sslmode=verify-full
# sslrootcert = /path/to/root/ca
# Proxy the above database connection via SSH
# [ssh-tunnel]
# host =
# port = 22
# user defaults to OS user
# user =
# password is not required if using SSH agent authentication
# password =
[data]
# Any fields in the data section are available in migration templates
# prefix = foo