add basic leader election on top of postgres

.env

@@ -1 +1,2 @@
 ORBIS_LOG_LEVEL=debug
+ORBIS_POSTGRES_DB=postgresql://postgres:orbis@localhost:5432/orbis

cuddle.yaml

@@ -21,3 +21,7 @@ scripts:
     type: shell
   test:
     type: shell
+  local_up:
+    type: shell
+  local_down:
+    type: shell

go.mod

@@ -2,12 +2,19 @@ module git.front.kjuulh.io/kjuulh/orbis
 
 go 1.23.4
 
-require github.com/spf13/cobra v1.8.1
+require (
+	github.com/jackc/pgx/v5 v5.7.2
+	github.com/joho/godotenv v1.5.1
+	github.com/spf13/cobra v1.8.1
+	gitlab.com/greyxor/slogor v1.6.0
+)
 
 require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/joho/godotenv v1.5.1 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	gitlab.com/greyxor/slogor v1.6.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 )

go.sum

@@ -1,16 +1,42 @@
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI=
+github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 gitlab.com/greyxor/slogor v1.6.0 h1:K9QsAoa4leFQfO2RF2MkZ8BYkk2HQpYWykmd4G5R5+Y=
 gitlab.com/greyxor/slogor v1.6.0/go.mod h1:6UWQsLLkeNL4o911soP9jvCMzXWgokLqzZP+eekAyyU=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/app/app.go

@@ -21,5 +21,5 @@ func (a *App) Logger() *slog.Logger {
 }
 
 func (a *App) Scheduler() *scheduler.Scheduler {
-	return scheduler.NewScheduler(a.logger)
+	return scheduler.NewScheduler(a.logger, Postgres())
 }

internal/app/postgres.go

@@ -0,0 +1,23 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"git.front.kjuulh.io/kjuulh/orbis/internal/utilities"
+	"github.com/jackc/pgx/v5"
+)
+
+var Postgres = utilities.Singleton(func() (*pgx.Conn, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+
+	conn, err := pgx.Connect(ctx, os.Getenv("ORBIS_POSTGRES_DB"))
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to orbis postgres database: %w", err)
+	}
+
+	return conn, nil
+})

internal/scheduler/scheduler.go

@@ -2,18 +2,24 @@ package scheduler
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
+	"math/rand"
 	"time"
+
+	"github.com/jackc/pgx/v5"
 )
 
 type Scheduler struct {
 	logger *slog.Logger
+	db     *pgx.Conn
 }
 
-func NewScheduler(logger *slog.Logger) *Scheduler {
+func NewScheduler(logger *slog.Logger, db *pgx.Conn) *Scheduler {
 	return &Scheduler{
 		logger: logger,
+		db:     db,
 	}
 }
 
@@ -48,8 +54,21 @@ func (s *Scheduler) acquireLeader(ctx context.Context) (bool, error) {
 			return false, nil
 
 		default:
-			// Attempt to acquire leader
-			//
+			var acquiredLock bool
+			if err := s.db.QueryRow(ctx, "SELECT pg_try_advisory_lock(1234)").Scan(&acquiredLock); err != nil {
+				if errors.Is(err, pgx.ErrNoRows) {
+					return false, nil
+				}
+			}
+
+			if !acquiredLock {
+				wait := time.Second * time.Duration(rand.Float32()*9+1)
+
+				s.logger.Debug("failed to acquire lock, parking non-elected scheduler", "wait_seconds", wait)
+				time.Sleep(wait)
+				continue
+			}
+
 			return true, nil
 
 		}

scripts/local_down.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env zsh
+
+set -e pipefail
+
+docker compose -f templates/docker-compose.yml down -v

scripts/local_up.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env zsh
+
+set -e pipefail
+
+docker compose -f templates/docker-compose.yml up -d

templates/docker-compose.yml

@@ -0,0 +1,17 @@
+name: orbis
+
+services:
+  postgres:
+    image: postgres:17
+    environment:
+      POSTGRES_DB: orbis
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: orbis
+    command: postgres -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c max_connections=200
+    ports:
+      - "5432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5