From 205adeb11868b8276157c68960869e0de45a9dce Mon Sep 17 00:00:00 2001 From: kjuulh Date: Tue, 15 Feb 2022 00:00:48 +0100 Subject: [PATCH] Fixed indexing on m:n table, as well as added seeding function --- docker-compose.yml | 1 + ...4_add_index_for_user_on_project_member.sql | 10 ++++ ...dd_index_for_project_on_project_member.sql | 9 +++ services/entry/go.mod | 1 + services/entry/go.sum | 2 + services/entry/main.go | 58 +++++++++++++++++++ 6 files changed, 81 insertions(+) create mode 100644 services/db/migrations/004_add_index_for_user_on_project_member.sql create mode 100644 services/db/migrations/005_add_index_for_project_on_project_member.sql diff --git a/docker-compose.yml b/docker-compose.yml index 88a5943..d7c2b78 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,7 @@ services: DATABASE_URL: "postgresql://serverctl:serverctlsecret@db/serverctl" ports: - "8080:8080" + - "6060:6060" # Profiler optional logging: *loki-logging depends_on: - db_migrator diff --git a/services/db/migrations/004_add_index_for_user_on_project_member.sql b/services/db/migrations/004_add_index_for_user_on_project_member.sql new file mode 100644 index 0000000..e0d2a35 --- /dev/null +++ b/services/db/migrations/004_add_index_for_user_on_project_member.sql @@ -0,0 +1,10 @@ +-- Write your migrate up statements here + +create index member_id_project_member_idx on sctl_project_member (member_id); + +---- create above / drop below ---- + +drop index member_id_project_member_idx; + +-- Write your migrate down statements here. If this migration is irreversible +-- Then delete the separator line above. diff --git a/services/db/migrations/005_add_index_for_project_on_project_member.sql b/services/db/migrations/005_add_index_for_project_on_project_member.sql new file mode 100644 index 0000000..5dfafde --- /dev/null +++ b/services/db/migrations/005_add_index_for_project_on_project_member.sql @@ -0,0 +1,9 @@ +-- Write your migrate up statements here + +create index project_id_project_member_idx on sctl_project_member (project_id); + +---- create above / drop below ---- + +drop index project_id_project_member_idx; +-- Write your migrate down statements here. If this migration is irreversible +-- Then delete the separator line above. diff --git a/services/entry/go.mod b/services/entry/go.mod index 17088ee..4dd1610 100644 --- a/services/entry/go.mod +++ b/services/entry/go.mod @@ -29,6 +29,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/protobuf v1.5.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.11.0 // indirect github.com/jackc/pgio v1.0.0 // indirect diff --git a/services/entry/go.sum b/services/entry/go.sum index 7b29924..e94e2cf 100644 --- a/services/entry/go.sum +++ b/services/entry/go.sum @@ -437,6 +437,8 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= diff --git a/services/entry/main.go b/services/entry/main.go index c6aaecb..83fa0a4 100644 --- a/services/entry/main.go +++ b/services/entry/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "github.com/dgraph-io/ristretto" "github.com/docker/docker/api/types" "github.com/docker/docker/client" @@ -10,10 +11,13 @@ import ( "github.com/eko/gocache/store" "github.com/gin-gonic/gin" "github.com/go-co-op/gocron" + "github.com/google/uuid" + "github.com/jackc/pgx/v4/pgxpool" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/zap" "go.uber.org/zap/zapcore" "io/ioutil" + "log" "net/http" "os" "serverctl/pkg/application/projects" @@ -23,6 +27,8 @@ import ( "time" ) +import _ "net/http/pprof" + func setupLogger() *zap.Logger { highPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { return lvl >= zapcore.ErrorLevel @@ -269,5 +275,57 @@ func main() { projectsRepository := postgres.NewProjectsRepository(database) projectsService := projects.NewService(logger, projectsRepository, cacheM) + setupProfiler() + addSeedData(database, usersRepository, projectsRepository) setupApi(logger, cacheM, usersService, projectsService) } + +func setupProfiler() { + go func() { + log.Println(http.ListenAndServe(":6060", nil)) + }() +} + +func addSeedData(database *db.Client, ur users.Repository, pr projects.Repository) { + conn := database.GetConn(context.Background()) + defer conn.Release() + + var numRows int + err := conn.QueryRow(context.Background(), "select count(id) from sctl_user").Scan(&numRows) + if err != nil { + panic(err) + } + + if numRows == 0 { + addTestData(conn, ur, pr) + } +} + +func addTestData(conn *pgxpool.Conn, ur users.Repository, pr projects.Repository) { + ctx := context.Background() + for jobs := 0; jobs < 100; jobs++ { + go func() { + for i := 0; i < 1_000; i++ { + var ( + user *users.CreateUser + err error + userId int + ) + user, err = users.NewCreateUser(fmt.Sprintf("%s@test.com", uuid.New().String()), "password", users.NewPlainTextPasswordHasher()) + if err != nil { + panic(err) + } + + userId, err = ur.Create(ctx, user) + if err != nil { + panic(err) + } + + _, err = pr.Create(ctx, projects.NewCreateProject(uuid.New().String()[:20], userId)) + if err != nil { + panic(err) + } + } + }() + } +}