From d9404525f59fcc9fe44d7f25d4d1a759b7599b7b Mon Sep 17 00:00:00 2001 From: kjuulh Date: Tue, 13 Sep 2022 20:23:24 +0200 Subject: [PATCH 01/15] add roadmpa item --- roadmap.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roadmap.md b/roadmap.md index 25b8252..8be14af 100644 --- a/roadmap.md +++ b/roadmap.md @@ -18,8 +18,8 @@ ## Version 0.1 +- [ ] Setup a way to choose actions and predicates - [ ] Allow instantiation of actions, kraken template repo etc. -- [ ] Create predicate handle - [ ] Think about some sort of isolation - [ ] Create CLI to trigger action - [ ] Setup pool of runners @@ -27,3 +27,5 @@ - [ ] Create queuing system ## Version 1.0 + +- [ ] Write README -- 2.45.2 From 1b14d33a53d7a93dad952517cf0b92c11d3e1deb Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Tue, 13 Sep 2022 20:54:59 +0200 Subject: [PATCH 02/15] another roadmap item (#5) Co-authored-by: kjuulh Reviewed-on: https://git.front.kjuulh.io/kjuulh/kraken/pulls/5 --- roadmap.md | 1 + 1 file changed, 1 insertion(+) diff --git a/roadmap.md b/roadmap.md index 8be14af..ae581eb 100644 --- a/roadmap.md +++ b/roadmap.md @@ -20,6 +20,7 @@ - [ ] Setup a way to choose actions and predicates - [ ] Allow instantiation of actions, kraken template repo etc. +- [ ] Create pr for gitea provider - [ ] Think about some sort of isolation - [ ] Create CLI to trigger action - [ ] Setup pool of runners -- 2.45.2 From 559fb41571c3f89f4872c0f9851fa38bdb06d2fb Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Tue, 13 Sep 2022 20:55:41 +0200 Subject: [PATCH 03/15] refac/process_command (#6) Co-authored-by: kjuulh Reviewed-on: https://git.front.kjuulh.io/kjuulh/kraken/pulls/6 --- internal/commands/process_repos.go | 214 +++++++++++++++++------------ roadmap.md | 3 + 2 files changed, 129 insertions(+), 88 deletions(-) diff --git a/internal/commands/process_repos.go b/internal/commands/process_repos.go index 5ea781c..45a580a 100644 --- a/internal/commands/process_repos.go +++ b/internal/commands/process_repos.go @@ -41,102 +41,17 @@ func NewProcessRepos(logger *zap.Logger, deps ProcessReposDeps) *ProcessRepos { } func (pr *ProcessRepos) Process(ctx context.Context, repositoryUrls []string) error { - // Clone repos + errChan := make(chan error, 1) + wg := sync.WaitGroup{} wg.Add(len(repositoryUrls)) - errChan := make(chan error, 1) for _, repoUrl := range repositoryUrls { go func(ctx context.Context, repoUrl string) { defer func() { wg.Done() }() - pr.logger.Debug("Creating area", zap.String("repoUrl", repoUrl)) - area, err := pr.storage.CreateArea(ctx) - if err != nil { - pr.logger.Error("failed to allocate area", zap.Error(err)) - errChan <- err - return - } - - defer func(ctx context.Context) { - pr.logger.Debug("Removing area", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) - err = pr.storage.RemoveArea(ctx, area) - if err != nil { - errChan <- err - return - } - - }(ctx) - - pr.logger.Debug("Cloning repo", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) - cloneCtx, _ := context.WithTimeout(ctx, time.Second*5) - repo, err := pr.git.Clone(cloneCtx, area, repoUrl) - if err != nil { - pr.logger.Error("could not clone repo", zap.Error(err)) - errChan <- err - return - } - - err = pr.git.CreateBranch(ctx, repo) - if err != nil { - pr.logger.Error("could not create branch", zap.Error(err)) - errChan <- err - return - } - - err = pr.action.Run( - ctx, - area, - func(_ context.Context, area *storage.Area) (bool, error) { - pr.logger.Debug("checking predicate", zap.String("area", area.Path)) - contains := false - filepath.WalkDir(area.Path, func(path string, d fs.DirEntry, err error) error { - if d.Name() == "roadmap.md" { - contains = true - } - return nil - }) - return contains, nil - }, - func(_ context.Context, area *storage.Area) error { - pr.logger.Debug("running action", zap.String("area", area.Path)) - readme := path.Join(area.Path, "README.md") - file, err := os.Create(readme) - if err != nil { - return fmt.Errorf("could not create readme: %w", err) - } - _, err = file.WriteString("# Readme") - if err != nil { - return fmt.Errorf("could not write readme: %w", err) - } - - _, err = pr.git.Add(ctx, area, repo) - if err != nil { - return fmt.Errorf("could not add file: %w", err) - } - - err = pr.git.Commit(ctx, repo) - if err != nil { - return fmt.Errorf("could not get diff: %w", err) - } - - return nil - }, false) - if err != nil { - pr.logger.Error("could not run action", zap.Error(err)) - errChan <- err - return - } - - err = pr.git.Push(ctx, repo) - if err != nil { - pr.logger.Error("could not push to repo", zap.Error(err)) - errChan <- err - return - } - - pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) + pr.processRepo(ctx, repoUrl, errChan) }(ctx, repoUrl) } @@ -150,3 +65,126 @@ func (pr *ProcessRepos) Process(ctx context.Context, repositoryUrls []string) er return nil } + +func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, errChan chan error) { + cleanup, area, err := pr.prepareAction(ctx) + defer func() { + if cleanup != nil { + cleanup(ctx, errChan) + } + }() + if err != nil { + errChan <- err + return + } + + repo, err := pr.clone(ctx, area, repoUrl) + if err != nil { + errChan <- err + return + } + + err = pr.action.Run( + ctx, + area, + func(_ context.Context, area *storage.Area) (bool, error) { + pr.logger.Debug("checking predicate", zap.String("area", area.Path)) + + // TODO: Run predicate instead + contains := false + filepath.WalkDir(area.Path, func(_ string, d fs.DirEntry, _ error) error { + if d.Name() == "roadmap.md" { + contains = true + } + return nil + }) + return contains, nil + }, + func(_ context.Context, area *storage.Area) error { + pr.logger.Debug("running action", zap.String("area", area.Path)) + + // TODO: Run action instead + readme := path.Join(area.Path, "README.md") + file, err := os.Create(readme) + if err != nil { + return fmt.Errorf("could not create readme: %w", err) + } + + _, err = file.WriteString("# Readme") + if err != nil { + return fmt.Errorf("could not write readme: %w", err) + } + + err = pr.commit(ctx, area, repo) + if err != nil { + return err + } + + return nil + }, false) + if err != nil { + pr.logger.Error("could not run action", zap.Error(err)) + errChan <- err + return + } + + pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) +} + +func (pr *ProcessRepos) prepareAction( + ctx context.Context, +) (func(ctx context.Context, errChan chan error), *storage.Area, error) { + pr.logger.Debug("Creating area") + area, err := pr.storage.CreateArea(ctx) + if err != nil { + pr.logger.Error("failed to allocate area", zap.Error(err)) + return nil, nil, err + } + + cleanupfunc := func(ctx context.Context, errChan chan error) { + pr.logger.Debug("Removing area", zap.String("path", area.Path)) + err = pr.storage.RemoveArea(ctx, area) + if err != nil { + errChan <- err + return + } + } + + return cleanupfunc, area, nil +} + +func (pr *ProcessRepos) clone(ctx context.Context, area *storage.Area, repoUrl string) (*providers.GitRepo, error) { + pr.logger.Debug("Cloning repo", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) + cloneCtx, _ := context.WithTimeout(ctx, time.Second*5) + repo, err := pr.git.Clone(cloneCtx, area, repoUrl) + if err != nil { + pr.logger.Error("could not clone repo", zap.Error(err)) + return nil, err + } + + err = pr.git.CreateBranch(ctx, repo) + if err != nil { + pr.logger.Error("could not create branch", zap.Error(err)) + return nil, err + } + + return repo, nil +} + +func (pr *ProcessRepos) commit(ctx context.Context, area *storage.Area, repo *providers.GitRepo) error { + _, err := pr.git.Add(ctx, area, repo) + if err != nil { + return fmt.Errorf("could not add file: %w", err) + } + + err = pr.git.Commit(ctx, repo) + if err != nil { + return fmt.Errorf("could not get diff: %w", err) + } + + err = pr.git.Push(ctx, repo) + if err != nil { + return fmt.Errorf("could not push to repo: %w", err) + } + return nil +} diff --git a/roadmap.md b/roadmap.md index ae581eb..ba55870 100644 --- a/roadmap.md +++ b/roadmap.md @@ -30,3 +30,6 @@ ## Version 1.0 - [ ] Write README +- [ ] Make configurable ssh user +- [ ] Make configurable gpg keyset +- [ ] Make configurable git provider -- 2.45.2 From c7e773be9343e73e83cd533bea47e52e28d652bf Mon Sep 17 00:00:00 2001 From: kjuulh Date: Tue, 13 Sep 2022 21:15:32 +0200 Subject: [PATCH 04/15] with public interfaces --- _examples/actions/write_a_readme/actions/action.go | 5 +++++ _examples/actions/write_a_readme/go.mod | 4 ++++ _examples/actions/write_a_readme/kraken.yml | 8 ++++++++ _examples/actions/write_a_readme/predicates/predicate.go | 1 + pkg/actions/interface.go | 6 ++++++ 5 files changed, 24 insertions(+) create mode 100644 _examples/actions/write_a_readme/actions/action.go create mode 100644 _examples/actions/write_a_readme/go.mod create mode 100644 _examples/actions/write_a_readme/kraken.yml create mode 100644 _examples/actions/write_a_readme/predicates/predicate.go create mode 100644 pkg/actions/interface.go diff --git a/_examples/actions/write_a_readme/actions/action.go b/_examples/actions/write_a_readme/actions/action.go new file mode 100644 index 0000000..7905807 --- /dev/null +++ b/_examples/actions/write_a_readme/actions/action.go @@ -0,0 +1,5 @@ +package main + +func main() { + +} diff --git a/_examples/actions/write_a_readme/go.mod b/_examples/actions/write_a_readme/go.mod new file mode 100644 index 0000000..066dd7a --- /dev/null +++ b/_examples/actions/write_a_readme/go.mod @@ -0,0 +1,4 @@ +module write_a_readme + +go 1.19 + diff --git a/_examples/actions/write_a_readme/kraken.yml b/_examples/actions/write_a_readme/kraken.yml new file mode 100644 index 0000000..cc49b8a --- /dev/null +++ b/_examples/actions/write_a_readme/kraken.yml @@ -0,0 +1,8 @@ +apiVersion: kraken.front.kjuulh.io/schema/v1 +name: write-a-readme +repositories: + - git@git.front.kjuulh.io:kjuulh/kraken-test.git +predicates: + - predicate.go +actions: + - action.go diff --git a/_examples/actions/write_a_readme/predicates/predicate.go b/_examples/actions/write_a_readme/predicates/predicate.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/_examples/actions/write_a_readme/predicates/predicate.go @@ -0,0 +1 @@ +package main diff --git a/pkg/actions/interface.go b/pkg/actions/interface.go new file mode 100644 index 0000000..8fc4590 --- /dev/null +++ b/pkg/actions/interface.go @@ -0,0 +1,6 @@ +package actions + +import "context" + +type Predicate func(ctx context.Context, path string) (bool, error) +type Action func(ctx context.Context, path string) error -- 2.45.2 From fd7b06e84011e2417bd5da0af8be43ebf33cf58d Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Tue, 13 Sep 2022 21:54:05 +0200 Subject: [PATCH 05/15] added example (#7) Co-authored-by: kjuulh Reviewed-on: https://git.front.kjuulh.io/kjuulh/kraken/pulls/7 --- .../actions/write_a_readme/actions/action.go | 5 ---- _examples/actions/write_a_readme/go.mod | 13 +++++++++ _examples/actions/write_a_readme/go.sum | 28 +++++++++++++++++++ _examples/actions/write_a_readme/kraken.yml | 14 +++++----- _examples/actions/write_a_readme/main.go | 9 ++++++ .../write_a_readme/predicates/predicate.go | 1 - pkg/actions/interface.go | 6 ---- 7 files changed, 57 insertions(+), 19 deletions(-) delete mode 100644 _examples/actions/write_a_readme/actions/action.go create mode 100644 _examples/actions/write_a_readme/go.sum create mode 100644 _examples/actions/write_a_readme/main.go delete mode 100644 _examples/actions/write_a_readme/predicates/predicate.go delete mode 100644 pkg/actions/interface.go diff --git a/_examples/actions/write_a_readme/actions/action.go b/_examples/actions/write_a_readme/actions/action.go deleted file mode 100644 index 7905807..0000000 --- a/_examples/actions/write_a_readme/actions/action.go +++ /dev/null @@ -1,5 +0,0 @@ -package main - -func main() { - -} diff --git a/_examples/actions/write_a_readme/go.mod b/_examples/actions/write_a_readme/go.mod index 066dd7a..99259a8 100644 --- a/_examples/actions/write_a_readme/go.mod +++ b/_examples/actions/write_a_readme/go.mod @@ -2,3 +2,16 @@ module write_a_readme go 1.19 +require kraken v0.0.0 + +require ( + bitbucket.org/creachadair/shell v0.0.7 // indirect + github.com/bitfield/script v0.20.2 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/itchyny/gojq v0.12.7 // indirect + github.com/itchyny/timefmt-go v0.1.3 // indirect + github.com/spf13/cobra v1.5.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) + +replace kraken => ../../../ diff --git a/_examples/actions/write_a_readme/go.sum b/_examples/actions/write_a_readme/go.sum new file mode 100644 index 0000000..b23403e --- /dev/null +++ b/_examples/actions/write_a_readme/go.sum @@ -0,0 +1,28 @@ +bitbucket.org/creachadair/shell v0.0.7 h1:Z96pB6DkSb7F3Y3BBnJeOZH2gazyMTWlvecSD4vDqfk= +bitbucket.org/creachadair/shell v0.0.7/go.mod h1:oqtXSSvSYr4624lnnabXHaBsYW6RD80caLi2b3hJk0U= +github.com/bitfield/script v0.20.2 h1:4DexsRtBILVMEn3EZwHbtJdDqdk43sXI8gM3F04JXgs= +github.com/bitfield/script v0.20.2/go.mod h1:l3AZPVAtKQrL03bwh7nlNTUtgrgSWurpJSbtqspYrOA= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= +github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/_examples/actions/write_a_readme/kraken.yml b/_examples/actions/write_a_readme/kraken.yml index cc49b8a..778a362 100644 --- a/_examples/actions/write_a_readme/kraken.yml +++ b/_examples/actions/write_a_readme/kraken.yml @@ -1,8 +1,8 @@ -apiVersion: kraken.front.kjuulh.io/schema/v1 +apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1 name: write-a-readme -repositories: - - git@git.front.kjuulh.io:kjuulh/kraken-test.git -predicates: - - predicate.go -actions: - - action.go +select: + repositories: + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + provider: + - gitea: git.front.kjuulh.io/kraken + organisation: "kraken" diff --git a/_examples/actions/write_a_readme/main.go b/_examples/actions/write_a_readme/main.go new file mode 100644 index 0000000..a5bd52f --- /dev/null +++ b/_examples/actions/write_a_readme/main.go @@ -0,0 +1,9 @@ +package main + +import "github.com/bitfield/script" + +func main() { + script. + Echo("# Readme"). + WriteFile("README.md") +} diff --git a/_examples/actions/write_a_readme/predicates/predicate.go b/_examples/actions/write_a_readme/predicates/predicate.go deleted file mode 100644 index 06ab7d0..0000000 --- a/_examples/actions/write_a_readme/predicates/predicate.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/pkg/actions/interface.go b/pkg/actions/interface.go deleted file mode 100644 index 8fc4590..0000000 --- a/pkg/actions/interface.go +++ /dev/null @@ -1,6 +0,0 @@ -package actions - -import "context" - -type Predicate func(ctx context.Context, path string) (bool, error) -type Action func(ctx context.Context, path string) error -- 2.45.2 From c759f8e6988ed73709c0e50d971228b9d7d281b4 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Tue, 13 Sep 2022 21:55:08 +0200 Subject: [PATCH 06/15] removed unused stuff --- _examples/actions/write_a_readme/go.mod | 6 +----- _examples/actions/write_a_readme/go.sum | 10 +--------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/_examples/actions/write_a_readme/go.mod b/_examples/actions/write_a_readme/go.mod index 99259a8..2fd95c5 100644 --- a/_examples/actions/write_a_readme/go.mod +++ b/_examples/actions/write_a_readme/go.mod @@ -2,16 +2,12 @@ module write_a_readme go 1.19 -require kraken v0.0.0 +require github.com/bitfield/script v0.20.2 require ( bitbucket.org/creachadair/shell v0.0.7 // indirect - github.com/bitfield/script v0.20.2 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/itchyny/gojq v0.12.7 // indirect github.com/itchyny/timefmt-go v0.1.3 // indirect - github.com/spf13/cobra v1.5.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect ) replace kraken => ../../../ diff --git a/_examples/actions/write_a_readme/go.sum b/_examples/actions/write_a_readme/go.sum index b23403e..234eb1e 100644 --- a/_examples/actions/write_a_readme/go.sum +++ b/_examples/actions/write_a_readme/go.sum @@ -2,27 +2,19 @@ bitbucket.org/creachadair/shell v0.0.7 h1:Z96pB6DkSb7F3Y3BBnJeOZH2gazyMTWlvecSD4 bitbucket.org/creachadair/shell v0.0.7/go.mod h1:oqtXSSvSYr4624lnnabXHaBsYW6RD80caLi2b3hJk0U= github.com/bitfield/script v0.20.2 h1:4DexsRtBILVMEn3EZwHbtJdDqdk43sXI8gM3F04JXgs= github.com/bitfield/script v0.20.2/go.mod h1:l3AZPVAtKQrL03bwh7nlNTUtgrgSWurpJSbtqspYrOA= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -- 2.45.2 From fcd369d48fd08c63f55ff41a0b4540b90d271226 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Tue, 13 Sep 2022 21:55:58 +0200 Subject: [PATCH 07/15] removed replace --- _examples/actions/write_a_readme/go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/_examples/actions/write_a_readme/go.mod b/_examples/actions/write_a_readme/go.mod index 2fd95c5..248ba10 100644 --- a/_examples/actions/write_a_readme/go.mod +++ b/_examples/actions/write_a_readme/go.mod @@ -9,5 +9,3 @@ require ( github.com/itchyny/gojq v0.12.7 // indirect github.com/itchyny/timefmt-go v0.1.3 // indirect ) - -replace kraken => ../../../ -- 2.45.2 From 564147eb6ab7279f4ac40f94d3bbe18aefeebb81 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Tue, 13 Sep 2022 21:56:56 +0200 Subject: [PATCH 08/15] with action --- _examples/actions/write_a_readme/kraken.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/_examples/actions/write_a_readme/kraken.yml b/_examples/actions/write_a_readme/kraken.yml index 778a362..739f293 100644 --- a/_examples/actions/write_a_readme/kraken.yml +++ b/_examples/actions/write_a_readme/kraken.yml @@ -6,3 +6,5 @@ select: provider: - gitea: git.front.kjuulh.io/kraken organisation: "kraken" +action: + command: go run main.go -- 2.45.2 From ce55f6523cfbaa940d45b6380f41770e87958108 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Tue, 13 Sep 2022 22:54:49 +0200 Subject: [PATCH 09/15] with action creator --- _examples/actions/write_a_readme/kraken.yml | 6 +- cmd/kraken/commands/process.go | 9 +- internal/actions/action.go | 16 +++ internal/actions/action_creator.go | 88 ++++++++++++++++ internal/api/process_command.go | 10 +- internal/commands/process_repos.go | 109 ++++++++------------ internal/schema/kraken.go | 25 +++++ internal/serverdeps/server_deps.go | 5 + internal/services/providers/git.go | 17 ++- 9 files changed, 208 insertions(+), 77 deletions(-) create mode 100644 internal/actions/action.go create mode 100644 internal/actions/action_creator.go create mode 100644 internal/schema/kraken.go diff --git a/_examples/actions/write_a_readme/kraken.yml b/_examples/actions/write_a_readme/kraken.yml index 739f293..0b7f7ae 100644 --- a/_examples/actions/write_a_readme/kraken.yml +++ b/_examples/actions/write_a_readme/kraken.yml @@ -3,8 +3,8 @@ name: write-a-readme select: repositories: - git@git.front.kjuulh.io:kjuulh/kraken-test.git - provider: + providers: - gitea: git.front.kjuulh.io/kraken organisation: "kraken" -action: - command: go run main.go +actions: + - "go run main.go" diff --git a/cmd/kraken/commands/process.go b/cmd/kraken/commands/process.go index 1097089..6ed29ee 100644 --- a/cmd/kraken/commands/process.go +++ b/cmd/kraken/commands/process.go @@ -17,9 +17,14 @@ func CreateKrakenProcessCmd() *cobra.Command { var buf bytes.Buffer err := json.NewEncoder(&buf). Encode(struct { - RepositoryUrls []string `json:"repositoryUrls"` + Repository string `json:"repository"` + Branch string `json:"branch"` + Path string `json:"path"` }{ - RepositoryUrls: []string{"git@git.front.kjuulh.io:kjuulh/kraken.git"}}) + Repository: "git@git.front.kjuulh.io:kjuulh/kraken.git", + Branch: "v0.1", + Path: "_examples/actions/write_a_readme/kraken.yml", + }) if err != nil { panic(err) } diff --git a/internal/actions/action.go b/internal/actions/action.go new file mode 100644 index 0000000..3d7e6aa --- /dev/null +++ b/internal/actions/action.go @@ -0,0 +1,16 @@ +package actions + +import ( + "context" + + "git.front.kjuulh.io/kjuulh/kraken/internal/schema" + "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" +) + +type Action struct { + Schema *schema.KrakenSchema +} + +func (a *Action) Execute(ctx context.Context, area *storage.Area) error { + return nil +} diff --git a/internal/actions/action_creator.go b/internal/actions/action_creator.go new file mode 100644 index 0000000..ebdebd3 --- /dev/null +++ b/internal/actions/action_creator.go @@ -0,0 +1,88 @@ +package actions + +import ( + "context" + "fmt" + "os" + "path" + "time" + + "git.front.kjuulh.io/kjuulh/kraken/internal/schema" + "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" + "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" + "go.uber.org/zap" +) + +type ( + ActionCreatorOps struct { + RepositoryUrl string + Branch string + Path string + } + + ActionCreator struct { + logger *zap.Logger + storage *storage.Service + git *providers.Git + } + + ActionCreatorDeps interface { + GetStorageService() *storage.Service + GetGitProvider() *providers.Git + } +) + +func NewActionCreator(logger *zap.Logger, deps ActionCreatorDeps) *ActionCreator { + return &ActionCreator{ + logger: logger, + storage: deps.GetStorageService(), + git: deps.GetGitProvider(), + } +} + +func (ac *ActionCreator) Prepare(ctx context.Context, ops *ActionCreatorOps) (*Action, error) { + area, err := ac.storage.CreateArea(ctx) + if err != nil { + ac.logger.Error("failed to allocate area", zap.Error(err)) + return nil, err + } + + cloneCtx, _ := context.WithTimeout(ctx, time.Second*5) + repo, err := ac.git.Clone(cloneCtx, area, ops.RepositoryUrl) + if err != nil { + ac.logger.Error("could not clone repo", zap.Error(err)) + return nil, err + } + + err = ac.git.Checkout(ctx, repo, ops.Branch) + if err != nil { + return nil, err + } + + executorUrl := path.Join(area.Path, ops.Path) + if _, err = os.Stat(executorUrl); os.IsNotExist(err) { + return nil, fmt.Errorf("path is invalid: %s", ops.Path) + } + + contents, err := os.ReadFile(path.Join(executorUrl, "kraken.yml")) + if err != nil { + return nil, err + } + + krakenSchema, err := schema.Unmarshal(string(contents)) + if err != nil { + return nil, err + } + + return &Action{ + Schema: krakenSchema, + }, nil +} + +func (ac *ActionCreator) Cleanup(ctx context.Context, area *storage.Area) { + ac.logger.Debug("Removing area", zap.String("path", area.Path)) + err := ac.storage.RemoveArea(ctx, area) + if err != nil { + panic(err) + } +} diff --git a/internal/api/process_command.go b/internal/api/process_command.go index 589dd81..549a678 100644 --- a/internal/api/process_command.go +++ b/internal/api/process_command.go @@ -16,7 +16,9 @@ func CommandRoute(logger *zap.Logger, app *gin.Engine, deps *serverdeps.ServerDe commandRoute := app.Group("commands") commandRoute.POST("processRepos", func(c *gin.Context) { type processReposRequest struct { - RepositoryUrls []string `json:"repositoryUrls"` + Repository string `json:"repository"` + Branch string `json:"branch"` + Path string `json:"path"` } var request processReposRequest err := c.BindJSON(&request) @@ -28,11 +30,11 @@ func CommandRoute(logger *zap.Logger, app *gin.Engine, deps *serverdeps.ServerDe jobId := uuid.New().String() - go func(repositoryUrls []string, jobId string) { + go func(repository string, branch string, path string, jobId string) { ctx := context.WithValue(context.Background(), jobs.JobId{}, jobId) processRepos := commands.NewProcessRepos(logger, deps) - err = processRepos.Process(ctx, repositoryUrls) - }(request.RepositoryUrls, jobId) + err = processRepos.Process(ctx, repository, branch, path) + }(request.Repository, request.Branch, request.Path, jobId) c.Status(http.StatusAccepted) }) diff --git a/internal/commands/process_repos.go b/internal/commands/process_repos.go index 45a580a..6b0bdf0 100644 --- a/internal/commands/process_repos.go +++ b/internal/commands/process_repos.go @@ -3,14 +3,10 @@ package commands import ( "context" "fmt" - "io/fs" - "os" - "path" - "path/filepath" "sync" "time" - "git.front.kjuulh.io/kjuulh/kraken/internal/services/actions" + "git.front.kjuulh.io/kjuulh/kraken/internal/actions" "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" "go.uber.org/zap" @@ -18,31 +14,42 @@ import ( type ( ProcessRepos struct { - logger *zap.Logger - storage *storage.Service - git *providers.Git - action *actions.Action + logger *zap.Logger + storage *storage.Service + git *providers.Git + actionCreator *actions.ActionCreator } ProcessReposDeps interface { GetStorageService() *storage.Service GetGitProvider() *providers.Git - GetAction() *actions.Action + GetActionCreator() *actions.ActionCreator } ) func NewProcessRepos(logger *zap.Logger, deps ProcessReposDeps) *ProcessRepos { return &ProcessRepos{ - logger: logger, - storage: deps.GetStorageService(), - git: deps.GetGitProvider(), - action: deps.GetAction(), + logger: logger, + storage: deps.GetStorageService(), + git: deps.GetGitProvider(), + actionCreator: deps.GetActionCreator(), } } -func (pr *ProcessRepos) Process(ctx context.Context, repositoryUrls []string) error { +func (pr *ProcessRepos) Process(ctx context.Context, repository string, branch string, actionPath string) error { errChan := make(chan error, 1) + action, err := pr.actionCreator.Prepare(ctx, &actions.ActionCreatorOps{ + RepositoryUrl: repository, + Branch: branch, + Path: actionPath, + }) + if err != nil { + return err + } + + repositoryUrls := make([]string, 0) + wg := sync.WaitGroup{} wg.Add(len(repositoryUrls)) @@ -51,7 +58,11 @@ func (pr *ProcessRepos) Process(ctx context.Context, repositoryUrls []string) er defer func() { wg.Done() }() - pr.processRepo(ctx, repoUrl, errChan) + err := pr.processRepo(ctx, repoUrl, action) + if err != nil { + pr.logger.Error("could not process repo", zap.Error(err)) + errChan <- err + } }(ctx, repoUrl) } @@ -66,74 +77,39 @@ func (pr *ProcessRepos) Process(ctx context.Context, repositoryUrls []string) er return nil } -func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, errChan chan error) { +func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, action *actions.Action) error { cleanup, area, err := pr.prepareAction(ctx) defer func() { if cleanup != nil { - cleanup(ctx, errChan) + cleanup(ctx) } }() if err != nil { - errChan <- err - return + return err } repo, err := pr.clone(ctx, area, repoUrl) if err != nil { - errChan <- err - return + return err } - err = pr.action.Run( - ctx, - area, - func(_ context.Context, area *storage.Area) (bool, error) { - pr.logger.Debug("checking predicate", zap.String("area", area.Path)) - - // TODO: Run predicate instead - contains := false - filepath.WalkDir(area.Path, func(_ string, d fs.DirEntry, _ error) error { - if d.Name() == "roadmap.md" { - contains = true - } - return nil - }) - return contains, nil - }, - func(_ context.Context, area *storage.Area) error { - pr.logger.Debug("running action", zap.String("area", area.Path)) - - // TODO: Run action instead - readme := path.Join(area.Path, "README.md") - file, err := os.Create(readme) - if err != nil { - return fmt.Errorf("could not create readme: %w", err) - } - - _, err = file.WriteString("# Readme") - if err != nil { - return fmt.Errorf("could not write readme: %w", err) - } - - err = pr.commit(ctx, area, repo) - if err != nil { - return err - } - - return nil - }, false) + err = action.Execute(ctx, area) if err != nil { - pr.logger.Error("could not run action", zap.Error(err)) - errChan <- err - return + return err + } + + err = pr.commit(ctx, area, repo) + if err != nil { + return err } pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) + return nil } func (pr *ProcessRepos) prepareAction( ctx context.Context, -) (func(ctx context.Context, errChan chan error), *storage.Area, error) { +) (func(ctx context.Context), *storage.Area, error) { pr.logger.Debug("Creating area") area, err := pr.storage.CreateArea(ctx) if err != nil { @@ -141,12 +117,11 @@ func (pr *ProcessRepos) prepareAction( return nil, nil, err } - cleanupfunc := func(ctx context.Context, errChan chan error) { + cleanupfunc := func(ctx context.Context) { pr.logger.Debug("Removing area", zap.String("path", area.Path)) err = pr.storage.RemoveArea(ctx, area) if err != nil { - errChan <- err - return + panic(err) } } diff --git a/internal/schema/kraken.go b/internal/schema/kraken.go new file mode 100644 index 0000000..13881bf --- /dev/null +++ b/internal/schema/kraken.go @@ -0,0 +1,25 @@ +package schema + +import "gopkg.in/yaml.v3" + +type KrakenSchema struct { + ApiVersion string `yaml:"apiVersion"` + Name string `yaml:"name"` + Select struct { + Repositories []string `yaml:"repositories"` + Providers []struct { + Gitea string `yaml:"gitea"` + Organisation string `yaml:"organisation"` + } `yaml:"providers"` + } `yaml:"select"` + Actions []string `yaml:"actions"` +} + +func Unmarshal(raw string) (*KrakenSchema, error) { + k := &KrakenSchema{} + err := yaml.Unmarshal([]byte(raw), k) + if err != nil { + return nil, err + } + return k, nil +} diff --git a/internal/serverdeps/server_deps.go b/internal/serverdeps/server_deps.go index bfa6d39..d81c052 100644 --- a/internal/serverdeps/server_deps.go +++ b/internal/serverdeps/server_deps.go @@ -1,6 +1,7 @@ package serverdeps import ( + actionc "git.front.kjuulh.io/kjuulh/kraken/internal/actions" "git.front.kjuulh.io/kjuulh/kraken/internal/services/actions" "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" "git.front.kjuulh.io/kjuulh/kraken/internal/services/signer" @@ -59,6 +60,10 @@ func (deps *ServerDeps) GetAction() *actions.Action { return actions.NewAction(deps.logger.With(zap.Namespace("action"))) } +func (deps *ServerDeps) GetActionCreator() *actionc.ActionCreator { + return actionc.NewActionCreator(deps.logger.With(zap.Namespace("action")), deps) +} + func (deps *ServerDeps) GetOpenPGP() *signer.OpenPGP { return deps.openPGP } diff --git a/internal/services/providers/git.go b/internal/services/providers/git.go index f4dee7f..4a19e57 100644 --- a/internal/services/providers/git.go +++ b/internal/services/providers/git.go @@ -70,7 +70,7 @@ func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl stri Auth: auth, RemoteName: "origin", ReferenceName: "refs/heads/main", - SingleBranch: true, + SingleBranch: false, NoCheckout: false, Depth: 1, RecurseSubmodules: 1, @@ -90,6 +90,21 @@ func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl stri return &GitRepo{repo: repo}, nil } +func (g *Git) Checkout(ctx context.Context, gitRepo *GitRepo, branch string) error { + wt, err := gitRepo.repo.Worktree() + if err != nil { + return err + } + + return wt.Checkout(&git.CheckoutOptions{ + Hash: [20]byte{}, + Branch: plumbing.NewBranchReferenceName(branch), + Create: false, + Force: false, + Keep: false, + }) +} + func (g *Git) getProgressWriter() *zapio.Writer { return &zapio.Writer{ Log: g.logger.With(zap.String("process", "go-git")), -- 2.45.2 From 0cb923a3a503974f7dd87f11969ddc6d81aa1c06 Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Wed, 14 Sep 2022 23:40:58 +0200 Subject: [PATCH 10/15] feature/add-actions (#9) Co-authored-by: kjuulh Reviewed-on: https://git.front.kjuulh.io/kjuulh/kraken/pulls/9 --- _examples/actions/write_a_readme/kraken.yml | 137 +++++++++++++++++++- _examples/actions/write_a_readme/main.go | 5 +- cmd/kraken/commands/process.go | 6 +- cmd/server/server.go | 2 + internal/actions/action.go | 25 +++- internal/actions/action_creator.go | 13 +- internal/actions/builders/go.go | 46 +++++++ internal/api/process_command.go | 3 + internal/commands/process_repos.go | 15 +-- internal/schema/kraken.go | 5 +- internal/services/providers/git.go | 48 +++++-- roadmap.md | 12 +- 12 files changed, 275 insertions(+), 42 deletions(-) create mode 100644 internal/actions/builders/go.go diff --git a/_examples/actions/write_a_readme/kraken.yml b/_examples/actions/write_a_readme/kraken.yml index 0b7f7ae..77db45b 100644 --- a/_examples/actions/write_a_readme/kraken.yml +++ b/_examples/actions/write_a_readme/kraken.yml @@ -3,8 +3,143 @@ name: write-a-readme select: repositories: - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + - git@git.front.kjuulh.io:kjuulh/kraken-test.git providers: - gitea: git.front.kjuulh.io/kraken organisation: "kraken" actions: - - "go run main.go" + - type: go + entry: "main.go" diff --git a/_examples/actions/write_a_readme/main.go b/_examples/actions/write_a_readme/main.go index a5bd52f..5c24133 100644 --- a/_examples/actions/write_a_readme/main.go +++ b/_examples/actions/write_a_readme/main.go @@ -3,7 +3,10 @@ package main import "github.com/bitfield/script" func main() { - script. + _, err := script. Echo("# Readme"). WriteFile("README.md") + if err != nil { + panic(err) + } } diff --git a/cmd/kraken/commands/process.go b/cmd/kraken/commands/process.go index 6ed29ee..274b299 100644 --- a/cmd/kraken/commands/process.go +++ b/cmd/kraken/commands/process.go @@ -11,7 +11,7 @@ import ( func CreateKrakenProcessCmd() *cobra.Command { cmd := &cobra.Command{ Use: "process", - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { client := http.Client{} var buf bytes.Buffer @@ -22,8 +22,8 @@ func CreateKrakenProcessCmd() *cobra.Command { Path string `json:"path"` }{ Repository: "git@git.front.kjuulh.io:kjuulh/kraken.git", - Branch: "v0.1", - Path: "_examples/actions/write_a_readme/kraken.yml", + Branch: "feature/add-actions", + Path: "_examples/actions/write_a_readme/", }) if err != nil { panic(err) diff --git a/cmd/server/server.go b/cmd/server/server.go index 49b713c..5c835ec 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -15,6 +15,8 @@ func main() { } _ = logger.Sync() + zap.ReplaceGlobals(logger) + Execute(logger) } diff --git a/internal/actions/action.go b/internal/actions/action.go index 3d7e6aa..3c6416e 100644 --- a/internal/actions/action.go +++ b/internal/actions/action.go @@ -2,15 +2,38 @@ package actions import ( "context" + "errors" + "git.front.kjuulh.io/kjuulh/kraken/internal/actions/builders" "git.front.kjuulh.io/kjuulh/kraken/internal/schema" "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" + "go.uber.org/zap" ) type Action struct { - Schema *schema.KrakenSchema + Schema *schema.KrakenSchema + SchemaPath string } func (a *Action) Execute(ctx context.Context, area *storage.Area) error { + for _, action := range a.Schema.Actions { + switch action.Type { + case "go": + exe, err := builders.NewGo(zap.L()).Build(ctx, a.SchemaPath, action.Entry) + if err != nil { + return err + } + err = exe(ctx, area.Path) + if err != nil { + return err + } + + zap.L().Debug("Execution done") + + default: + return errors.New("could not determine action type") + } + } + return nil } diff --git a/internal/actions/action_creator.go b/internal/actions/action_creator.go index ebdebd3..b731519 100644 --- a/internal/actions/action_creator.go +++ b/internal/actions/action_creator.go @@ -47,18 +47,13 @@ func (ac *ActionCreator) Prepare(ctx context.Context, ops *ActionCreatorOps) (*A return nil, err } - cloneCtx, _ := context.WithTimeout(ctx, time.Second*5) - repo, err := ac.git.Clone(cloneCtx, area, ops.RepositoryUrl) + cloneCtx, _ := context.WithTimeout(ctx, time.Second*10) + _, err = ac.git.CloneBranch(cloneCtx, area, ops.RepositoryUrl, ops.Branch) if err != nil { ac.logger.Error("could not clone repo", zap.Error(err)) return nil, err } - err = ac.git.Checkout(ctx, repo, ops.Branch) - if err != nil { - return nil, err - } - executorUrl := path.Join(area.Path, ops.Path) if _, err = os.Stat(executorUrl); os.IsNotExist(err) { return nil, fmt.Errorf("path is invalid: %s", ops.Path) @@ -74,8 +69,10 @@ func (ac *ActionCreator) Prepare(ctx context.Context, ops *ActionCreatorOps) (*A return nil, err } + ac.logger.Debug("Action creator done") return &Action{ - Schema: krakenSchema, + Schema: krakenSchema, + SchemaPath: executorUrl, }, nil } diff --git a/internal/actions/builders/go.go b/internal/actions/builders/go.go new file mode 100644 index 0000000..6d828ee --- /dev/null +++ b/internal/actions/builders/go.go @@ -0,0 +1,46 @@ +package builders + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + + "go.uber.org/zap" +) + +type Go struct { + logger *zap.Logger +} + +func NewGo(logger *zap.Logger) *Go { + return &Go{logger: logger} +} + +type GoExecutable func(ctx context.Context, victimPath string) error + +func (g *Go) Build(ctx context.Context, modulePath, entryPath string) (GoExecutable, error) { + g.logger.Debug("Building go binary", zap.String("actiondir", modulePath), zap.String("entry", entryPath)) + + if _, err := os.Stat(fmt.Sprintf("%s/%s", modulePath, entryPath)); os.IsNotExist(err) { + return nil, errors.New("could not find entry") + } + + err := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + fmt.Sprintf("(cd %s; go build -o main %s)", modulePath, entryPath), + ).Run() + if err != nil { + return nil, err + } + + g.logger.Debug("Go binary built!") + + return func(ctx context.Context, victimPath string) error { + g.logger.Debug("Executing script", zap.String("victim", victimPath)) + return exec.CommandContext(ctx, "/bin/bash", "-c", fmt.Sprintf("(cd %s; %s/main)", victimPath, modulePath)).Run() + }, nil +} diff --git a/internal/api/process_command.go b/internal/api/process_command.go index 549a678..2e852b5 100644 --- a/internal/api/process_command.go +++ b/internal/api/process_command.go @@ -34,6 +34,9 @@ func CommandRoute(logger *zap.Logger, app *gin.Engine, deps *serverdeps.ServerDe ctx := context.WithValue(context.Background(), jobs.JobId{}, jobId) processRepos := commands.NewProcessRepos(logger, deps) err = processRepos.Process(ctx, repository, branch, path) + if err != nil { + logger.Error("could not process repo", zap.Error(err)) + } }(request.Repository, request.Branch, request.Path, jobId) c.Status(http.StatusAccepted) diff --git a/internal/commands/process_repos.go b/internal/commands/process_repos.go index 6b0bdf0..d29a30b 100644 --- a/internal/commands/process_repos.go +++ b/internal/commands/process_repos.go @@ -48,7 +48,7 @@ func (pr *ProcessRepos) Process(ctx context.Context, repository string, branch s return err } - repositoryUrls := make([]string, 0) + repositoryUrls := action.Schema.Select.Repositories wg := sync.WaitGroup{} wg.Add(len(repositoryUrls)) @@ -60,7 +60,6 @@ func (pr *ProcessRepos) Process(ctx context.Context, repository string, branch s }() err := pr.processRepo(ctx, repoUrl, action) if err != nil { - pr.logger.Error("could not process repo", zap.Error(err)) errChan <- err } }(ctx, repoUrl) @@ -113,7 +112,6 @@ func (pr *ProcessRepos) prepareAction( pr.logger.Debug("Creating area") area, err := pr.storage.CreateArea(ctx) if err != nil { - pr.logger.Error("failed to allocate area", zap.Error(err)) return nil, nil, err } @@ -133,13 +131,11 @@ func (pr *ProcessRepos) clone(ctx context.Context, area *storage.Area, repoUrl s cloneCtx, _ := context.WithTimeout(ctx, time.Second*5) repo, err := pr.git.Clone(cloneCtx, area, repoUrl) if err != nil { - pr.logger.Error("could not clone repo", zap.Error(err)) return nil, err } err = pr.git.CreateBranch(ctx, repo) if err != nil { - pr.logger.Error("could not create branch", zap.Error(err)) return nil, err } @@ -157,9 +153,12 @@ func (pr *ProcessRepos) commit(ctx context.Context, area *storage.Area, repo *pr return fmt.Errorf("could not get diff: %w", err) } - err = pr.git.Push(ctx, repo) - if err != nil { - return fmt.Errorf("could not push to repo: %w", err) + dryrun := true + if !dryrun { + err = pr.git.Push(ctx, repo) + if err != nil { + return fmt.Errorf("could not push to repo: %w", err) + } } return nil } diff --git a/internal/schema/kraken.go b/internal/schema/kraken.go index 13881bf..fb7d75e 100644 --- a/internal/schema/kraken.go +++ b/internal/schema/kraken.go @@ -12,7 +12,10 @@ type KrakenSchema struct { Organisation string `yaml:"organisation"` } `yaml:"providers"` } `yaml:"select"` - Actions []string `yaml:"actions"` + Actions []struct { + Type string `yaml:"type"` + Entry string `yaml:"entry"` + } `yaml:"actions"` } func Unmarshal(raw string) (*KrakenSchema, error) { diff --git a/internal/services/providers/git.go b/internal/services/providers/git.go index 4a19e57..4a91121 100644 --- a/internal/services/providers/git.go +++ b/internal/services/providers/git.go @@ -53,7 +53,7 @@ func NewGit(logger *zap.Logger, gitConfig *GitConfig, openPGP *signer.OpenPGP) * return &Git{logger: logger, gitConfig: gitConfig, openPGP: openPGP} } -func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl string) (*GitRepo, error) { +func (g *Git) CloneBranch(ctx context.Context, storageArea *storage.Area, repoUrl string, branch string) (*GitRepo, error) { g.logger.Debug( "cloning repository", zap.String("repoUrl", repoUrl), @@ -69,8 +69,8 @@ func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl stri URL: repoUrl, Auth: auth, RemoteName: "origin", - ReferenceName: "refs/heads/main", - SingleBranch: false, + ReferenceName: plumbing.NewBranchReferenceName(branch), + SingleBranch: true, NoCheckout: false, Depth: 1, RecurseSubmodules: 1, @@ -90,19 +90,41 @@ func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl stri return &GitRepo{repo: repo}, nil } -func (g *Git) Checkout(ctx context.Context, gitRepo *GitRepo, branch string) error { - wt, err := gitRepo.repo.Worktree() +func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl string) (*GitRepo, error) { + g.logger.Debug( + "cloning repository", + zap.String("repoUrl", repoUrl), + zap.String("path", storageArea.Path), + ) + + auth, err := g.GetAuth() if err != nil { - return err + return nil, err } - return wt.Checkout(&git.CheckoutOptions{ - Hash: [20]byte{}, - Branch: plumbing.NewBranchReferenceName(branch), - Create: false, - Force: false, - Keep: false, - }) + cloneOptions := git.CloneOptions{ + URL: repoUrl, + Auth: auth, + RemoteName: "origin", + ReferenceName: "refs/heads/main", + SingleBranch: true, + NoCheckout: false, + Depth: 1, + RecurseSubmodules: 1, + Progress: g.getProgressWriter(), + Tags: 0, + InsecureSkipTLS: false, + CABundle: []byte{}, + } + + repo, err := git.PlainCloneContext(ctx, storageArea.Path, false, &cloneOptions) + if err != nil { + return nil, err + } + + g.logger.Debug("done cloning repo") + + return &GitRepo{repo: repo}, nil } func (g *Git) getProgressWriter() *zapio.Writer { diff --git a/roadmap.md b/roadmap.md index ba55870..dbb3f3e 100644 --- a/roadmap.md +++ b/roadmap.md @@ -11,15 +11,15 @@ ### Not in scope -- [ ] Pooled runners -- [ ] CLI with options -- [ ] Server app -- [ ] Git hosting providers +- Pooled runners +- CLI with options +- Server app +- Git hosting providers ## Version 0.1 -- [ ] Setup a way to choose actions and predicates -- [ ] Allow instantiation of actions, kraken template repo etc. +- [x] Setup a way to choose actions and predicates +- [x] Allow instantiation of actions, kraken template repo etc. - [ ] Create pr for gitea provider - [ ] Think about some sort of isolation - [ ] Create CLI to trigger action -- 2.45.2 From 9696270d226207dc991d298fe08e3435e9623a94 Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Sun, 18 Sep 2022 00:10:44 +0200 Subject: [PATCH 11/15] feature/gitea-integration (#10) Co-authored-by: kjuulh Reviewed-on: https://git.front.kjuulh.io/kjuulh/kraken/pulls/10 --- _examples/actions/write_a_readme/kraken.yml | 140 +------------------- cmd/kraken/commands/process.go | 2 +- cuddle.yaml | 6 + go.mod | 3 + go.sum | 12 ++ internal/commands/process_repos.go | 88 ++++++++++-- internal/gitproviders/gitea.go | 133 +++++++++++++++++++ internal/serverdeps/server_deps.go | 5 + internal/services/providers/git.go | 37 ++++++ roadmap.md | 4 +- scripts/run.sh | 15 +++ scripts/run_client.sh | 5 + scripts/run_server.sh | 5 + 13 files changed, 302 insertions(+), 153 deletions(-) create mode 100644 internal/gitproviders/gitea.go create mode 100755 scripts/run.sh create mode 100755 scripts/run_client.sh create mode 100755 scripts/run_server.sh diff --git a/_examples/actions/write_a_readme/kraken.yml b/_examples/actions/write_a_readme/kraken.yml index 77db45b..0e5c44e 100644 --- a/_examples/actions/write_a_readme/kraken.yml +++ b/_examples/actions/write_a_readme/kraken.yml @@ -3,143 +3,9 @@ name: write-a-readme select: repositories: - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - - git@git.front.kjuulh.io:kjuulh/kraken-test.git - providers: - - gitea: git.front.kjuulh.io/kraken - organisation: "kraken" + # providers: + # - gitea: https://git.front.kjuulh.io + # organisation: "cibus" actions: - type: go entry: "main.go" diff --git a/cmd/kraken/commands/process.go b/cmd/kraken/commands/process.go index 274b299..342ebc4 100644 --- a/cmd/kraken/commands/process.go +++ b/cmd/kraken/commands/process.go @@ -22,7 +22,7 @@ func CreateKrakenProcessCmd() *cobra.Command { Path string `json:"path"` }{ Repository: "git@git.front.kjuulh.io:kjuulh/kraken.git", - Branch: "feature/add-actions", + Branch: "feature/gitea-integration", Path: "_examples/actions/write_a_readme/", }) if err != nil { diff --git a/cuddle.yaml b/cuddle.yaml index 8b285a2..87fafac 100644 --- a/cuddle.yaml +++ b/cuddle.yaml @@ -9,3 +9,9 @@ vars: scripts: push_github: type: shell + run_server: + type: shell + run_client: + type: shell + run: + type: shell diff --git a/go.mod b/go.mod index cd1b597..2fab6ad 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( ) require ( + code.gitea.io/sdk/gitea v0.15.1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/ProtonMail/go-mime v0.0.0-20220302105931-303f85f7fe0f // indirect github.com/acomagu/bufpipe v1.0.3 // indirect @@ -30,6 +31,7 @@ require ( github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-playground/validator/v10 v10.10.0 // indirect github.com/goccy/go-json v0.9.7 // indirect + github.com/hashicorp/go-version v1.2.1 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -47,6 +49,7 @@ require ( github.com/sirupsen/logrus v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ugorji/go/codec v1.2.7 // indirect + github.com/whilp/git-urls v1.0.0 // indirect github.com/xanzy/ssh-agent v0.3.2 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect diff --git a/go.sum b/go.sum index 03a8012..5f09503 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,6 @@ +code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE= +code.gitea.io/sdk/gitea v0.15.1 h1:WJreC7YYuxbn0UDaPuWIe/mtiNKTvLN8MLkaw71yx/M= +code.gitea.io/sdk/gitea v0.15.1/go.mod h1:klY2LVI3s3NChzIk/MzMn7G1FHrfU7qd63iSMVoHRBA= git.front.kjuulh.io/kjuulh/curre v1.2.2 h1:0OwWIfekrMykdQg9bdmG80I+Mjc2k4i+sy903phuDWs= git.front.kjuulh.io/kjuulh/curre v1.2.2/go.mod h1:m7WpSehONLqPh/XF3F0BI0UOpLOfGuDmDEFI1XsM6fE= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -71,6 +74,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -148,9 +153,12 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= +github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xanzy/ssh-agent v0.3.2 h1:eKj4SX2Fe7mui28ZgnFW5fmTz1EIr7ugo5s6wDxdHBM= github.com/xanzy/ssh-agent v0.3.2/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= @@ -184,10 +192,12 @@ golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -195,6 +205,7 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -233,6 +244,7 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/commands/process_repos.go b/internal/commands/process_repos.go index d29a30b..ae8049d 100644 --- a/internal/commands/process_repos.go +++ b/internal/commands/process_repos.go @@ -3,12 +3,16 @@ package commands import ( "context" "fmt" + "strings" "sync" "time" "git.front.kjuulh.io/kjuulh/kraken/internal/actions" + "git.front.kjuulh.io/kjuulh/kraken/internal/gitproviders" + "git.front.kjuulh.io/kjuulh/kraken/internal/schema" "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" + giturls "github.com/whilp/git-urls" "go.uber.org/zap" ) @@ -18,12 +22,14 @@ type ( storage *storage.Service git *providers.Git actionCreator *actions.ActionCreator + gitea *gitproviders.Gitea } ProcessReposDeps interface { GetStorageService() *storage.Service GetGitProvider() *providers.Git GetActionCreator() *actions.ActionCreator + GetGitea() *gitproviders.Gitea } ) @@ -33,12 +39,11 @@ func NewProcessRepos(logger *zap.Logger, deps ProcessReposDeps) *ProcessRepos { storage: deps.GetStorageService(), git: deps.GetGitProvider(), actionCreator: deps.GetActionCreator(), + gitea: deps.GetGitea(), } } func (pr *ProcessRepos) Process(ctx context.Context, repository string, branch string, actionPath string) error { - errChan := make(chan error, 1) - action, err := pr.actionCreator.Prepare(ctx, &actions.ActionCreatorOps{ RepositoryUrl: repository, Branch: branch, @@ -48,7 +53,10 @@ func (pr *ProcessRepos) Process(ctx context.Context, repository string, branch s return err } - repositoryUrls := action.Schema.Select.Repositories + repositoryUrls, err := pr.getRepoUrls(ctx, action.Schema) + if err != nil { + return err + } wg := sync.WaitGroup{} wg.Add(len(repositoryUrls)) @@ -60,22 +68,34 @@ func (pr *ProcessRepos) Process(ctx context.Context, repository string, branch s }() err := pr.processRepo(ctx, repoUrl, action) if err != nil { - errChan <- err + pr.logger.Error("could not process repo", zap.Error(err)) } }(ctx, repoUrl) } wg.Wait() - close(errChan) - pr.logger.Debug("finished processing all repos") - - for err := range errChan { - return err - } + pr.logger.Debug("finished processing all repos", zap.Strings("repos", repositoryUrls)) return nil } +func (pr *ProcessRepos) getRepoUrls(ctx context.Context, schema *schema.KrakenSchema) ([]string, error) { + repoUrls := make([]string, 0) + + repoUrls = append(repoUrls, schema.Select.Repositories...) + + for _, provider := range schema.Select.Providers { + repos, err := pr.gitea.ListRepositoriesForOrganization(ctx, provider.Gitea, provider.Organisation) + if err != nil { + return nil, err + } + + repoUrls = append(repoUrls, repos...) + } + + return repoUrls, nil +} + func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, action *actions.Action) error { cleanup, area, err := pr.prepareAction(ctx) defer func() { @@ -97,7 +117,7 @@ func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, action return err } - err = pr.commit(ctx, area, repo) + err = pr.commit(ctx, area, repo, repoUrl) if err != nil { return err } @@ -142,8 +162,8 @@ func (pr *ProcessRepos) clone(ctx context.Context, area *storage.Area, repoUrl s return repo, nil } -func (pr *ProcessRepos) commit(ctx context.Context, area *storage.Area, repo *providers.GitRepo) error { - _, err := pr.git.Add(ctx, area, repo) +func (pr *ProcessRepos) commit(ctx context.Context, area *storage.Area, repo *providers.GitRepo, repoUrl string) error { + wt, err := pr.git.Add(ctx, area, repo) if err != nil { return fmt.Errorf("could not add file: %w", err) } @@ -153,12 +173,52 @@ func (pr *ProcessRepos) commit(ctx context.Context, area *storage.Area, repo *pr return fmt.Errorf("could not get diff: %w", err) } - dryrun := true + dryrun := false if !dryrun { + status, err := wt.Status() + if err != nil { + return err + } + + if status.IsClean() { + pr.logger.Info("Returning early, as no modifications are detected") + return nil + } + err = pr.git.Push(ctx, repo) if err != nil { return fmt.Errorf("could not push to repo: %w", err) } + + url, err := giturls.Parse(repoUrl) + if err != nil { + return err + } + + head, err := repo.GetHEAD() + if err != nil { + return err + } + + path := strings.Split(url.Path, "/") + pr.logger.Debug("path string", zap.Strings("paths", path), zap.String("HEAD", head)) + + org := path[0] + repoName := path[1] + semanticName, _, ok := strings.Cut(repoName, ".") + if !ok { + semanticName = repoName + } + + originHead, err := pr.git.GetOriginHEADForRepo(ctx, repo) + if err != nil { + return err + } + + err = pr.gitea.CreatePr(ctx, fmt.Sprintf("%s://%s", "https", url.Host), org, semanticName, head, originHead, "kraken-apply") + if err != nil { + return err + } } return nil } diff --git a/internal/gitproviders/gitea.go b/internal/gitproviders/gitea.go new file mode 100644 index 0000000..8fa3690 --- /dev/null +++ b/internal/gitproviders/gitea.go @@ -0,0 +1,133 @@ +package gitproviders + +import ( + "context" + "fmt" + "sync" + + "code.gitea.io/sdk/gitea" + "go.uber.org/zap" +) + +type Gitea struct { + logger *zap.Logger + giteamu sync.Mutex + giteaClients map[string]*gitea.Client +} + +func NewGitea(logger *zap.Logger) *Gitea { + return &Gitea{ + logger: logger, + giteamu: sync.Mutex{}, + giteaClients: make(map[string]*gitea.Client, 0), + } +} + +func (g *Gitea) ListRepositoriesForOrganization( + ctx context.Context, + server string, + organization string, +) ([]string, error) { + client, err := g.getOrCreateClient(ctx, server) + if err != nil { + return nil, err + } + + g.logger.Debug("Listing repos for gitea", zap.String("server", server)) + repos, resp, err := client.ListOrgRepos(organization, gitea.ListOrgReposOptions{ + ListOptions: gitea.ListOptions{ + Page: 0, + PageSize: 20, + }, + }) + if err != nil { + return nil, fmt.Errorf("could not list repos: %w", err) + } + + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("gitea responded with a non 200 status code (gitea response: %s)", resp.Status) + } + + repoUrls := make([]string, len(repos)) + for i, repo := range repos { + repoUrls[i] = repo.SSHURL + } + + return repoUrls, err +} + +func (g *Gitea) CreatePr( + ctx context.Context, + server string, + organization string, + repository string, + head string, + base string, + actionName string, +) error { + client, err := g.getOrCreateClient(ctx, server) + if err != nil { + return err + } + + prs, _, err := client.ListRepoPullRequests(organization, repository, gitea.ListPullRequestsOptions{ + ListOptions: gitea.ListOptions{ + Page: 0, + PageSize: 30, + }, + State: gitea.StateOpen, + Sort: "recentupdate", + Milestone: 0, + }) + if err != nil { + return fmt.Errorf( + "could not list repos, needed because we need to check for conflicts. Original error: %w", + err, + ) + } + for _, pr := range prs { + if pr.Head.Name == head { + g.logger.Info( + "returning early from creating pull-request, as it already exists.", + zap.String("repository", repository), + zap.String("pull-request", pr.URL), + ) + return nil + } + } + + pr, _, err := client.CreatePullRequest(organization, repository, gitea.CreatePullRequestOption{ + Head: head, + Base: base, + Title: actionName, + }) + if err != nil { + return err + } + + g.logger.Debug( + "Created pr", + zap.String("repository", repository), + zap.String("branch", head), + zap.String("pull-request", pr.URL), + ) + + return nil +} + +func (g *Gitea) getOrCreateClient(ctx context.Context, server string) (*gitea.Client, error) { + g.giteamu.Lock() + defer g.giteamu.Unlock() + client, ok := g.giteaClients[server] + if !ok || client == nil { + c, err := gitea.NewClient(server) + c.SetBasicAuth("kjuulh", "c0bd801cc9a7f2ed559ea45d603afc92f5443f19") + if err != nil { + return nil, err + } + g.giteaClients[server] = c + return c, nil + } + + return client, nil +} diff --git a/internal/serverdeps/server_deps.go b/internal/serverdeps/server_deps.go index d81c052..da9c333 100644 --- a/internal/serverdeps/server_deps.go +++ b/internal/serverdeps/server_deps.go @@ -2,6 +2,7 @@ package serverdeps import ( actionc "git.front.kjuulh.io/kjuulh/kraken/internal/actions" + "git.front.kjuulh.io/kjuulh/kraken/internal/gitproviders" "git.front.kjuulh.io/kjuulh/kraken/internal/services/actions" "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" "git.front.kjuulh.io/kjuulh/kraken/internal/services/signer" @@ -64,6 +65,10 @@ func (deps *ServerDeps) GetActionCreator() *actionc.ActionCreator { return actionc.NewActionCreator(deps.logger.With(zap.Namespace("action")), deps) } +func (deps *ServerDeps) GetGitea() *gitproviders.Gitea { + return gitproviders.NewGitea(deps.logger.With(zap.Namespace("gitea"))) +} + func (deps *ServerDeps) GetOpenPGP() *signer.OpenPGP { return deps.openPGP } diff --git a/internal/services/providers/git.go b/internal/services/providers/git.go index 4a91121..07018b6 100644 --- a/internal/services/providers/git.go +++ b/internal/services/providers/git.go @@ -30,6 +30,15 @@ type GitRepo struct { repo *git.Repository } +func (gr *GitRepo) GetHEAD() (string, error) { + head, err := gr.repo.Head() + if err != nil { + return "", err + } + + return head.Name().Short(), nil +} + type GitAuth string const ( @@ -53,6 +62,34 @@ func NewGit(logger *zap.Logger, gitConfig *GitConfig, openPGP *signer.OpenPGP) * return &Git{logger: logger, gitConfig: gitConfig, openPGP: openPGP} } +func (g *Git) GetOriginHEADForRepo(ctx context.Context, gitRepo *GitRepo) (string, error) { + remote, err := gitRepo.repo.Remote("origin") + if err != nil { + return "", err + } + + auth, err := g.GetAuth() + if err != nil { + return "", err + } + + refs, err := remote.ListContext(ctx, &git.ListOptions{ + Auth: auth, + }) + if err != nil { + return "", err + } + + headRef := "" + for _, ref := range refs { + if !ref.Name().IsBranch() { + headRef = ref.Target().Short() + } + } + + return headRef, nil +} + func (g *Git) CloneBranch(ctx context.Context, storageArea *storage.Area, repoUrl string, branch string) (*GitRepo, error) { g.logger.Debug( "cloning repository", diff --git a/roadmap.md b/roadmap.md index dbb3f3e..ae23587 100644 --- a/roadmap.md +++ b/roadmap.md @@ -20,7 +20,9 @@ - [x] Setup a way to choose actions and predicates - [x] Allow instantiation of actions, kraken template repo etc. -- [ ] Create pr for gitea provider +- [ ] Implement docker action +- [ ] Providing query results +- [x] Create pr for gitea provider - [ ] Think about some sort of isolation - [ ] Create CLI to trigger action - [ ] Setup pool of runners diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100755 index 0000000..e2f75ac --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +run_server="cuddle_cli x run_server" + +$run_server & + +sleep 1s + +cuddle_cli x run_client + +sleep 5s + +kill %1 diff --git a/scripts/run_client.sh b/scripts/run_client.sh new file mode 100755 index 0000000..cfe216e --- /dev/null +++ b/scripts/run_client.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +go run cmd/kraken/kraken.go process diff --git a/scripts/run_server.sh b/scripts/run_server.sh new file mode 100755 index 0000000..6fb5602 --- /dev/null +++ b/scripts/run_server.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +go run cmd/server/server.go start -- 2.45.2 From 1ff0014ad4f29795dd3a0d85f0360de586a1dd01 Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Sun, 18 Sep 2022 11:51:22 +0200 Subject: [PATCH 12/15] feature/docker-action (#11) Co-authored-by: kjuulh Reviewed-on: https://git.front.kjuulh.io/kjuulh/kraken/pulls/11 --- .gitignore | 1 + _examples/actions/docker_action/Dockerfile | 8 ++ _examples/actions/docker_action/entry.sh | 5 ++ _examples/actions/docker_action/go.mod | 11 +++ _examples/actions/docker_action/go.sum | 20 +++++ _examples/actions/docker_action/kraken.yml | 11 +++ cmd/kraken/commands/process.go | 4 +- internal/actions/action.go | 12 +++ internal/actions/builders/docker.go | 95 ++++++++++++++++++++++ internal/commands/process_repos.go | 19 +++-- internal/gitproviders/gitea.go | 12 ++- internal/services/providers/git.go | 21 +++-- roadmap.md | 2 +- scripts/run_server.sh | 2 + 14 files changed, 202 insertions(+), 21 deletions(-) create mode 100644 _examples/actions/docker_action/Dockerfile create mode 100755 _examples/actions/docker_action/entry.sh create mode 100644 _examples/actions/docker_action/go.mod create mode 100644 _examples/actions/docker_action/go.sum create mode 100644 _examples/actions/docker_action/kraken.yml create mode 100644 internal/actions/builders/docker.go diff --git a/.gitignore b/.gitignore index 75d1871..dddecf9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .cuddle/ +.env diff --git a/_examples/actions/docker_action/Dockerfile b/_examples/actions/docker_action/Dockerfile new file mode 100644 index 0000000..572de73 --- /dev/null +++ b/_examples/actions/docker_action/Dockerfile @@ -0,0 +1,8 @@ +FROM debian:bullseye-slim + +# Kraken relies on this path being the specified path +WORKDIR /src/work/ + +COPY entry.sh /src/script.sh + +CMD [ "/src/script.sh" ] diff --git a/_examples/actions/docker_action/entry.sh b/_examples/actions/docker_action/entry.sh new file mode 100755 index 0000000..11771f4 --- /dev/null +++ b/_examples/actions/docker_action/entry.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +echo "# README docker" > README.md diff --git a/_examples/actions/docker_action/go.mod b/_examples/actions/docker_action/go.mod new file mode 100644 index 0000000..248ba10 --- /dev/null +++ b/_examples/actions/docker_action/go.mod @@ -0,0 +1,11 @@ +module write_a_readme + +go 1.19 + +require github.com/bitfield/script v0.20.2 + +require ( + bitbucket.org/creachadair/shell v0.0.7 // indirect + github.com/itchyny/gojq v0.12.7 // indirect + github.com/itchyny/timefmt-go v0.1.3 // indirect +) diff --git a/_examples/actions/docker_action/go.sum b/_examples/actions/docker_action/go.sum new file mode 100644 index 0000000..234eb1e --- /dev/null +++ b/_examples/actions/docker_action/go.sum @@ -0,0 +1,20 @@ +bitbucket.org/creachadair/shell v0.0.7 h1:Z96pB6DkSb7F3Y3BBnJeOZH2gazyMTWlvecSD4vDqfk= +bitbucket.org/creachadair/shell v0.0.7/go.mod h1:oqtXSSvSYr4624lnnabXHaBsYW6RD80caLi2b3hJk0U= +github.com/bitfield/script v0.20.2 h1:4DexsRtBILVMEn3EZwHbtJdDqdk43sXI8gM3F04JXgs= +github.com/bitfield/script v0.20.2/go.mod h1:l3AZPVAtKQrL03bwh7nlNTUtgrgSWurpJSbtqspYrOA= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= +github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/_examples/actions/docker_action/kraken.yml b/_examples/actions/docker_action/kraken.yml new file mode 100644 index 0000000..f2c8dba --- /dev/null +++ b/_examples/actions/docker_action/kraken.yml @@ -0,0 +1,11 @@ +apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1 +name: write-a-readme +select: + repositories: + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + # providers: + # - gitea: https://git.front.kjuulh.io + # organisation: "cibus" +actions: + - type: docker-build + entry: Dockerfile diff --git a/cmd/kraken/commands/process.go b/cmd/kraken/commands/process.go index 342ebc4..1ffd16e 100644 --- a/cmd/kraken/commands/process.go +++ b/cmd/kraken/commands/process.go @@ -22,8 +22,8 @@ func CreateKrakenProcessCmd() *cobra.Command { Path string `json:"path"` }{ Repository: "git@git.front.kjuulh.io:kjuulh/kraken.git", - Branch: "feature/gitea-integration", - Path: "_examples/actions/write_a_readme/", + Branch: "feature/docker-action", + Path: "_examples/actions/docker_action/", }) if err != nil { panic(err) diff --git a/internal/actions/action.go b/internal/actions/action.go index 3c6416e..bc9fdb4 100644 --- a/internal/actions/action.go +++ b/internal/actions/action.go @@ -30,6 +30,18 @@ func (a *Action) Execute(ctx context.Context, area *storage.Area) error { zap.L().Debug("Execution done") + case "docker-build": + zap.L().Debug("Building docker-build") + runCmd, err := builders.NewDockerBuild(zap.L()).Build(ctx, a.SchemaPath, action.Entry) + if err != nil { + return err + } + err = runCmd(ctx, area.Path) + if err != nil { + return err + } + return nil + default: return errors.New("could not determine action type") } diff --git a/internal/actions/builders/docker.go b/internal/actions/builders/docker.go new file mode 100644 index 0000000..4120229 --- /dev/null +++ b/internal/actions/builders/docker.go @@ -0,0 +1,95 @@ +package builders + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + + "go.uber.org/zap" + "go.uber.org/zap/zapio" +) + +type DockerBuild struct { + logger *zap.Logger +} + +func NewDockerBuild(logger *zap.Logger) *DockerBuild { + return &DockerBuild{logger: logger} +} + +type DockerRunCommand func(ctx context.Context, victimPath string) error + +func (g *DockerBuild) Build(ctx context.Context, modulePath, entryPath string) (DockerRunCommand, error) { + g.logger.Debug("Building docker image", zap.String("actiondir", modulePath), zap.String("entry", entryPath)) + + if _, err := os.Stat(fmt.Sprintf("%s/%s", modulePath, entryPath)); os.IsNotExist(err) { + return nil, errors.New("could not find entry") + } + + b := make([]byte, 20) + _, err := rand.Reader.Read(b) + if err != nil { + return nil, err + } + tag := hex.EncodeToString(b) + buildDockerCmd := fmt.Sprintf("(cd %s; docker build -f %s --tag kraken/%s .)", modulePath, entryPath, tag) + g.logger.Debug("Running command", zap.String("command", buildDockerCmd)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + buildDockerCmd, + ) + + debugwriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer debugwriter.Close() + + cmd.Stdout = debugwriter + cmd.Stderr = debugwriter + err = cmd.Start() + if err != nil { + return nil, err + } + + err = cmd.Wait() + if err != nil { + return nil, err + } + + g.logger.Debug("Docker image built!") + + return func(ctx context.Context, victimPath string) error { + g.logger.Debug("Executing script", zap.String("victim", victimPath)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + fmt.Sprintf("docker run --rm -v %s/:/src/work/ kraken/%s", victimPath, tag), + ) + + runDockerWriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer runDockerWriter.Close() + + cmd.Stdout = runDockerWriter + cmd.Stderr = runDockerWriter + + err = cmd.Start() + if err != nil { + return err + } + + return cmd.Wait() + }, nil +} diff --git a/internal/commands/process_repos.go b/internal/commands/process_repos.go index ae8049d..f801240 100644 --- a/internal/commands/process_repos.go +++ b/internal/commands/process_repos.go @@ -168,6 +168,16 @@ func (pr *ProcessRepos) commit(ctx context.Context, area *storage.Area, repo *pr return fmt.Errorf("could not add file: %w", err) } + status, err := wt.Status() + if err != nil { + return err + } + + if status.IsClean() { + pr.logger.Info("Returning early, as no modifications are detected") + return nil + } + err = pr.git.Commit(ctx, repo) if err != nil { return fmt.Errorf("could not get diff: %w", err) @@ -175,15 +185,6 @@ func (pr *ProcessRepos) commit(ctx context.Context, area *storage.Area, repo *pr dryrun := false if !dryrun { - status, err := wt.Status() - if err != nil { - return err - } - - if status.IsClean() { - pr.logger.Info("Returning early, as no modifications are detected") - return nil - } err = pr.git.Push(ctx, repo) if err != nil { diff --git a/internal/gitproviders/gitea.go b/internal/gitproviders/gitea.go index 8fa3690..fcc277a 100644 --- a/internal/gitproviders/gitea.go +++ b/internal/gitproviders/gitea.go @@ -2,7 +2,9 @@ package gitproviders import ( "context" + "errors" "fmt" + "os" "sync" "code.gitea.io/sdk/gitea" @@ -121,7 +123,15 @@ func (g *Gitea) getOrCreateClient(ctx context.Context, server string) (*gitea.Cl client, ok := g.giteaClients[server] if !ok || client == nil { c, err := gitea.NewClient(server) - c.SetBasicAuth("kjuulh", "c0bd801cc9a7f2ed559ea45d603afc92f5443f19") + username, ok := os.LookupEnv("GITEA_USERNAME") + if !ok { + return nil, errors.New("missing environment variable GITEA_USERNAME") + } + apitoken, ok := os.LookupEnv("GITEA_API_TOKEN") + if !ok { + return nil, errors.New("missing environment variable GITEA_API_TOKEN") + } + c.SetBasicAuth(username, apitoken) if err != nil { return nil, err } diff --git a/internal/services/providers/git.go b/internal/services/providers/git.go index 07018b6..45ed04e 100644 --- a/internal/services/providers/git.go +++ b/internal/services/providers/git.go @@ -2,6 +2,7 @@ package providers import ( "context" + "errors" "fmt" "time" @@ -63,12 +64,12 @@ func NewGit(logger *zap.Logger, gitConfig *GitConfig, openPGP *signer.OpenPGP) * } func (g *Git) GetOriginHEADForRepo(ctx context.Context, gitRepo *GitRepo) (string, error) { - remote, err := gitRepo.repo.Remote("origin") + auth, err := g.GetAuth() if err != nil { return "", err } - auth, err := g.GetAuth() + remote, err := gitRepo.repo.Remote("origin") if err != nil { return "", err } @@ -82,11 +83,16 @@ func (g *Git) GetOriginHEADForRepo(ctx context.Context, gitRepo *GitRepo) (strin headRef := "" for _, ref := range refs { + //g.logger.Debug(ref.String()) if !ref.Name().IsBranch() { headRef = ref.Target().Short() } } + if headRef == "" { + return "", errors.New("no upstream HEAD branch could be found") + } + return headRef, nil } @@ -107,7 +113,7 @@ func (g *Git) CloneBranch(ctx context.Context, storageArea *storage.Area, repoUr Auth: auth, RemoteName: "origin", ReferenceName: plumbing.NewBranchReferenceName(branch), - SingleBranch: true, + SingleBranch: false, NoCheckout: false, Depth: 1, RecurseSubmodules: 1, @@ -118,7 +124,7 @@ func (g *Git) CloneBranch(ctx context.Context, storageArea *storage.Area, repoUr } repo, err := git.PlainCloneContext(ctx, storageArea.Path, false, &cloneOptions) - if err != nil { + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { return nil, err } @@ -144,7 +150,7 @@ func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl stri Auth: auth, RemoteName: "origin", ReferenceName: "refs/heads/main", - SingleBranch: true, + SingleBranch: false, NoCheckout: false, Depth: 1, RecurseSubmodules: 1, @@ -245,7 +251,7 @@ func (g *Git) CreateBranch(ctx context.Context, gitRepo *GitRepo) error { InsecureSkipTLS: false, CABundle: []byte{}, }) - if err != nil { + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { return fmt.Errorf("could not pull from origin: %w", err) } @@ -287,12 +293,11 @@ func (g *Git) Push(ctx context.Context, gitRepo *GitRepo) error { Auth: auth, Progress: g.getProgressWriter(), Prune: false, - Force: false, + Force: true, InsecureSkipTLS: false, CABundle: []byte{}, RequireRemoteRefs: []config.RefSpec{}, }) - if err != nil { return err } diff --git a/roadmap.md b/roadmap.md index ae23587..8c2a5af 100644 --- a/roadmap.md +++ b/roadmap.md @@ -20,7 +20,7 @@ - [x] Setup a way to choose actions and predicates - [x] Allow instantiation of actions, kraken template repo etc. -- [ ] Implement docker action +- [x] Implement docker action - [ ] Providing query results - [x] Create pr for gitea provider - [ ] Think about some sort of isolation diff --git a/scripts/run_server.sh b/scripts/run_server.sh index 6fb5602..59ec179 100755 --- a/scripts/run_server.sh +++ b/scripts/run_server.sh @@ -2,4 +2,6 @@ set -e +export $(cat .env | xargs) + go run cmd/server/server.go start -- 2.45.2 From fa8985a0e7b220330a4825cb3b3c8e4794099f8e Mon Sep 17 00:00:00 2001 From: kjuulh Date: Sun, 18 Sep 2022 15:16:35 +0200 Subject: [PATCH 13/15] with reordered roadmap --- roadmap.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/roadmap.md b/roadmap.md index 8c2a5af..5414be2 100644 --- a/roadmap.md +++ b/roadmap.md @@ -21,13 +21,11 @@ - [x] Setup a way to choose actions and predicates - [x] Allow instantiation of actions, kraken template repo etc. - [x] Implement docker action -- [ ] Providing query results - [x] Create pr for gitea provider -- [ ] Think about some sort of isolation +- [ ] Providing query results - [ ] Create CLI to trigger action -- [ ] Setup pool of runners -- [ ] Run authenticated on servers -- [ ] Create queuing system + +### Not in scope ## Version 1.0 @@ -35,3 +33,11 @@ - [ ] Make configurable ssh user - [ ] Make configurable gpg keyset - [ ] Make configurable git provider +- [ ] Create templating function + +## Version 1.x + +- Think about some sort of isolation +- Run authenticated on servers +- Create queuing system +- Setup pool of runners -- 2.45.2 From 3ef688a00d2a59fcf2fdf212a062035b26995880 Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Sun, 18 Sep 2022 16:11:22 +0200 Subject: [PATCH 14/15] feature/query-results (#12) Co-authored-by: kjuulh Reviewed-on: https://git.front.kjuulh.io/kjuulh/kraken/pulls/12 --- _examples/queries/scrape_readme/kraken.yml | 11 +++ cmd/kraken/commands/process.go | 4 +- internal/actions/action.go | 26 +++++ internal/actions/querier/ripgrep.go | 106 +++++++++++++++++++++ internal/commands/process_repos.go | 27 ++++-- internal/schema/kraken.go | 4 + roadmap.md | 3 +- 7 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 _examples/queries/scrape_readme/kraken.yml create mode 100644 internal/actions/querier/ripgrep.go diff --git a/_examples/queries/scrape_readme/kraken.yml b/_examples/queries/scrape_readme/kraken.yml new file mode 100644 index 0000000..35aa3a3 --- /dev/null +++ b/_examples/queries/scrape_readme/kraken.yml @@ -0,0 +1,11 @@ +apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1 +name: write-a-readme +select: + repositories: + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + # providers: + # - gitea: https://git.front.kjuulh.io + # organisation: "cibus" +queries: + - type: grep + query: "# README" diff --git a/cmd/kraken/commands/process.go b/cmd/kraken/commands/process.go index 1ffd16e..91bd822 100644 --- a/cmd/kraken/commands/process.go +++ b/cmd/kraken/commands/process.go @@ -22,8 +22,8 @@ func CreateKrakenProcessCmd() *cobra.Command { Path string `json:"path"` }{ Repository: "git@git.front.kjuulh.io:kjuulh/kraken.git", - Branch: "feature/docker-action", - Path: "_examples/actions/docker_action/", + Branch: "feature/query-results", + Path: "_examples/queries/scrape_readme/", }) if err != nil { panic(err) diff --git a/internal/actions/action.go b/internal/actions/action.go index bc9fdb4..b36155d 100644 --- a/internal/actions/action.go +++ b/internal/actions/action.go @@ -5,6 +5,7 @@ import ( "errors" "git.front.kjuulh.io/kjuulh/kraken/internal/actions/builders" + "git.front.kjuulh.io/kjuulh/kraken/internal/actions/querier" "git.front.kjuulh.io/kjuulh/kraken/internal/schema" "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" "go.uber.org/zap" @@ -49,3 +50,28 @@ func (a *Action) Execute(ctx context.Context, area *storage.Area) error { return nil } + +func (a *Action) Query(ctx context.Context, area *storage.Area) ([]string, bool, error) { + for _, query := range a.Schema.Queries { + switch query.Type { + case "grep": + exe, err := querier.NewRipGrep(zap.L()).Build(ctx, a.SchemaPath, query.Query) + if err != nil { + return nil, false, err + } + output, found, err := exe(ctx, area.Path) + if err != nil { + return nil, false, err + } + + zap.L().Debug("Execution done") + + return output, found, nil + + default: + return nil, false, errors.New("could not determine query type") + } + } + + return nil, false, nil +} diff --git a/internal/actions/querier/ripgrep.go b/internal/actions/querier/ripgrep.go new file mode 100644 index 0000000..99a415b --- /dev/null +++ b/internal/actions/querier/ripgrep.go @@ -0,0 +1,106 @@ +package querier + +import ( + "context" + "fmt" + "io" + "os/exec" + "strings" + + "go.uber.org/zap" + "go.uber.org/zap/zapio" +) + +type RipGrep struct { + logger *zap.Logger +} + +func NewRipGrep(logger *zap.Logger) *RipGrep { + return &RipGrep{logger: logger} +} + +type RipGrepCommand func(ctx context.Context, victimPath string) ([]string, bool, error) + +func (g *RipGrep) Build(ctx context.Context, modulePath, query string) (RipGrepCommand, error) { + g.logger.Debug("Pulling docker image", zap.String("actiondir", modulePath), zap.String("query", query)) + + pullDockerImage := "docker pull mbologna/docker-ripgrep" + g.logger.Debug("Running command", zap.String("command", pullDockerImage)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + pullDockerImage, + ) + + debugwriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer debugwriter.Close() + + cmd.Stdout = debugwriter + cmd.Stderr = debugwriter + err := cmd.Start() + if err != nil { + return nil, err + } + + err = cmd.Wait() + if err != nil { + return nil, err + } + + g.logger.Debug("Docker image pulled") + + return func(ctx context.Context, victimPath string) ([]string, bool, error) { + g.logger.Debug("Executing script", zap.String("victim", victimPath)) + + runRipGrepCmd := fmt.Sprintf("docker run --rm -v %s/:/data:ro mbologna/docker-ripgrep rg -i '%s' || true", victimPath, query) + + g.logger.Debug("Execute ripgrep query", zap.String("command", runRipGrepCmd)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + runRipGrepCmd, + ) + + runDockerWriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer runDockerWriter.Close() + + builder := &strings.Builder{} + combinedWriter := io.MultiWriter(runDockerWriter, builder) + + cmd.Stdout = combinedWriter + cmd.Stderr = combinedWriter + + err = cmd.Start() + if err != nil { + return nil, false, err + } + + err = cmd.Wait() + if err != nil { + return nil, false, err + } + + contents := strings.Split(builder.String(), "\n") + validatedOutput := make([]string, 0) + + for _, c := range contents { + if !strings.Contains(c, "WARNING:") { + validatedOutput = append(validatedOutput, c) + } + } + + found := len(validatedOutput) > 0 + + return validatedOutput, found, nil + }, nil +} diff --git a/internal/commands/process_repos.go b/internal/commands/process_repos.go index f801240..02c7c27 100644 --- a/internal/commands/process_repos.go +++ b/internal/commands/process_repos.go @@ -112,17 +112,32 @@ func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, action return err } - err = action.Execute(ctx, area) - if err != nil { - return err + if len(action.Schema.Queries) > 0 { + result, found, err := action.Query(ctx, area) + if err != nil { + return err + } + + if found { + pr.logger.Info("Query result", zap.Strings("result", result)) + // TODO: Append to real result, and return together + } } - err = pr.commit(ctx, area, repo, repoUrl) - if err != nil { - return err + if len(action.Schema.Actions) > 0 { + err = action.Execute(ctx, area) + if err != nil { + return err + } + + err = pr.commit(ctx, area, repo, repoUrl) + if err != nil { + return err + } } pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) + return nil } diff --git a/internal/schema/kraken.go b/internal/schema/kraken.go index fb7d75e..b1c76f5 100644 --- a/internal/schema/kraken.go +++ b/internal/schema/kraken.go @@ -16,6 +16,10 @@ type KrakenSchema struct { Type string `yaml:"type"` Entry string `yaml:"entry"` } `yaml:"actions"` + Queries []struct { + Type string `yaml:"type"` + Query string `yaml:"query"` + } `yaml:"queries"` } func Unmarshal(raw string) (*KrakenSchema, error) { diff --git a/roadmap.md b/roadmap.md index 5414be2..cff2a5c 100644 --- a/roadmap.md +++ b/roadmap.md @@ -22,7 +22,7 @@ - [x] Allow instantiation of actions, kraken template repo etc. - [x] Implement docker action - [x] Create pr for gitea provider -- [ ] Providing query results +- [x] Providing query results - [ ] Create CLI to trigger action ### Not in scope @@ -34,6 +34,7 @@ - [ ] Make configurable gpg keyset - [ ] Make configurable git provider - [ ] Create templating function +- [ ] Add way to see progress of runners ## Version 1.x -- 2.45.2 From 6691b7f1ec5f9fbafc065ed1d468e01c666449b8 Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Sun, 18 Sep 2022 16:49:15 +0200 Subject: [PATCH 15/15] feature/add-cmd (#13) Co-authored-by: kjuulh Reviewed-on: https://git.front.kjuulh.io/kjuulh/kraken/pulls/13 --- cmd/kraken/commands/process.go | 28 ++++++++++++++++++++++++---- internal/services/providers/git.go | 2 +- roadmap.md | 2 ++ scripts/run_client.sh | 5 ++++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/cmd/kraken/commands/process.go b/cmd/kraken/commands/process.go index 91bd822..89434a9 100644 --- a/cmd/kraken/commands/process.go +++ b/cmd/kraken/commands/process.go @@ -9,9 +9,19 @@ import ( ) func CreateKrakenProcessCmd() *cobra.Command { + + var ( + actionsRepo string + branch string + path string + ) cmd := &cobra.Command{ Use: "process", - Run: func(cmd *cobra.Command, _ []string) { + RunE: func(cmd *cobra.Command, args []string) error { + if err := cmd.ParseFlags(args); err != nil { + return err + } + client := http.Client{} var buf bytes.Buffer @@ -21,9 +31,9 @@ func CreateKrakenProcessCmd() *cobra.Command { Branch string `json:"branch"` Path string `json:"path"` }{ - Repository: "git@git.front.kjuulh.io:kjuulh/kraken.git", - Branch: "feature/query-results", - Path: "_examples/queries/scrape_readme/", + Repository: actionsRepo, + Branch: branch, + Path: path, }) if err != nil { panic(err) @@ -46,8 +56,18 @@ func CreateKrakenProcessCmd() *cobra.Command { if resp.StatusCode >= 300 { panic(resp.Status) } + + return nil }, } + pf := cmd.PersistentFlags() + + pf.StringVar(&actionsRepo, "actions-repo", "", "actions repo is the location of your actions, not where to apply the actions themselves, that should be self contained") + cmd.MarkPersistentFlagRequired("actions-repo") + pf.StringVar(&branch, "branch", "main", "which branch to look for actions in, will default to main") + pf.StringVar(&path, "path", "", "the location of the path inside the repository") + cmd.MarkPersistentFlagRequired("path") + return cmd } diff --git a/internal/services/providers/git.go b/internal/services/providers/git.go index 45ed04e..32cc1a7 100644 --- a/internal/services/providers/git.go +++ b/internal/services/providers/git.go @@ -242,7 +242,7 @@ func (g *Git) CreateBranch(ctx context.Context, gitRepo *GitRepo) error { err = worktree.PullContext(ctx, &git.PullOptions{ RemoteName: "origin", ReferenceName: "refs/heads/main", - SingleBranch: true, + SingleBranch: false, Depth: 1, Auth: auth, RecurseSubmodules: 1, diff --git a/roadmap.md b/roadmap.md index cff2a5c..34cf540 100644 --- a/roadmap.md +++ b/roadmap.md @@ -35,6 +35,8 @@ - [ ] Make configurable git provider - [ ] Create templating function - [ ] Add way to see progress of runners +- [ ] Implement global .kraken store for easy access +- [ ] Move builders to start instead of every time ## Version 1.x diff --git a/scripts/run_client.sh b/scripts/run_client.sh index cfe216e..237c23d 100755 --- a/scripts/run_client.sh +++ b/scripts/run_client.sh @@ -2,4 +2,7 @@ set -e -go run cmd/kraken/kraken.go process +current_branch=$(git branch --show-current) + +go run cmd/kraken/kraken.go process --actions-repo "git@git.front.kjuulh.io:kjuulh/kraken.git" --branch "$current_branch" --path "_examples/actions/write_a_readme" +go run cmd/kraken/kraken.go process --actions-repo "git@git.front.kjuulh.io:kjuulh/kraken.git" --branch "$current_branch" --path "_examples/queries/scrape_readme" -- 2.45.2