From 1f46f6ac8d37241f578634ef5a2cdfa461d69115 Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Sun, 18 Sep 2022 16:49:34 +0200 Subject: [PATCH] v0.1 (#8) Co-authored-by: kjuulh Reviewed-on: https://git.front.kjuulh.io/kjuulh/kraken/pulls/8 --- .gitignore | 1 + _examples/actions/docker_action/Dockerfile | 8 + _examples/actions/docker_action/entry.sh | 5 + _examples/actions/docker_action/go.mod | 11 + _examples/actions/docker_action/go.sum | 20 ++ _examples/actions/docker_action/kraken.yml | 11 + _examples/actions/write_a_readme/go.mod | 11 + _examples/actions/write_a_readme/go.sum | 20 ++ _examples/actions/write_a_readme/kraken.yml | 11 + _examples/actions/write_a_readme/main.go | 12 + _examples/queries/scrape_readme/kraken.yml | 11 + cmd/kraken/commands/process.go | 31 +- cmd/server/server.go | 2 + cuddle.yaml | 6 + go.mod | 3 + go.sum | 12 + internal/actions/action.go | 77 +++++ internal/actions/action_creator.go | 85 ++++++ internal/actions/builders/docker.go | 95 ++++++ internal/actions/builders/go.go | 46 +++ internal/actions/querier/ripgrep.go | 106 +++++++ internal/api/process_command.go | 13 +- internal/commands/process_repos.go | 302 +++++++++++++------- internal/gitproviders/gitea.go | 143 +++++++++ internal/schema/kraken.go | 32 +++ internal/serverdeps/server_deps.go | 10 + internal/services/providers/git.go | 89 +++++- roadmap.md | 37 ++- scripts/run.sh | 15 + scripts/run_client.sh | 8 + scripts/run_server.sh | 7 + 31 files changed, 1111 insertions(+), 129 deletions(-) create mode 100644 _examples/actions/docker_action/Dockerfile create mode 100755 _examples/actions/docker_action/entry.sh create mode 100644 _examples/actions/docker_action/go.mod create mode 100644 _examples/actions/docker_action/go.sum create mode 100644 _examples/actions/docker_action/kraken.yml create mode 100644 _examples/actions/write_a_readme/go.mod create mode 100644 _examples/actions/write_a_readme/go.sum create mode 100644 _examples/actions/write_a_readme/kraken.yml create mode 100644 _examples/actions/write_a_readme/main.go create mode 100644 _examples/queries/scrape_readme/kraken.yml create mode 100644 internal/actions/action.go create mode 100644 internal/actions/action_creator.go create mode 100644 internal/actions/builders/docker.go create mode 100644 internal/actions/builders/go.go create mode 100644 internal/actions/querier/ripgrep.go create mode 100644 internal/gitproviders/gitea.go create mode 100644 internal/schema/kraken.go create mode 100755 scripts/run.sh create mode 100755 scripts/run_client.sh create mode 100755 scripts/run_server.sh diff --git a/.gitignore b/.gitignore index 75d1871..dddecf9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .cuddle/ +.env diff --git a/_examples/actions/docker_action/Dockerfile b/_examples/actions/docker_action/Dockerfile new file mode 100644 index 0000000..572de73 --- /dev/null +++ b/_examples/actions/docker_action/Dockerfile @@ -0,0 +1,8 @@ +FROM debian:bullseye-slim + +# Kraken relies on this path being the specified path +WORKDIR /src/work/ + +COPY entry.sh /src/script.sh + +CMD [ "/src/script.sh" ] diff --git a/_examples/actions/docker_action/entry.sh b/_examples/actions/docker_action/entry.sh new file mode 100755 index 0000000..11771f4 --- /dev/null +++ b/_examples/actions/docker_action/entry.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +echo "# README docker" > README.md diff --git a/_examples/actions/docker_action/go.mod b/_examples/actions/docker_action/go.mod new file mode 100644 index 0000000..248ba10 --- /dev/null +++ b/_examples/actions/docker_action/go.mod @@ -0,0 +1,11 @@ +module write_a_readme + +go 1.19 + +require github.com/bitfield/script v0.20.2 + +require ( + bitbucket.org/creachadair/shell v0.0.7 // indirect + github.com/itchyny/gojq v0.12.7 // indirect + github.com/itchyny/timefmt-go v0.1.3 // indirect +) diff --git a/_examples/actions/docker_action/go.sum b/_examples/actions/docker_action/go.sum new file mode 100644 index 0000000..234eb1e --- /dev/null +++ b/_examples/actions/docker_action/go.sum @@ -0,0 +1,20 @@ +bitbucket.org/creachadair/shell v0.0.7 h1:Z96pB6DkSb7F3Y3BBnJeOZH2gazyMTWlvecSD4vDqfk= +bitbucket.org/creachadair/shell v0.0.7/go.mod h1:oqtXSSvSYr4624lnnabXHaBsYW6RD80caLi2b3hJk0U= +github.com/bitfield/script v0.20.2 h1:4DexsRtBILVMEn3EZwHbtJdDqdk43sXI8gM3F04JXgs= +github.com/bitfield/script v0.20.2/go.mod h1:l3AZPVAtKQrL03bwh7nlNTUtgrgSWurpJSbtqspYrOA= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= +github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/_examples/actions/docker_action/kraken.yml b/_examples/actions/docker_action/kraken.yml new file mode 100644 index 0000000..f2c8dba --- /dev/null +++ b/_examples/actions/docker_action/kraken.yml @@ -0,0 +1,11 @@ +apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1 +name: write-a-readme +select: + repositories: + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + # providers: + # - gitea: https://git.front.kjuulh.io + # organisation: "cibus" +actions: + - type: docker-build + entry: Dockerfile diff --git a/_examples/actions/write_a_readme/go.mod b/_examples/actions/write_a_readme/go.mod new file mode 100644 index 0000000..248ba10 --- /dev/null +++ b/_examples/actions/write_a_readme/go.mod @@ -0,0 +1,11 @@ +module write_a_readme + +go 1.19 + +require github.com/bitfield/script v0.20.2 + +require ( + bitbucket.org/creachadair/shell v0.0.7 // indirect + github.com/itchyny/gojq v0.12.7 // indirect + github.com/itchyny/timefmt-go v0.1.3 // indirect +) diff --git a/_examples/actions/write_a_readme/go.sum b/_examples/actions/write_a_readme/go.sum new file mode 100644 index 0000000..234eb1e --- /dev/null +++ b/_examples/actions/write_a_readme/go.sum @@ -0,0 +1,20 @@ +bitbucket.org/creachadair/shell v0.0.7 h1:Z96pB6DkSb7F3Y3BBnJeOZH2gazyMTWlvecSD4vDqfk= +bitbucket.org/creachadair/shell v0.0.7/go.mod h1:oqtXSSvSYr4624lnnabXHaBsYW6RD80caLi2b3hJk0U= +github.com/bitfield/script v0.20.2 h1:4DexsRtBILVMEn3EZwHbtJdDqdk43sXI8gM3F04JXgs= +github.com/bitfield/script v0.20.2/go.mod h1:l3AZPVAtKQrL03bwh7nlNTUtgrgSWurpJSbtqspYrOA= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= +github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/_examples/actions/write_a_readme/kraken.yml b/_examples/actions/write_a_readme/kraken.yml new file mode 100644 index 0000000..0e5c44e --- /dev/null +++ b/_examples/actions/write_a_readme/kraken.yml @@ -0,0 +1,11 @@ +apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1 +name: write-a-readme +select: + repositories: + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + # providers: + # - gitea: https://git.front.kjuulh.io + # organisation: "cibus" +actions: + - type: go + entry: "main.go" diff --git a/_examples/actions/write_a_readme/main.go b/_examples/actions/write_a_readme/main.go new file mode 100644 index 0000000..5c24133 --- /dev/null +++ b/_examples/actions/write_a_readme/main.go @@ -0,0 +1,12 @@ +package main + +import "github.com/bitfield/script" + +func main() { + _, err := script. + Echo("# Readme"). + WriteFile("README.md") + if err != nil { + panic(err) + } +} diff --git a/_examples/queries/scrape_readme/kraken.yml b/_examples/queries/scrape_readme/kraken.yml new file mode 100644 index 0000000..35aa3a3 --- /dev/null +++ b/_examples/queries/scrape_readme/kraken.yml @@ -0,0 +1,11 @@ +apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1 +name: write-a-readme +select: + repositories: + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + # providers: + # - gitea: https://git.front.kjuulh.io + # organisation: "cibus" +queries: + - type: grep + query: "# README" diff --git a/cmd/kraken/commands/process.go b/cmd/kraken/commands/process.go index 1097089..89434a9 100644 --- a/cmd/kraken/commands/process.go +++ b/cmd/kraken/commands/process.go @@ -9,17 +9,32 @@ import ( ) func CreateKrakenProcessCmd() *cobra.Command { + + var ( + actionsRepo string + branch string + path string + ) cmd := &cobra.Command{ Use: "process", - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { + if err := cmd.ParseFlags(args); err != nil { + return err + } + client := http.Client{} var buf bytes.Buffer err := json.NewEncoder(&buf). Encode(struct { - RepositoryUrls []string `json:"repositoryUrls"` + Repository string `json:"repository"` + Branch string `json:"branch"` + Path string `json:"path"` }{ - RepositoryUrls: []string{"git@git.front.kjuulh.io:kjuulh/kraken.git"}}) + Repository: actionsRepo, + Branch: branch, + Path: path, + }) if err != nil { panic(err) } @@ -41,8 +56,18 @@ func CreateKrakenProcessCmd() *cobra.Command { if resp.StatusCode >= 300 { panic(resp.Status) } + + return nil }, } + pf := cmd.PersistentFlags() + + pf.StringVar(&actionsRepo, "actions-repo", "", "actions repo is the location of your actions, not where to apply the actions themselves, that should be self contained") + cmd.MarkPersistentFlagRequired("actions-repo") + pf.StringVar(&branch, "branch", "main", "which branch to look for actions in, will default to main") + pf.StringVar(&path, "path", "", "the location of the path inside the repository") + cmd.MarkPersistentFlagRequired("path") + return cmd } diff --git a/cmd/server/server.go b/cmd/server/server.go index 49b713c..5c835ec 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -15,6 +15,8 @@ func main() { } _ = logger.Sync() + zap.ReplaceGlobals(logger) + Execute(logger) } diff --git a/cuddle.yaml b/cuddle.yaml index 8b285a2..87fafac 100644 --- a/cuddle.yaml +++ b/cuddle.yaml @@ -9,3 +9,9 @@ vars: scripts: push_github: type: shell + run_server: + type: shell + run_client: + type: shell + run: + type: shell diff --git a/go.mod b/go.mod index cd1b597..2fab6ad 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( ) require ( + code.gitea.io/sdk/gitea v0.15.1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/ProtonMail/go-mime v0.0.0-20220302105931-303f85f7fe0f // indirect github.com/acomagu/bufpipe v1.0.3 // indirect @@ -30,6 +31,7 @@ require ( github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-playground/validator/v10 v10.10.0 // indirect github.com/goccy/go-json v0.9.7 // indirect + github.com/hashicorp/go-version v1.2.1 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -47,6 +49,7 @@ require ( github.com/sirupsen/logrus v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ugorji/go/codec v1.2.7 // indirect + github.com/whilp/git-urls v1.0.0 // indirect github.com/xanzy/ssh-agent v0.3.2 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect diff --git a/go.sum b/go.sum index 03a8012..5f09503 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,6 @@ +code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE= +code.gitea.io/sdk/gitea v0.15.1 h1:WJreC7YYuxbn0UDaPuWIe/mtiNKTvLN8MLkaw71yx/M= +code.gitea.io/sdk/gitea v0.15.1/go.mod h1:klY2LVI3s3NChzIk/MzMn7G1FHrfU7qd63iSMVoHRBA= git.front.kjuulh.io/kjuulh/curre v1.2.2 h1:0OwWIfekrMykdQg9bdmG80I+Mjc2k4i+sy903phuDWs= git.front.kjuulh.io/kjuulh/curre v1.2.2/go.mod h1:m7WpSehONLqPh/XF3F0BI0UOpLOfGuDmDEFI1XsM6fE= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -71,6 +74,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -148,9 +153,12 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= +github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xanzy/ssh-agent v0.3.2 h1:eKj4SX2Fe7mui28ZgnFW5fmTz1EIr7ugo5s6wDxdHBM= github.com/xanzy/ssh-agent v0.3.2/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= @@ -184,10 +192,12 @@ golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -195,6 +205,7 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -233,6 +244,7 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/actions/action.go b/internal/actions/action.go new file mode 100644 index 0000000..b36155d --- /dev/null +++ b/internal/actions/action.go @@ -0,0 +1,77 @@ +package actions + +import ( + "context" + "errors" + + "git.front.kjuulh.io/kjuulh/kraken/internal/actions/builders" + "git.front.kjuulh.io/kjuulh/kraken/internal/actions/querier" + "git.front.kjuulh.io/kjuulh/kraken/internal/schema" + "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" + "go.uber.org/zap" +) + +type Action struct { + Schema *schema.KrakenSchema + SchemaPath string +} + +func (a *Action) Execute(ctx context.Context, area *storage.Area) error { + for _, action := range a.Schema.Actions { + switch action.Type { + case "go": + exe, err := builders.NewGo(zap.L()).Build(ctx, a.SchemaPath, action.Entry) + if err != nil { + return err + } + err = exe(ctx, area.Path) + if err != nil { + return err + } + + zap.L().Debug("Execution done") + + case "docker-build": + zap.L().Debug("Building docker-build") + runCmd, err := builders.NewDockerBuild(zap.L()).Build(ctx, a.SchemaPath, action.Entry) + if err != nil { + return err + } + err = runCmd(ctx, area.Path) + if err != nil { + return err + } + return nil + + default: + return errors.New("could not determine action type") + } + } + + return nil +} + +func (a *Action) Query(ctx context.Context, area *storage.Area) ([]string, bool, error) { + for _, query := range a.Schema.Queries { + switch query.Type { + case "grep": + exe, err := querier.NewRipGrep(zap.L()).Build(ctx, a.SchemaPath, query.Query) + if err != nil { + return nil, false, err + } + output, found, err := exe(ctx, area.Path) + if err != nil { + return nil, false, err + } + + zap.L().Debug("Execution done") + + return output, found, nil + + default: + return nil, false, errors.New("could not determine query type") + } + } + + return nil, false, nil +} diff --git a/internal/actions/action_creator.go b/internal/actions/action_creator.go new file mode 100644 index 0000000..b731519 --- /dev/null +++ b/internal/actions/action_creator.go @@ -0,0 +1,85 @@ +package actions + +import ( + "context" + "fmt" + "os" + "path" + "time" + + "git.front.kjuulh.io/kjuulh/kraken/internal/schema" + "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" + "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" + "go.uber.org/zap" +) + +type ( + ActionCreatorOps struct { + RepositoryUrl string + Branch string + Path string + } + + ActionCreator struct { + logger *zap.Logger + storage *storage.Service + git *providers.Git + } + + ActionCreatorDeps interface { + GetStorageService() *storage.Service + GetGitProvider() *providers.Git + } +) + +func NewActionCreator(logger *zap.Logger, deps ActionCreatorDeps) *ActionCreator { + return &ActionCreator{ + logger: logger, + storage: deps.GetStorageService(), + git: deps.GetGitProvider(), + } +} + +func (ac *ActionCreator) Prepare(ctx context.Context, ops *ActionCreatorOps) (*Action, error) { + area, err := ac.storage.CreateArea(ctx) + if err != nil { + ac.logger.Error("failed to allocate area", zap.Error(err)) + return nil, err + } + + cloneCtx, _ := context.WithTimeout(ctx, time.Second*10) + _, err = ac.git.CloneBranch(cloneCtx, area, ops.RepositoryUrl, ops.Branch) + if err != nil { + ac.logger.Error("could not clone repo", zap.Error(err)) + return nil, err + } + + executorUrl := path.Join(area.Path, ops.Path) + if _, err = os.Stat(executorUrl); os.IsNotExist(err) { + return nil, fmt.Errorf("path is invalid: %s", ops.Path) + } + + contents, err := os.ReadFile(path.Join(executorUrl, "kraken.yml")) + if err != nil { + return nil, err + } + + krakenSchema, err := schema.Unmarshal(string(contents)) + if err != nil { + return nil, err + } + + ac.logger.Debug("Action creator done") + return &Action{ + Schema: krakenSchema, + SchemaPath: executorUrl, + }, nil +} + +func (ac *ActionCreator) Cleanup(ctx context.Context, area *storage.Area) { + ac.logger.Debug("Removing area", zap.String("path", area.Path)) + err := ac.storage.RemoveArea(ctx, area) + if err != nil { + panic(err) + } +} diff --git a/internal/actions/builders/docker.go b/internal/actions/builders/docker.go new file mode 100644 index 0000000..4120229 --- /dev/null +++ b/internal/actions/builders/docker.go @@ -0,0 +1,95 @@ +package builders + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + + "go.uber.org/zap" + "go.uber.org/zap/zapio" +) + +type DockerBuild struct { + logger *zap.Logger +} + +func NewDockerBuild(logger *zap.Logger) *DockerBuild { + return &DockerBuild{logger: logger} +} + +type DockerRunCommand func(ctx context.Context, victimPath string) error + +func (g *DockerBuild) Build(ctx context.Context, modulePath, entryPath string) (DockerRunCommand, error) { + g.logger.Debug("Building docker image", zap.String("actiondir", modulePath), zap.String("entry", entryPath)) + + if _, err := os.Stat(fmt.Sprintf("%s/%s", modulePath, entryPath)); os.IsNotExist(err) { + return nil, errors.New("could not find entry") + } + + b := make([]byte, 20) + _, err := rand.Reader.Read(b) + if err != nil { + return nil, err + } + tag := hex.EncodeToString(b) + buildDockerCmd := fmt.Sprintf("(cd %s; docker build -f %s --tag kraken/%s .)", modulePath, entryPath, tag) + g.logger.Debug("Running command", zap.String("command", buildDockerCmd)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + buildDockerCmd, + ) + + debugwriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer debugwriter.Close() + + cmd.Stdout = debugwriter + cmd.Stderr = debugwriter + err = cmd.Start() + if err != nil { + return nil, err + } + + err = cmd.Wait() + if err != nil { + return nil, err + } + + g.logger.Debug("Docker image built!") + + return func(ctx context.Context, victimPath string) error { + g.logger.Debug("Executing script", zap.String("victim", victimPath)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + fmt.Sprintf("docker run --rm -v %s/:/src/work/ kraken/%s", victimPath, tag), + ) + + runDockerWriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer runDockerWriter.Close() + + cmd.Stdout = runDockerWriter + cmd.Stderr = runDockerWriter + + err = cmd.Start() + if err != nil { + return err + } + + return cmd.Wait() + }, nil +} diff --git a/internal/actions/builders/go.go b/internal/actions/builders/go.go new file mode 100644 index 0000000..6d828ee --- /dev/null +++ b/internal/actions/builders/go.go @@ -0,0 +1,46 @@ +package builders + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + + "go.uber.org/zap" +) + +type Go struct { + logger *zap.Logger +} + +func NewGo(logger *zap.Logger) *Go { + return &Go{logger: logger} +} + +type GoExecutable func(ctx context.Context, victimPath string) error + +func (g *Go) Build(ctx context.Context, modulePath, entryPath string) (GoExecutable, error) { + g.logger.Debug("Building go binary", zap.String("actiondir", modulePath), zap.String("entry", entryPath)) + + if _, err := os.Stat(fmt.Sprintf("%s/%s", modulePath, entryPath)); os.IsNotExist(err) { + return nil, errors.New("could not find entry") + } + + err := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + fmt.Sprintf("(cd %s; go build -o main %s)", modulePath, entryPath), + ).Run() + if err != nil { + return nil, err + } + + g.logger.Debug("Go binary built!") + + return func(ctx context.Context, victimPath string) error { + g.logger.Debug("Executing script", zap.String("victim", victimPath)) + return exec.CommandContext(ctx, "/bin/bash", "-c", fmt.Sprintf("(cd %s; %s/main)", victimPath, modulePath)).Run() + }, nil +} diff --git a/internal/actions/querier/ripgrep.go b/internal/actions/querier/ripgrep.go new file mode 100644 index 0000000..99a415b --- /dev/null +++ b/internal/actions/querier/ripgrep.go @@ -0,0 +1,106 @@ +package querier + +import ( + "context" + "fmt" + "io" + "os/exec" + "strings" + + "go.uber.org/zap" + "go.uber.org/zap/zapio" +) + +type RipGrep struct { + logger *zap.Logger +} + +func NewRipGrep(logger *zap.Logger) *RipGrep { + return &RipGrep{logger: logger} +} + +type RipGrepCommand func(ctx context.Context, victimPath string) ([]string, bool, error) + +func (g *RipGrep) Build(ctx context.Context, modulePath, query string) (RipGrepCommand, error) { + g.logger.Debug("Pulling docker image", zap.String("actiondir", modulePath), zap.String("query", query)) + + pullDockerImage := "docker pull mbologna/docker-ripgrep" + g.logger.Debug("Running command", zap.String("command", pullDockerImage)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + pullDockerImage, + ) + + debugwriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer debugwriter.Close() + + cmd.Stdout = debugwriter + cmd.Stderr = debugwriter + err := cmd.Start() + if err != nil { + return nil, err + } + + err = cmd.Wait() + if err != nil { + return nil, err + } + + g.logger.Debug("Docker image pulled") + + return func(ctx context.Context, victimPath string) ([]string, bool, error) { + g.logger.Debug("Executing script", zap.String("victim", victimPath)) + + runRipGrepCmd := fmt.Sprintf("docker run --rm -v %s/:/data:ro mbologna/docker-ripgrep rg -i '%s' || true", victimPath, query) + + g.logger.Debug("Execute ripgrep query", zap.String("command", runRipGrepCmd)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + runRipGrepCmd, + ) + + runDockerWriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer runDockerWriter.Close() + + builder := &strings.Builder{} + combinedWriter := io.MultiWriter(runDockerWriter, builder) + + cmd.Stdout = combinedWriter + cmd.Stderr = combinedWriter + + err = cmd.Start() + if err != nil { + return nil, false, err + } + + err = cmd.Wait() + if err != nil { + return nil, false, err + } + + contents := strings.Split(builder.String(), "\n") + validatedOutput := make([]string, 0) + + for _, c := range contents { + if !strings.Contains(c, "WARNING:") { + validatedOutput = append(validatedOutput, c) + } + } + + found := len(validatedOutput) > 0 + + return validatedOutput, found, nil + }, nil +} diff --git a/internal/api/process_command.go b/internal/api/process_command.go index 589dd81..2e852b5 100644 --- a/internal/api/process_command.go +++ b/internal/api/process_command.go @@ -16,7 +16,9 @@ func CommandRoute(logger *zap.Logger, app *gin.Engine, deps *serverdeps.ServerDe commandRoute := app.Group("commands") commandRoute.POST("processRepos", func(c *gin.Context) { type processReposRequest struct { - RepositoryUrls []string `json:"repositoryUrls"` + Repository string `json:"repository"` + Branch string `json:"branch"` + Path string `json:"path"` } var request processReposRequest err := c.BindJSON(&request) @@ -28,11 +30,14 @@ func CommandRoute(logger *zap.Logger, app *gin.Engine, deps *serverdeps.ServerDe jobId := uuid.New().String() - go func(repositoryUrls []string, jobId string) { + go func(repository string, branch string, path string, jobId string) { ctx := context.WithValue(context.Background(), jobs.JobId{}, jobId) processRepos := commands.NewProcessRepos(logger, deps) - err = processRepos.Process(ctx, repositoryUrls) - }(request.RepositoryUrls, jobId) + err = processRepos.Process(ctx, repository, branch, path) + if err != nil { + logger.Error("could not process repo", zap.Error(err)) + } + }(request.Repository, request.Branch, request.Path, jobId) c.Status(http.StatusAccepted) }) diff --git a/internal/commands/process_repos.go b/internal/commands/process_repos.go index 5ea781c..02c7c27 100644 --- a/internal/commands/process_repos.go +++ b/internal/commands/process_repos.go @@ -3,150 +3,238 @@ package commands import ( "context" "fmt" - "io/fs" - "os" - "path" - "path/filepath" + "strings" "sync" "time" - "git.front.kjuulh.io/kjuulh/kraken/internal/services/actions" + "git.front.kjuulh.io/kjuulh/kraken/internal/actions" + "git.front.kjuulh.io/kjuulh/kraken/internal/gitproviders" + "git.front.kjuulh.io/kjuulh/kraken/internal/schema" "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" + giturls "github.com/whilp/git-urls" "go.uber.org/zap" ) type ( ProcessRepos struct { - logger *zap.Logger - storage *storage.Service - git *providers.Git - action *actions.Action + logger *zap.Logger + storage *storage.Service + git *providers.Git + actionCreator *actions.ActionCreator + gitea *gitproviders.Gitea } ProcessReposDeps interface { GetStorageService() *storage.Service GetGitProvider() *providers.Git - GetAction() *actions.Action + GetActionCreator() *actions.ActionCreator + GetGitea() *gitproviders.Gitea } ) func NewProcessRepos(logger *zap.Logger, deps ProcessReposDeps) *ProcessRepos { return &ProcessRepos{ - logger: logger, - storage: deps.GetStorageService(), - git: deps.GetGitProvider(), - action: deps.GetAction(), + logger: logger, + storage: deps.GetStorageService(), + git: deps.GetGitProvider(), + actionCreator: deps.GetActionCreator(), + gitea: deps.GetGitea(), } } -func (pr *ProcessRepos) Process(ctx context.Context, repositoryUrls []string) error { - // Clone repos +func (pr *ProcessRepos) Process(ctx context.Context, repository string, branch string, actionPath string) error { + action, err := pr.actionCreator.Prepare(ctx, &actions.ActionCreatorOps{ + RepositoryUrl: repository, + Branch: branch, + Path: actionPath, + }) + if err != nil { + return err + } + + repositoryUrls, err := pr.getRepoUrls(ctx, action.Schema) + if err != nil { + return err + } + wg := sync.WaitGroup{} wg.Add(len(repositoryUrls)) - errChan := make(chan error, 1) for _, repoUrl := range repositoryUrls { go func(ctx context.Context, repoUrl string) { defer func() { wg.Done() }() - pr.logger.Debug("Creating area", zap.String("repoUrl", repoUrl)) - area, err := pr.storage.CreateArea(ctx) + err := pr.processRepo(ctx, repoUrl, action) if err != nil { - pr.logger.Error("failed to allocate area", zap.Error(err)) - errChan <- err - return + pr.logger.Error("could not process repo", zap.Error(err)) } - - defer func(ctx context.Context) { - pr.logger.Debug("Removing area", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) - err = pr.storage.RemoveArea(ctx, area) - if err != nil { - errChan <- err - return - } - - }(ctx) - - pr.logger.Debug("Cloning repo", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) - cloneCtx, _ := context.WithTimeout(ctx, time.Second*5) - repo, err := pr.git.Clone(cloneCtx, area, repoUrl) - if err != nil { - pr.logger.Error("could not clone repo", zap.Error(err)) - errChan <- err - return - } - - err = pr.git.CreateBranch(ctx, repo) - if err != nil { - pr.logger.Error("could not create branch", zap.Error(err)) - errChan <- err - return - } - - err = pr.action.Run( - ctx, - area, - func(_ context.Context, area *storage.Area) (bool, error) { - pr.logger.Debug("checking predicate", zap.String("area", area.Path)) - contains := false - filepath.WalkDir(area.Path, func(path string, d fs.DirEntry, err error) error { - if d.Name() == "roadmap.md" { - contains = true - } - return nil - }) - return contains, nil - }, - func(_ context.Context, area *storage.Area) error { - pr.logger.Debug("running action", zap.String("area", area.Path)) - readme := path.Join(area.Path, "README.md") - file, err := os.Create(readme) - if err != nil { - return fmt.Errorf("could not create readme: %w", err) - } - _, err = file.WriteString("# Readme") - if err != nil { - return fmt.Errorf("could not write readme: %w", err) - } - - _, err = pr.git.Add(ctx, area, repo) - if err != nil { - return fmt.Errorf("could not add file: %w", err) - } - - err = pr.git.Commit(ctx, repo) - if err != nil { - return fmt.Errorf("could not get diff: %w", err) - } - - return nil - }, false) - if err != nil { - pr.logger.Error("could not run action", zap.Error(err)) - errChan <- err - return - } - - err = pr.git.Push(ctx, repo) - if err != nil { - pr.logger.Error("could not push to repo", zap.Error(err)) - errChan <- err - return - } - - pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) }(ctx, repoUrl) } wg.Wait() - close(errChan) - pr.logger.Debug("finished processing all repos") - - for err := range errChan { - return err - } + pr.logger.Debug("finished processing all repos", zap.Strings("repos", repositoryUrls)) return nil } + +func (pr *ProcessRepos) getRepoUrls(ctx context.Context, schema *schema.KrakenSchema) ([]string, error) { + repoUrls := make([]string, 0) + + repoUrls = append(repoUrls, schema.Select.Repositories...) + + for _, provider := range schema.Select.Providers { + repos, err := pr.gitea.ListRepositoriesForOrganization(ctx, provider.Gitea, provider.Organisation) + if err != nil { + return nil, err + } + + repoUrls = append(repoUrls, repos...) + } + + return repoUrls, nil +} + +func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, action *actions.Action) error { + cleanup, area, err := pr.prepareAction(ctx) + defer func() { + if cleanup != nil { + cleanup(ctx) + } + }() + if err != nil { + return err + } + + repo, err := pr.clone(ctx, area, repoUrl) + if err != nil { + return err + } + + if len(action.Schema.Queries) > 0 { + result, found, err := action.Query(ctx, area) + if err != nil { + return err + } + + if found { + pr.logger.Info("Query result", zap.Strings("result", result)) + // TODO: Append to real result, and return together + } + } + + if len(action.Schema.Actions) > 0 { + err = action.Execute(ctx, area) + if err != nil { + return err + } + + err = pr.commit(ctx, area, repo, repoUrl) + if err != nil { + return err + } + } + + pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) + + return nil +} + +func (pr *ProcessRepos) prepareAction( + ctx context.Context, +) (func(ctx context.Context), *storage.Area, error) { + pr.logger.Debug("Creating area") + area, err := pr.storage.CreateArea(ctx) + if err != nil { + return nil, nil, err + } + + cleanupfunc := func(ctx context.Context) { + pr.logger.Debug("Removing area", zap.String("path", area.Path)) + err = pr.storage.RemoveArea(ctx, area) + if err != nil { + panic(err) + } + } + + return cleanupfunc, area, nil +} + +func (pr *ProcessRepos) clone(ctx context.Context, area *storage.Area, repoUrl string) (*providers.GitRepo, error) { + pr.logger.Debug("Cloning repo", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) + cloneCtx, _ := context.WithTimeout(ctx, time.Second*5) + repo, err := pr.git.Clone(cloneCtx, area, repoUrl) + if err != nil { + return nil, err + } + + err = pr.git.CreateBranch(ctx, repo) + if err != nil { + return nil, err + } + + return repo, nil +} + +func (pr *ProcessRepos) commit(ctx context.Context, area *storage.Area, repo *providers.GitRepo, repoUrl string) error { + wt, err := pr.git.Add(ctx, area, repo) + if err != nil { + return fmt.Errorf("could not add file: %w", err) + } + + status, err := wt.Status() + if err != nil { + return err + } + + if status.IsClean() { + pr.logger.Info("Returning early, as no modifications are detected") + return nil + } + + err = pr.git.Commit(ctx, repo) + if err != nil { + return fmt.Errorf("could not get diff: %w", err) + } + + dryrun := false + if !dryrun { + + err = pr.git.Push(ctx, repo) + if err != nil { + return fmt.Errorf("could not push to repo: %w", err) + } + + url, err := giturls.Parse(repoUrl) + if err != nil { + return err + } + + head, err := repo.GetHEAD() + if err != nil { + return err + } + + path := strings.Split(url.Path, "/") + pr.logger.Debug("path string", zap.Strings("paths", path), zap.String("HEAD", head)) + + org := path[0] + repoName := path[1] + semanticName, _, ok := strings.Cut(repoName, ".") + if !ok { + semanticName = repoName + } + + originHead, err := pr.git.GetOriginHEADForRepo(ctx, repo) + if err != nil { + return err + } + + err = pr.gitea.CreatePr(ctx, fmt.Sprintf("%s://%s", "https", url.Host), org, semanticName, head, originHead, "kraken-apply") + if err != nil { + return err + } + } + return nil +} diff --git a/internal/gitproviders/gitea.go b/internal/gitproviders/gitea.go new file mode 100644 index 0000000..fcc277a --- /dev/null +++ b/internal/gitproviders/gitea.go @@ -0,0 +1,143 @@ +package gitproviders + +import ( + "context" + "errors" + "fmt" + "os" + "sync" + + "code.gitea.io/sdk/gitea" + "go.uber.org/zap" +) + +type Gitea struct { + logger *zap.Logger + giteamu sync.Mutex + giteaClients map[string]*gitea.Client +} + +func NewGitea(logger *zap.Logger) *Gitea { + return &Gitea{ + logger: logger, + giteamu: sync.Mutex{}, + giteaClients: make(map[string]*gitea.Client, 0), + } +} + +func (g *Gitea) ListRepositoriesForOrganization( + ctx context.Context, + server string, + organization string, +) ([]string, error) { + client, err := g.getOrCreateClient(ctx, server) + if err != nil { + return nil, err + } + + g.logger.Debug("Listing repos for gitea", zap.String("server", server)) + repos, resp, err := client.ListOrgRepos(organization, gitea.ListOrgReposOptions{ + ListOptions: gitea.ListOptions{ + Page: 0, + PageSize: 20, + }, + }) + if err != nil { + return nil, fmt.Errorf("could not list repos: %w", err) + } + + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("gitea responded with a non 200 status code (gitea response: %s)", resp.Status) + } + + repoUrls := make([]string, len(repos)) + for i, repo := range repos { + repoUrls[i] = repo.SSHURL + } + + return repoUrls, err +} + +func (g *Gitea) CreatePr( + ctx context.Context, + server string, + organization string, + repository string, + head string, + base string, + actionName string, +) error { + client, err := g.getOrCreateClient(ctx, server) + if err != nil { + return err + } + + prs, _, err := client.ListRepoPullRequests(organization, repository, gitea.ListPullRequestsOptions{ + ListOptions: gitea.ListOptions{ + Page: 0, + PageSize: 30, + }, + State: gitea.StateOpen, + Sort: "recentupdate", + Milestone: 0, + }) + if err != nil { + return fmt.Errorf( + "could not list repos, needed because we need to check for conflicts. Original error: %w", + err, + ) + } + for _, pr := range prs { + if pr.Head.Name == head { + g.logger.Info( + "returning early from creating pull-request, as it already exists.", + zap.String("repository", repository), + zap.String("pull-request", pr.URL), + ) + return nil + } + } + + pr, _, err := client.CreatePullRequest(organization, repository, gitea.CreatePullRequestOption{ + Head: head, + Base: base, + Title: actionName, + }) + if err != nil { + return err + } + + g.logger.Debug( + "Created pr", + zap.String("repository", repository), + zap.String("branch", head), + zap.String("pull-request", pr.URL), + ) + + return nil +} + +func (g *Gitea) getOrCreateClient(ctx context.Context, server string) (*gitea.Client, error) { + g.giteamu.Lock() + defer g.giteamu.Unlock() + client, ok := g.giteaClients[server] + if !ok || client == nil { + c, err := gitea.NewClient(server) + username, ok := os.LookupEnv("GITEA_USERNAME") + if !ok { + return nil, errors.New("missing environment variable GITEA_USERNAME") + } + apitoken, ok := os.LookupEnv("GITEA_API_TOKEN") + if !ok { + return nil, errors.New("missing environment variable GITEA_API_TOKEN") + } + c.SetBasicAuth(username, apitoken) + if err != nil { + return nil, err + } + g.giteaClients[server] = c + return c, nil + } + + return client, nil +} diff --git a/internal/schema/kraken.go b/internal/schema/kraken.go new file mode 100644 index 0000000..b1c76f5 --- /dev/null +++ b/internal/schema/kraken.go @@ -0,0 +1,32 @@ +package schema + +import "gopkg.in/yaml.v3" + +type KrakenSchema struct { + ApiVersion string `yaml:"apiVersion"` + Name string `yaml:"name"` + Select struct { + Repositories []string `yaml:"repositories"` + Providers []struct { + Gitea string `yaml:"gitea"` + Organisation string `yaml:"organisation"` + } `yaml:"providers"` + } `yaml:"select"` + Actions []struct { + Type string `yaml:"type"` + Entry string `yaml:"entry"` + } `yaml:"actions"` + Queries []struct { + Type string `yaml:"type"` + Query string `yaml:"query"` + } `yaml:"queries"` +} + +func Unmarshal(raw string) (*KrakenSchema, error) { + k := &KrakenSchema{} + err := yaml.Unmarshal([]byte(raw), k) + if err != nil { + return nil, err + } + return k, nil +} diff --git a/internal/serverdeps/server_deps.go b/internal/serverdeps/server_deps.go index bfa6d39..da9c333 100644 --- a/internal/serverdeps/server_deps.go +++ b/internal/serverdeps/server_deps.go @@ -1,6 +1,8 @@ package serverdeps import ( + actionc "git.front.kjuulh.io/kjuulh/kraken/internal/actions" + "git.front.kjuulh.io/kjuulh/kraken/internal/gitproviders" "git.front.kjuulh.io/kjuulh/kraken/internal/services/actions" "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" "git.front.kjuulh.io/kjuulh/kraken/internal/services/signer" @@ -59,6 +61,14 @@ func (deps *ServerDeps) GetAction() *actions.Action { return actions.NewAction(deps.logger.With(zap.Namespace("action"))) } +func (deps *ServerDeps) GetActionCreator() *actionc.ActionCreator { + return actionc.NewActionCreator(deps.logger.With(zap.Namespace("action")), deps) +} + +func (deps *ServerDeps) GetGitea() *gitproviders.Gitea { + return gitproviders.NewGitea(deps.logger.With(zap.Namespace("gitea"))) +} + func (deps *ServerDeps) GetOpenPGP() *signer.OpenPGP { return deps.openPGP } diff --git a/internal/services/providers/git.go b/internal/services/providers/git.go index f4dee7f..32cc1a7 100644 --- a/internal/services/providers/git.go +++ b/internal/services/providers/git.go @@ -2,6 +2,7 @@ package providers import ( "context" + "errors" "fmt" "time" @@ -30,6 +31,15 @@ type GitRepo struct { repo *git.Repository } +func (gr *GitRepo) GetHEAD() (string, error) { + head, err := gr.repo.Head() + if err != nil { + return "", err + } + + return head.Name().Short(), nil +} + type GitAuth string const ( @@ -53,6 +63,76 @@ func NewGit(logger *zap.Logger, gitConfig *GitConfig, openPGP *signer.OpenPGP) * return &Git{logger: logger, gitConfig: gitConfig, openPGP: openPGP} } +func (g *Git) GetOriginHEADForRepo(ctx context.Context, gitRepo *GitRepo) (string, error) { + auth, err := g.GetAuth() + if err != nil { + return "", err + } + + remote, err := gitRepo.repo.Remote("origin") + if err != nil { + return "", err + } + + refs, err := remote.ListContext(ctx, &git.ListOptions{ + Auth: auth, + }) + if err != nil { + return "", err + } + + headRef := "" + for _, ref := range refs { + //g.logger.Debug(ref.String()) + if !ref.Name().IsBranch() { + headRef = ref.Target().Short() + } + } + + if headRef == "" { + return "", errors.New("no upstream HEAD branch could be found") + } + + return headRef, nil +} + +func (g *Git) CloneBranch(ctx context.Context, storageArea *storage.Area, repoUrl string, branch string) (*GitRepo, error) { + g.logger.Debug( + "cloning repository", + zap.String("repoUrl", repoUrl), + zap.String("path", storageArea.Path), + ) + + auth, err := g.GetAuth() + if err != nil { + return nil, err + } + + cloneOptions := git.CloneOptions{ + URL: repoUrl, + Auth: auth, + RemoteName: "origin", + ReferenceName: plumbing.NewBranchReferenceName(branch), + SingleBranch: false, + NoCheckout: false, + Depth: 1, + RecurseSubmodules: 1, + Progress: g.getProgressWriter(), + Tags: 0, + InsecureSkipTLS: false, + CABundle: []byte{}, + } + + repo, err := git.PlainCloneContext(ctx, storageArea.Path, false, &cloneOptions) + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return nil, err + } + + g.logger.Debug("done cloning repo") + + return &GitRepo{repo: repo}, nil +} + func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl string) (*GitRepo, error) { g.logger.Debug( "cloning repository", @@ -70,7 +150,7 @@ func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl stri Auth: auth, RemoteName: "origin", ReferenceName: "refs/heads/main", - SingleBranch: true, + SingleBranch: false, NoCheckout: false, Depth: 1, RecurseSubmodules: 1, @@ -162,7 +242,7 @@ func (g *Git) CreateBranch(ctx context.Context, gitRepo *GitRepo) error { err = worktree.PullContext(ctx, &git.PullOptions{ RemoteName: "origin", ReferenceName: "refs/heads/main", - SingleBranch: true, + SingleBranch: false, Depth: 1, Auth: auth, RecurseSubmodules: 1, @@ -171,7 +251,7 @@ func (g *Git) CreateBranch(ctx context.Context, gitRepo *GitRepo) error { InsecureSkipTLS: false, CABundle: []byte{}, }) - if err != nil { + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { return fmt.Errorf("could not pull from origin: %w", err) } @@ -213,12 +293,11 @@ func (g *Git) Push(ctx context.Context, gitRepo *GitRepo) error { Auth: auth, Progress: g.getProgressWriter(), Prune: false, - Force: false, + Force: true, InsecureSkipTLS: false, CABundle: []byte{}, RequireRemoteRefs: []config.RefSpec{}, }) - if err != nil { return err } diff --git a/roadmap.md b/roadmap.md index 25b8252..34cf540 100644 --- a/roadmap.md +++ b/roadmap.md @@ -11,19 +11,36 @@ ### Not in scope -- [ ] Pooled runners -- [ ] CLI with options -- [ ] Server app -- [ ] Git hosting providers +- Pooled runners +- CLI with options +- Server app +- Git hosting providers ## Version 0.1 -- [ ] Allow instantiation of actions, kraken template repo etc. -- [ ] Create predicate handle -- [ ] Think about some sort of isolation +- [x] Setup a way to choose actions and predicates +- [x] Allow instantiation of actions, kraken template repo etc. +- [x] Implement docker action +- [x] Create pr for gitea provider +- [x] Providing query results - [ ] Create CLI to trigger action -- [ ] Setup pool of runners -- [ ] Run authenticated on servers -- [ ] Create queuing system + +### Not in scope ## Version 1.0 + +- [ ] Write README +- [ ] Make configurable ssh user +- [ ] Make configurable gpg keyset +- [ ] Make configurable git provider +- [ ] Create templating function +- [ ] Add way to see progress of runners +- [ ] Implement global .kraken store for easy access +- [ ] Move builders to start instead of every time + +## Version 1.x + +- Think about some sort of isolation +- Run authenticated on servers +- Create queuing system +- Setup pool of runners diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100755 index 0000000..e2f75ac --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +run_server="cuddle_cli x run_server" + +$run_server & + +sleep 1s + +cuddle_cli x run_client + +sleep 5s + +kill %1 diff --git a/scripts/run_client.sh b/scripts/run_client.sh new file mode 100755 index 0000000..237c23d --- /dev/null +++ b/scripts/run_client.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +current_branch=$(git branch --show-current) + +go run cmd/kraken/kraken.go process --actions-repo "git@git.front.kjuulh.io:kjuulh/kraken.git" --branch "$current_branch" --path "_examples/actions/write_a_readme" +go run cmd/kraken/kraken.go process --actions-repo "git@git.front.kjuulh.io:kjuulh/kraken.git" --branch "$current_branch" --path "_examples/queries/scrape_readme" diff --git a/scripts/run_server.sh b/scripts/run_server.sh new file mode 100755 index 0000000..59ec179 --- /dev/null +++ b/scripts/run_server.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +export $(cat .env | xargs) + +go run cmd/server/server.go start