kjuulh/octopush
1
0
Fork 0
Code Issues 1 Pull Requests 2 Packages Projects Releases Wiki Activity

v0.1 #8

Merged
kjuulh merged 15 commits from v0.1 into main 2022-09-18 16:49:35 +02:00
Conversation 0 Commits 15 Files Changed 31 +1111 -129
31 changed files with 1111 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1 +1,2 @@
.cuddle/ .cuddle/
.env

8
_examples/actions/docker_action/Dockerfile Normal file
View File

@ -0,0 +1,8 @@
FROM debian:bullseye-slim
# Kraken relies on this path being the specified path
WORKDIR /src/work/
COPY entry.sh /src/script.sh
CMD [ "/src/script.sh" ]

5
_examples/actions/docker_action/entry.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
set -e
echo "# README docker" > README.md

11
_examples/actions/docker_action/go.mod Normal file
View File

@ -0,0 +1,11 @@
module write_a_readme
go 1.19
require github.com/bitfield/script v0.20.2
require (
bitbucket.org/creachadair/shell v0.0.7 // indirect
github.com/itchyny/gojq v0.12.7 // indirect
github.com/itchyny/timefmt-go v0.1.3 // indirect
)

20
_examples/actions/docker_action/go.sum Normal file
View File

@ -0,0 +1,20 @@
bitbucket.org/creachadair/shell v0.0.7 h1:Z96pB6DkSb7F3Y3BBnJeOZH2gazyMTWlvecSD4vDqfk=
bitbucket.org/creachadair/shell v0.0.7/go.mod h1:oqtXSSvSYr4624lnnabXHaBsYW6RD80caLi2b3hJk0U=
github.com/bitfield/script v0.20.2 h1:4DexsRtBILVMEn3EZwHbtJdDqdk43sXI8gM3F04JXgs=
github.com/bitfield/script v0.20.2/go.mod h1:l3AZPVAtKQrL03bwh7nlNTUtgrgSWurpJSbtqspYrOA=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ=
github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw=
github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

11
_examples/actions/docker_action/kraken.yml Normal file
View File

@ -0,0 +1,11 @@
apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1
name: write-a-readme
select:
repositories:
- git@git.front.kjuulh.io:kjuulh/kraken-test.git
# providers:
# - gitea: https://git.front.kjuulh.io
# organisation: "cibus"
actions:
- type: docker-build
entry: Dockerfile

11
_examples/actions/write_a_readme/go.mod Normal file
View File

@ -0,0 +1,11 @@
module write_a_readme
go 1.19
require github.com/bitfield/script v0.20.2
require (
bitbucket.org/creachadair/shell v0.0.7 // indirect
github.com/itchyny/gojq v0.12.7 // indirect
github.com/itchyny/timefmt-go v0.1.3 // indirect
)

20
_examples/actions/write_a_readme/go.sum Normal file
View File

@ -0,0 +1,20 @@
bitbucket.org/creachadair/shell v0.0.7 h1:Z96pB6DkSb7F3Y3BBnJeOZH2gazyMTWlvecSD4vDqfk=
bitbucket.org/creachadair/shell v0.0.7/go.mod h1:oqtXSSvSYr4624lnnabXHaBsYW6RD80caLi2b3hJk0U=
github.com/bitfield/script v0.20.2 h1:4DexsRtBILVMEn3EZwHbtJdDqdk43sXI8gM3F04JXgs=
github.com/bitfield/script v0.20.2/go.mod h1:l3AZPVAtKQrL03bwh7nlNTUtgrgSWurpJSbtqspYrOA=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ=
github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw=
github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

11
_examples/actions/write_a_readme/kraken.yml Normal file
View File

@ -0,0 +1,11 @@
apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1
name: write-a-readme
select:
repositories:
- git@git.front.kjuulh.io:kjuulh/kraken-test.git
# providers:
# - gitea: https://git.front.kjuulh.io
# organisation: "cibus"
actions:
- type: go
entry: "main.go"

12
_examples/actions/write_a_readme/main.go Normal file
View File

@ -0,0 +1,12 @@
package main
import "github.com/bitfield/script"
func main() {
_, err := script.
Echo("# Readme").
WriteFile("README.md")
if err != nil {
panic(err)
}
}

11
_examples/queries/scrape_readme/kraken.yml Normal file
View File

@ -0,0 +1,11 @@
apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1
name: write-a-readme
select:
repositories:
- git@git.front.kjuulh.io:kjuulh/kraken-test.git
# providers:
# - gitea: https://git.front.kjuulh.io
# organisation: "cibus"
queries:
- type: grep
query: "# README"

31
cmd/kraken/commands/process.go
View File

@ -9,17 +9,32 @@ import (
) )
func CreateKrakenProcessCmd() *cobra.Command { func CreateKrakenProcessCmd() *cobra.Command {
var (
actionsRepo string
branch string
path string
)
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "process", Use: "process",
Run: func(cmd *cobra.Command, args []string) { RunE: func(cmd *cobra.Command, args []string) error {
if err := cmd.ParseFlags(args); err != nil {
return err
}
client := http.Client{} client := http.Client{}
var buf bytes.Buffer var buf bytes.Buffer
err := json.NewEncoder(&buf). err := json.NewEncoder(&buf).
Encode(struct { Encode(struct {
RepositoryUrls []string `json:"repositoryUrls"` Repository string `json:"repository"`
Branch string `json:"branch"`
Path string `json:"path"`
}{ }{
RepositoryUrls: []string{"git@git.front.kjuulh.io:kjuulh/kraken.git"}}) Repository: actionsRepo,
Branch: branch,
Path: path,
})
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -41,8 +56,18 @@ func CreateKrakenProcessCmd() *cobra.Command {
if resp.StatusCode >= 300 { if resp.StatusCode >= 300 {
panic(resp.Status) panic(resp.Status)
} }
return nil
}, },
} }
pf := cmd.PersistentFlags()
pf.StringVar(&actionsRepo, "actions-repo", "", "actions repo is the location of your actions, not where to apply the actions themselves, that should be self contained")
cmd.MarkPersistentFlagRequired("actions-repo")
pf.StringVar(&branch, "branch", "main", "which branch to look for actions in, will default to main")
pf.StringVar(&path, "path", "", "the location of the path inside the repository")
cmd.MarkPersistentFlagRequired("path")
return cmd return cmd
} }

2
cmd/server/server.go
View File

@ -15,6 +15,8 @@ func main() {
} }
_ = logger.Sync() _ = logger.Sync()
zap.ReplaceGlobals(logger)
Execute(logger) Execute(logger)
} }

6
cuddle.yaml
View File

@ -9,3 +9,9 @@ vars:
scripts: scripts:
push_github: push_github:
type: shell type: shell
run_server:
type: shell
run_client:
type: shell
run:
type: shell

3
go.mod
View File

@ -17,6 +17,7 @@ require (
) )
require ( require (
code.gitea.io/sdk/gitea v0.15.1 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/ProtonMail/go-mime v0.0.0-20220302105931-303f85f7fe0f // indirect github.com/ProtonMail/go-mime v0.0.0-20220302105931-303f85f7fe0f // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect
@ -30,6 +31,7 @@ require (
github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-playground/validator/v10 v10.10.0 // indirect github.com/go-playground/validator/v10 v10.10.0 // indirect
github.com/goccy/go-json v0.9.7 // indirect github.com/goccy/go-json v0.9.7 // indirect
github.com/hashicorp/go-version v1.2.1 // indirect
github.com/imdario/mergo v0.3.12 // indirect github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@ -47,6 +49,7 @@ require (
github.com/sirupsen/logrus v1.7.0 // indirect github.com/sirupsen/logrus v1.7.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/ugorji/go/codec v1.2.7 // indirect github.com/ugorji/go/codec v1.2.7 // indirect
github.com/whilp/git-urls v1.0.0 // indirect
github.com/xanzy/ssh-agent v0.3.2 // indirect github.com/xanzy/ssh-agent v0.3.2 // indirect
go.uber.org/atomic v1.10.0 // indirect go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect go.uber.org/multierr v1.8.0 // indirect

12
go.sum
View File

@ -1,3 +1,6 @@
code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE=
code.gitea.io/sdk/gitea v0.15.1 h1:WJreC7YYuxbn0UDaPuWIe/mtiNKTvLN8MLkaw71yx/M=
code.gitea.io/sdk/gitea v0.15.1/go.mod h1:klY2LVI3s3NChzIk/MzMn7G1FHrfU7qd63iSMVoHRBA=
git.front.kjuulh.io/kjuulh/curre v1.2.2 h1:0OwWIfekrMykdQg9bdmG80I+Mjc2k4i+sy903phuDWs= git.front.kjuulh.io/kjuulh/curre v1.2.2 h1:0OwWIfekrMykdQg9bdmG80I+Mjc2k4i+sy903phuDWs=
git.front.kjuulh.io/kjuulh/curre v1.2.2/go.mod h1:m7WpSehONLqPh/XF3F0BI0UOpLOfGuDmDEFI1XsM6fE= git.front.kjuulh.io/kjuulh/curre v1.2.2/go.mod h1:m7WpSehONLqPh/XF3F0BI0UOpLOfGuDmDEFI1XsM6fE=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@ -71,6 +74,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@ -148,9 +153,12 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
github.com/xanzy/ssh-agent v0.3.2 h1:eKj4SX2Fe7mui28ZgnFW5fmTz1EIr7ugo5s6wDxdHBM= github.com/xanzy/ssh-agent v0.3.2 h1:eKj4SX2Fe7mui28ZgnFW5fmTz1EIr7ugo5s6wDxdHBM=
github.com/xanzy/ssh-agent v0.3.2/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xanzy/ssh-agent v0.3.2/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
@ -184,10 +192,12 @@ golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
@ -195,6 +205,7 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI=
golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -233,6 +244,7 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

77
internal/actions/action.go Normal file
View File

@ -0,0 +1,77 @@
package actions
import (
"context"
"errors"
"git.front.kjuulh.io/kjuulh/kraken/internal/actions/builders"
"git.front.kjuulh.io/kjuulh/kraken/internal/actions/querier"
"git.front.kjuulh.io/kjuulh/kraken/internal/schema"
"git.front.kjuulh.io/kjuulh/kraken/internal/services/storage"
"go.uber.org/zap"
)
type Action struct {
Schema *schema.KrakenSchema
SchemaPath string
}
func (a *Action) Execute(ctx context.Context, area *storage.Area) error {
for _, action := range a.Schema.Actions {
switch action.Type {
case "go":
exe, err := builders.NewGo(zap.L()).Build(ctx, a.SchemaPath, action.Entry)
if err != nil {
return err
}
err = exe(ctx, area.Path)
if err != nil {
return err
}
zap.L().Debug("Execution done")
case "docker-build":
zap.L().Debug("Building docker-build")
runCmd, err := builders.NewDockerBuild(zap.L()).Build(ctx, a.SchemaPath, action.Entry)
if err != nil {
return err
}
err = runCmd(ctx, area.Path)
if err != nil {
return err
}
return nil
default:
return errors.New("could not determine action type")
}
}
return nil
}
func (a *Action) Query(ctx context.Context, area *storage.Area) ([]string, bool, error) {
for _, query := range a.Schema.Queries {
switch query.Type {
case "grep":
exe, err := querier.NewRipGrep(zap.L()).Build(ctx, a.SchemaPath, query.Query)
if err != nil {
return nil, false, err
}
output, found, err := exe(ctx, area.Path)
if err != nil {
return nil, false, err
}
zap.L().Debug("Execution done")
return output, found, nil
default:
return nil, false, errors.New("could not determine query type")
}
}
return nil, false, nil
}

85
internal/actions/action_creator.go Normal file
View File

@ -0,0 +1,85 @@
package actions
import (
"context"
"fmt"
"os"
"path"
"time"
"git.front.kjuulh.io/kjuulh/kraken/internal/schema"
"git.front.kjuulh.io/kjuulh/kraken/internal/services/providers"
"git.front.kjuulh.io/kjuulh/kraken/internal/services/storage"
"go.uber.org/zap"
)
type (
ActionCreatorOps struct {
RepositoryUrl string
Branch string
Path string
}
ActionCreator struct {
logger *zap.Logger
storage *storage.Service
git *providers.Git
}
ActionCreatorDeps interface {
GetStorageService() *storage.Service
GetGitProvider() *providers.Git
}
)
func NewActionCreator(logger *zap.Logger, deps ActionCreatorDeps) *ActionCreator {
return &ActionCreator{
logger: logger,
storage: deps.GetStorageService(),
git: deps.GetGitProvider(),
}
}
func (ac *ActionCreator) Prepare(ctx context.Context, ops *ActionCreatorOps) (*Action, error) {
area, err := ac.storage.CreateArea(ctx)
if err != nil {
ac.logger.Error("failed to allocate area", zap.Error(err))
return nil, err
}
cloneCtx, _ := context.WithTimeout(ctx, time.Second*10)
_, err = ac.git.CloneBranch(cloneCtx, area, ops.RepositoryUrl, ops.Branch)
if err != nil {
ac.logger.Error("could not clone repo", zap.Error(err))
return nil, err
}
executorUrl := path.Join(area.Path, ops.Path)
if _, err = os.Stat(executorUrl); os.IsNotExist(err) {
return nil, fmt.Errorf("path is invalid: %s", ops.Path)
}
contents, err := os.ReadFile(path.Join(executorUrl, "kraken.yml"))
if err != nil {
return nil, err
}
krakenSchema, err := schema.Unmarshal(string(contents))
if err != nil {
return nil, err
}
ac.logger.Debug("Action creator done")
return &Action{
Schema: krakenSchema,
SchemaPath: executorUrl,
}, nil
}
func (ac *ActionCreator) Cleanup(ctx context.Context, area *storage.Area) {
ac.logger.Debug("Removing area", zap.String("path", area.Path))
err := ac.storage.RemoveArea(ctx, area)
if err != nil {
panic(err)
}
}

95
internal/actions/builders/docker.go Normal file
View File

@ -0,0 +1,95 @@
package builders
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"os"
"os/exec"
"go.uber.org/zap"
"go.uber.org/zap/zapio"
)
type DockerBuild struct {
logger *zap.Logger
}
func NewDockerBuild(logger *zap.Logger) *DockerBuild {
return &DockerBuild{logger: logger}
}
type DockerRunCommand func(ctx context.Context, victimPath string) error
func (g *DockerBuild) Build(ctx context.Context, modulePath, entryPath string) (DockerRunCommand, error) {
g.logger.Debug("Building docker image", zap.String("actiondir", modulePath), zap.String("entry", entryPath))
if _, err := os.Stat(fmt.Sprintf("%s/%s", modulePath, entryPath)); os.IsNotExist(err) {
return nil, errors.New("could not find entry")
}
b := make([]byte, 20)
_, err := rand.Reader.Read(b)
if err != nil {
return nil, err
}
tag := hex.EncodeToString(b)
buildDockerCmd := fmt.Sprintf("(cd %s; docker build -f %s --tag kraken/%s .)", modulePath, entryPath, tag)
g.logger.Debug("Running command", zap.String("command", buildDockerCmd))
cmd := exec.CommandContext(
ctx,
"/bin/bash",
"-c",
buildDockerCmd,
)
debugwriter := &zapio.Writer{
Log: g.logger,
Level: zap.DebugLevel,
}
defer debugwriter.Close()
cmd.Stdout = debugwriter
cmd.Stderr = debugwriter
err = cmd.Start()
if err != nil {
return nil, err
}
err = cmd.Wait()
if err != nil {
return nil, err
}
g.logger.Debug("Docker image built!")
return func(ctx context.Context, victimPath string) error {
g.logger.Debug("Executing script", zap.String("victim", victimPath))
cmd := exec.CommandContext(
ctx,
"/bin/bash",
"-c",
fmt.Sprintf("docker run --rm -v %s/:/src/work/ kraken/%s", victimPath, tag),
)
runDockerWriter := &zapio.Writer{
Log: g.logger,
Level: zap.DebugLevel,
}
defer runDockerWriter.Close()
cmd.Stdout = runDockerWriter
cmd.Stderr = runDockerWriter
err = cmd.Start()
if err != nil {
return err
}
return cmd.Wait()
}, nil
}

46
internal/actions/builders/go.go Normal file
View File

@ -0,0 +1,46 @@
package builders
import (
"context"
"errors"
"fmt"
"os"
"os/exec"
"go.uber.org/zap"
)
type Go struct {
logger *zap.Logger
}
func NewGo(logger *zap.Logger) *Go {
return &Go{logger: logger}
}
type GoExecutable func(ctx context.Context, victimPath string) error
func (g *Go) Build(ctx context.Context, modulePath, entryPath string) (GoExecutable, error) {
g.logger.Debug("Building go binary", zap.String("actiondir", modulePath), zap.String("entry", entryPath))
if _, err := os.Stat(fmt.Sprintf("%s/%s", modulePath, entryPath)); os.IsNotExist(err) {
return nil, errors.New("could not find entry")
}
err := exec.CommandContext(
ctx,
"/bin/bash",
"-c",
fmt.Sprintf("(cd %s; go build -o main %s)", modulePath, entryPath),
).Run()
if err != nil {
return nil, err
}
g.logger.Debug("Go binary built!")
return func(ctx context.Context, victimPath string) error {
g.logger.Debug("Executing script", zap.String("victim", victimPath))
return exec.CommandContext(ctx, "/bin/bash", "-c", fmt.Sprintf("(cd %s; %s/main)", victimPath, modulePath)).Run()
}, nil
}

106
internal/actions/querier/ripgrep.go Normal file
View File

@ -0,0 +1,106 @@
package querier
import (
"context"
"fmt"
"io"
"os/exec"
"strings"
"go.uber.org/zap"
"go.uber.org/zap/zapio"
)
type RipGrep struct {
logger *zap.Logger
}
func NewRipGrep(logger *zap.Logger) *RipGrep {
return &RipGrep{logger: logger}
}
type RipGrepCommand func(ctx context.Context, victimPath string) ([]string, bool, error)
func (g *RipGrep) Build(ctx context.Context, modulePath, query string) (RipGrepCommand, error) {
g.logger.Debug("Pulling docker image", zap.String("actiondir", modulePath), zap.String("query", query))
pullDockerImage := "docker pull mbologna/docker-ripgrep"
g.logger.Debug("Running command", zap.String("command", pullDockerImage))
cmd := exec.CommandContext(
ctx,
"/bin/bash",
"-c",
pullDockerImage,
)
debugwriter := &zapio.Writer{
Log: g.logger,
Level: zap.DebugLevel,
}
defer debugwriter.Close()
cmd.Stdout = debugwriter
cmd.Stderr = debugwriter
err := cmd.Start()
if err != nil {
return nil, err
}
err = cmd.Wait()
if err != nil {
return nil, err
}
g.logger.Debug("Docker image pulled")
return func(ctx context.Context, victimPath string) ([]string, bool, error) {
g.logger.Debug("Executing script", zap.String("victim", victimPath))
runRipGrepCmd := fmt.Sprintf("docker run --rm -v %s/:/data:ro mbologna/docker-ripgrep rg -i '%s' || true", victimPath, query)
g.logger.Debug("Execute ripgrep query", zap.String("command", runRipGrepCmd))
cmd := exec.CommandContext(
ctx,
"/bin/bash",
"-c",
runRipGrepCmd,
)
runDockerWriter := &zapio.Writer{
Log: g.logger,
Level: zap.DebugLevel,
}
defer runDockerWriter.Close()
builder := &strings.Builder{}
combinedWriter := io.MultiWriter(runDockerWriter, builder)
cmd.Stdout = combinedWriter
cmd.Stderr = combinedWriter
err = cmd.Start()
if err != nil {
return nil, false, err
}
err = cmd.Wait()
if err != nil {
return nil, false, err
}
contents := strings.Split(builder.String(), "\n")
validatedOutput := make([]string, 0)
for _, c := range contents {
if !strings.Contains(c, "WARNING:") {
validatedOutput = append(validatedOutput, c)
}
}
found := len(validatedOutput) > 0
return validatedOutput, found, nil
}, nil
}

13
internal/api/process_command.go
View File

@ -16,7 +16,9 @@ func CommandRoute(logger *zap.Logger, app *gin.Engine, deps *serverdeps.ServerDe
commandRoute := app.Group("commands") commandRoute := app.Group("commands")
commandRoute.POST("processRepos", func(c *gin.Context) { commandRoute.POST("processRepos", func(c *gin.Context) {
type processReposRequest struct { type processReposRequest struct {
RepositoryUrls []string `json:"repositoryUrls"` Repository string `json:"repository"`
Branch string `json:"branch"`
Path string `json:"path"`
} }
var request processReposRequest var request processReposRequest
err := c.BindJSON(&request) err := c.BindJSON(&request)
@ -28,11 +30,14 @@ func CommandRoute(logger *zap.Logger, app *gin.Engine, deps *serverdeps.ServerDe
jobId := uuid.New().String() jobId := uuid.New().String()
go func(repositoryUrls []string, jobId string) { go func(repository string, branch string, path string, jobId string) {
ctx := context.WithValue(context.Background(), jobs.JobId{}, jobId) ctx := context.WithValue(context.Background(), jobs.JobId{}, jobId)
processRepos := commands.NewProcessRepos(logger, deps) processRepos := commands.NewProcessRepos(logger, deps)
err = processRepos.Process(ctx, repositoryUrls) err = processRepos.Process(ctx, repository, branch, path)
}(request.RepositoryUrls, jobId) if err != nil {
logger.Error("could not process repo", zap.Error(err))
}
}(request.Repository, request.Branch, request.Path, jobId)
c.Status(http.StatusAccepted) c.Status(http.StatusAccepted)
}) })

230
internal/commands/process_repos.go
View File

@ -3,16 +3,16 @@ package commands
import ( import (
"context" "context"
"fmt" "fmt"
"io/fs" "strings"
"os"
"path"
"path/filepath"
"sync" "sync"
"time" "time"
"git.front.kjuulh.io/kjuulh/kraken/internal/services/actions" "git.front.kjuulh.io/kjuulh/kraken/internal/actions"
"git.front.kjuulh.io/kjuulh/kraken/internal/gitproviders"
"git.front.kjuulh.io/kjuulh/kraken/internal/schema"
"git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers"
"git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage"
giturls "github.com/whilp/git-urls"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -21,13 +21,15 @@ type (
logger *zap.Logger logger *zap.Logger
storage *storage.Service storage *storage.Service
git *providers.Git git *providers.Git
action *actions.Action actionCreator *actions.ActionCreator
gitea *gitproviders.Gitea
} }
ProcessReposDeps interface { ProcessReposDeps interface {
GetStorageService() *storage.Service GetStorageService() *storage.Service
GetGitProvider() *providers.Git GetGitProvider() *providers.Git
GetAction() *actions.Action GetActionCreator() *actions.ActionCreator
GetGitea() *gitproviders.Gitea
} }
) )
@ -36,117 +38,203 @@ func NewProcessRepos(logger *zap.Logger, deps ProcessReposDeps) *ProcessRepos {
logger: logger, logger: logger,
storage: deps.GetStorageService(), storage: deps.GetStorageService(),
git: deps.GetGitProvider(), git: deps.GetGitProvider(),
action: deps.GetAction(), actionCreator: deps.GetActionCreator(),
gitea: deps.GetGitea(),
} }
} }
func (pr *ProcessRepos) Process(ctx context.Context, repositoryUrls []string) error { func (pr *ProcessRepos) Process(ctx context.Context, repository string, branch string, actionPath string) error {
// Clone repos action, err := pr.actionCreator.Prepare(ctx, &actions.ActionCreatorOps{
RepositoryUrl: repository,
Branch: branch,
Path: actionPath,
})
if err != nil {
return err
}
repositoryUrls, err := pr.getRepoUrls(ctx, action.Schema)
if err != nil {
return err
}
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
wg.Add(len(repositoryUrls)) wg.Add(len(repositoryUrls))
errChan := make(chan error, 1)
for _, repoUrl := range repositoryUrls { for _, repoUrl := range repositoryUrls {
go func(ctx context.Context, repoUrl string) { go func(ctx context.Context, repoUrl string) {
defer func() { defer func() {
wg.Done() wg.Done()
}() }()
pr.logger.Debug("Creating area", zap.String("repoUrl", repoUrl)) err := pr.processRepo(ctx, repoUrl, action)
if err != nil {
pr.logger.Error("could not process repo", zap.Error(err))
}
}(ctx, repoUrl)
}
wg.Wait()
pr.logger.Debug("finished processing all repos", zap.Strings("repos", repositoryUrls))
return nil
}
func (pr *ProcessRepos) getRepoUrls(ctx context.Context, schema *schema.KrakenSchema) ([]string, error) {
repoUrls := make([]string, 0)
repoUrls = append(repoUrls, schema.Select.Repositories...)
for _, provider := range schema.Select.Providers {
repos, err := pr.gitea.ListRepositoriesForOrganization(ctx, provider.Gitea, provider.Organisation)
if err != nil {
return nil, err
}
repoUrls = append(repoUrls, repos...)
}
return repoUrls, nil
}
func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, action *actions.Action) error {
cleanup, area, err := pr.prepareAction(ctx)
defer func() {
if cleanup != nil {
cleanup(ctx)
}
}()
if err != nil {
return err
}
repo, err := pr.clone(ctx, area, repoUrl)
if err != nil {
return err
}
if len(action.Schema.Queries) > 0 {
result, found, err := action.Query(ctx, area)
if err != nil {
return err
}
if found {
pr.logger.Info("Query result", zap.Strings("result", result))
// TODO: Append to real result, and return together
}
}
if len(action.Schema.Actions) > 0 {
err = action.Execute(ctx, area)
if err != nil {
return err
}
err = pr.commit(ctx, area, repo, repoUrl)
if err != nil {
return err
}
}
pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl))
return nil
}
func (pr *ProcessRepos) prepareAction(
ctx context.Context,
) (func(ctx context.Context), *storage.Area, error) {
pr.logger.Debug("Creating area")
area, err := pr.storage.CreateArea(ctx) area, err := pr.storage.CreateArea(ctx)
if err != nil { if err != nil {
pr.logger.Error("failed to allocate area", zap.Error(err)) return nil, nil, err
errChan <- err
return
} }
defer func(ctx context.Context) { cleanupfunc := func(ctx context.Context) {
pr.logger.Debug("Removing area", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) pr.logger.Debug("Removing area", zap.String("path", area.Path))
err = pr.storage.RemoveArea(ctx, area) err = pr.storage.RemoveArea(ctx, area)
if err != nil { if err != nil {
errChan <- err panic(err)
return }
} }
}(ctx) return cleanupfunc, area, nil
}
func (pr *ProcessRepos) clone(ctx context.Context, area *storage.Area, repoUrl string) (*providers.GitRepo, error) {
pr.logger.Debug("Cloning repo", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) pr.logger.Debug("Cloning repo", zap.String("path", area.Path), zap.String("repoUrl", repoUrl))
cloneCtx, _ := context.WithTimeout(ctx, time.Second*5) cloneCtx, _ := context.WithTimeout(ctx, time.Second*5)
repo, err := pr.git.Clone(cloneCtx, area, repoUrl) repo, err := pr.git.Clone(cloneCtx, area, repoUrl)
if err != nil { if err != nil {
pr.logger.Error("could not clone repo", zap.Error(err)) return nil, err
errChan <- err
return
} }
err = pr.git.CreateBranch(ctx, repo) err = pr.git.CreateBranch(ctx, repo)
if err != nil { if err != nil {
pr.logger.Error("could not create branch", zap.Error(err)) return nil, err
errChan <- err
return
} }
err = pr.action.Run( return repo, nil
ctx, }
area,
func(_ context.Context, area *storage.Area) (bool, error) {
pr.logger.Debug("checking predicate", zap.String("area", area.Path))
contains := false
filepath.WalkDir(area.Path, func(path string, d fs.DirEntry, err error) error {
if d.Name() == "roadmap.md" {
contains = true
}
return nil
})
return contains, nil
},
func(_ context.Context, area *storage.Area) error {
pr.logger.Debug("running action", zap.String("area", area.Path))
readme := path.Join(area.Path, "README.md")
file, err := os.Create(readme)
if err != nil {
return fmt.Errorf("could not create readme: %w", err)
}
_, err = file.WriteString("# Readme")
if err != nil {
return fmt.Errorf("could not write readme: %w", err)
}
_, err = pr.git.Add(ctx, area, repo) func (pr *ProcessRepos) commit(ctx context.Context, area *storage.Area, repo *providers.GitRepo, repoUrl string) error {
wt, err := pr.git.Add(ctx, area, repo)
if err != nil { if err != nil {
return fmt.Errorf("could not add file: %w", err) return fmt.Errorf("could not add file: %w", err)
} }
status, err := wt.Status()
if err != nil {
return err
}
if status.IsClean() {
pr.logger.Info("Returning early, as no modifications are detected")
return nil
}
err = pr.git.Commit(ctx, repo) err = pr.git.Commit(ctx, repo)
if err != nil { if err != nil {
return fmt.Errorf("could not get diff: %w", err) return fmt.Errorf("could not get diff: %w", err)
} }
return nil dryrun := false
}, false) if !dryrun {
if err != nil {
pr.logger.Error("could not run action", zap.Error(err))
errChan <- err
return
}
err = pr.git.Push(ctx, repo) err = pr.git.Push(ctx, repo)
if err != nil { if err != nil {
pr.logger.Error("could not push to repo", zap.Error(err)) return fmt.Errorf("could not push to repo: %w", err)
errChan <- err
return
} }
pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) url, err := giturls.Parse(repoUrl)
}(ctx, repoUrl) if err != nil {
}
wg.Wait()
close(errChan)
pr.logger.Debug("finished processing all repos")
for err := range errChan {
return err return err
} }
head, err := repo.GetHEAD()
if err != nil {
return err
}
path := strings.Split(url.Path, "/")
pr.logger.Debug("path string", zap.Strings("paths", path), zap.String("HEAD", head))
org := path[0]
repoName := path[1]
semanticName, _, ok := strings.Cut(repoName, ".")
if !ok {
semanticName = repoName
}
originHead, err := pr.git.GetOriginHEADForRepo(ctx, repo)
if err != nil {
return err
}
err = pr.gitea.CreatePr(ctx, fmt.Sprintf("%s://%s", "https", url.Host), org, semanticName, head, originHead, "kraken-apply")
if err != nil {
return err
}
}
return nil return nil
} }

143
internal/gitproviders/gitea.go Normal file
View File

@ -0,0 +1,143 @@
package gitproviders
import (
"context"
"errors"
"fmt"
"os"
"sync"
"code.gitea.io/sdk/gitea"
"go.uber.org/zap"
)
type Gitea struct {
logger *zap.Logger
giteamu sync.Mutex
giteaClients map[string]*gitea.Client
}
func NewGitea(logger *zap.Logger) *Gitea {
return &Gitea{
logger: logger,
giteamu: sync.Mutex{},
giteaClients: make(map[string]*gitea.Client, 0),
}
}
func (g *Gitea) ListRepositoriesForOrganization(
ctx context.Context,
server string,
organization string,
) ([]string, error) {
client, err := g.getOrCreateClient(ctx, server)
if err != nil {
return nil, err
}
g.logger.Debug("Listing repos for gitea", zap.String("server", server))
repos, resp, err := client.ListOrgRepos(organization, gitea.ListOrgReposOptions{
ListOptions: gitea.ListOptions{
Page: 0,
PageSize: 20,
},
})
if err != nil {
return nil, fmt.Errorf("could not list repos: %w", err)
}
if resp.StatusCode >= 300 {
return nil, fmt.Errorf("gitea responded with a non 200 status code (gitea response: %s)", resp.Status)
}
repoUrls := make([]string, len(repos))
for i, repo := range repos {
repoUrls[i] = repo.SSHURL
}
return repoUrls, err
}
func (g *Gitea) CreatePr(
ctx context.Context,
server string,
organization string,
repository string,
head string,
base string,
actionName string,
) error {
client, err := g.getOrCreateClient(ctx, server)
if err != nil {
return err
}
prs, _, err := client.ListRepoPullRequests(organization, repository, gitea.ListPullRequestsOptions{
ListOptions: gitea.ListOptions{
Page: 0,
PageSize: 30,
},
State: gitea.StateOpen,
Sort: "recentupdate",
Milestone: 0,
})
if err != nil {
return fmt.Errorf(
"could not list repos, needed because we need to check for conflicts. Original error: %w",
err,
)
}
for _, pr := range prs {
if pr.Head.Name == head {
g.logger.Info(
"returning early from creating pull-request, as it already exists.",
zap.String("repository", repository),
zap.String("pull-request", pr.URL),
)
return nil
}
}
pr, _, err := client.CreatePullRequest(organization, repository, gitea.CreatePullRequestOption{
Head: head,
Base: base,
Title: actionName,
})
if err != nil {
return err
}
g.logger.Debug(
"Created pr",
zap.String("repository", repository),
zap.String("branch", head),
zap.String("pull-request", pr.URL),
)
return nil
}
func (g *Gitea) getOrCreateClient(ctx context.Context, server string) (*gitea.Client, error) {
g.giteamu.Lock()
defer g.giteamu.Unlock()
client, ok := g.giteaClients[server]
if !ok || client == nil {
c, err := gitea.NewClient(server)
username, ok := os.LookupEnv("GITEA_USERNAME")
if !ok {
return nil, errors.New("missing environment variable GITEA_USERNAME")
}
apitoken, ok := os.LookupEnv("GITEA_API_TOKEN")
if !ok {
return nil, errors.New("missing environment variable GITEA_API_TOKEN")
}
c.SetBasicAuth(username, apitoken)
if err != nil {
return nil, err
}
g.giteaClients[server] = c
return c, nil
}
return client, nil
}

32
internal/schema/kraken.go Normal file
View File

@ -0,0 +1,32 @@
package schema
import "gopkg.in/yaml.v3"
type KrakenSchema struct {
ApiVersion string `yaml:"apiVersion"`
Name string `yaml:"name"`
Select struct {
Repositories []string `yaml:"repositories"`
Providers []struct {
Gitea string `yaml:"gitea"`
Organisation string `yaml:"organisation"`
} `yaml:"providers"`
} `yaml:"select"`
Actions []struct {
Type string `yaml:"type"`
Entry string `yaml:"entry"`
} `yaml:"actions"`
Queries []struct {
Type string `yaml:"type"`
Query string `yaml:"query"`
} `yaml:"queries"`
}
func Unmarshal(raw string) (*KrakenSchema, error) {
k := &KrakenSchema{}
err := yaml.Unmarshal([]byte(raw), k)
if err != nil {
return nil, err
}
return k, nil
}

10
internal/serverdeps/server_deps.go
View File

@ -1,6 +1,8 @@
package serverdeps package serverdeps
import ( import (
actionc "git.front.kjuulh.io/kjuulh/kraken/internal/actions"
"git.front.kjuulh.io/kjuulh/kraken/internal/gitproviders"
"git.front.kjuulh.io/kjuulh/kraken/internal/services/actions" "git.front.kjuulh.io/kjuulh/kraken/internal/services/actions"
"git.front.kjuulh.io/kjuulh/kraken/internal/services/providers" "git.front.kjuulh.io/kjuulh/kraken/internal/services/providers"
"git.front.kjuulh.io/kjuulh/kraken/internal/services/signer" "git.front.kjuulh.io/kjuulh/kraken/internal/services/signer"
@ -59,6 +61,14 @@ func (deps *ServerDeps) GetAction() *actions.Action {
return actions.NewAction(deps.logger.With(zap.Namespace("action"))) return actions.NewAction(deps.logger.With(zap.Namespace("action")))
} }
func (deps *ServerDeps) GetActionCreator() *actionc.ActionCreator {
return actionc.NewActionCreator(deps.logger.With(zap.Namespace("action")), deps)
}
func (deps *ServerDeps) GetGitea() *gitproviders.Gitea {
return gitproviders.NewGitea(deps.logger.With(zap.Namespace("gitea")))
}
func (deps *ServerDeps) GetOpenPGP() *signer.OpenPGP { func (deps *ServerDeps) GetOpenPGP() *signer.OpenPGP {
return deps.openPGP return deps.openPGP
} }

89
internal/services/providers/git.go
View File

@ -2,6 +2,7 @@ package providers
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"time" "time"
@ -30,6 +31,15 @@ type GitRepo struct {
repo *git.Repository repo *git.Repository
} }
func (gr *GitRepo) GetHEAD() (string, error) {
head, err := gr.repo.Head()
if err != nil {
return "", err
}
return head.Name().Short(), nil
}
type GitAuth string type GitAuth string
const ( const (
@ -53,6 +63,76 @@ func NewGit(logger *zap.Logger, gitConfig *GitConfig, openPGP *signer.OpenPGP) *
return &Git{logger: logger, gitConfig: gitConfig, openPGP: openPGP} return &Git{logger: logger, gitConfig: gitConfig, openPGP: openPGP}
} }
func (g *Git) GetOriginHEADForRepo(ctx context.Context, gitRepo *GitRepo) (string, error) {
auth, err := g.GetAuth()
if err != nil {
return "", err
}
remote, err := gitRepo.repo.Remote("origin")
if err != nil {
return "", err
}
refs, err := remote.ListContext(ctx, &git.ListOptions{
Auth: auth,
})
if err != nil {
return "", err
}
headRef := ""
for _, ref := range refs {
//g.logger.Debug(ref.String())
if !ref.Name().IsBranch() {
headRef = ref.Target().Short()
}
}
if headRef == "" {
return "", errors.New("no upstream HEAD branch could be found")
}
return headRef, nil
}
func (g *Git) CloneBranch(ctx context.Context, storageArea *storage.Area, repoUrl string, branch string) (*GitRepo, error) {
g.logger.Debug(
"cloning repository",
zap.String("repoUrl", repoUrl),
zap.String("path", storageArea.Path),
)
auth, err := g.GetAuth()
if err != nil {
return nil, err
}
cloneOptions := git.CloneOptions{
URL: repoUrl,
Auth: auth,
RemoteName: "origin",
ReferenceName: plumbing.NewBranchReferenceName(branch),
SingleBranch: false,
NoCheckout: false,
Depth: 1,
RecurseSubmodules: 1,
Progress: g.getProgressWriter(),
Tags: 0,
InsecureSkipTLS: false,
CABundle: []byte{},
}
repo, err := git.PlainCloneContext(ctx, storageArea.Path, false, &cloneOptions)
if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
return nil, err
}
g.logger.Debug("done cloning repo")
return &GitRepo{repo: repo}, nil
}
func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl string) (*GitRepo, error) { func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl string) (*GitRepo, error) {
g.logger.Debug( g.logger.Debug(
"cloning repository", "cloning repository",
@ -70,7 +150,7 @@ func (g *Git) Clone(ctx context.Context, storageArea *storage.Area, repoUrl stri
Auth: auth, Auth: auth,
RemoteName: "origin", RemoteName: "origin",
ReferenceName: "refs/heads/main", ReferenceName: "refs/heads/main",
SingleBranch: true, SingleBranch: false,
NoCheckout: false, NoCheckout: false,
Depth: 1, Depth: 1,
RecurseSubmodules: 1, RecurseSubmodules: 1,
@ -162,7 +242,7 @@ func (g *Git) CreateBranch(ctx context.Context, gitRepo *GitRepo) error {
err = worktree.PullContext(ctx, &git.PullOptions{ err = worktree.PullContext(ctx, &git.PullOptions{
RemoteName: "origin", RemoteName: "origin",
ReferenceName: "refs/heads/main", ReferenceName: "refs/heads/main",
SingleBranch: true, SingleBranch: false,
Depth: 1, Depth: 1,
Auth: auth, Auth: auth,
RecurseSubmodules: 1, RecurseSubmodules: 1,
@ -171,7 +251,7 @@ func (g *Git) CreateBranch(ctx context.Context, gitRepo *GitRepo) error {
InsecureSkipTLS: false, InsecureSkipTLS: false,
CABundle: []byte{}, CABundle: []byte{},
}) })
if err != nil { if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
return fmt.Errorf("could not pull from origin: %w", err) return fmt.Errorf("could not pull from origin: %w", err)
} }
@ -213,12 +293,11 @@ func (g *Git) Push(ctx context.Context, gitRepo *GitRepo) error {
Auth: auth, Auth: auth,
Progress: g.getProgressWriter(), Progress: g.getProgressWriter(),
Prune: false, Prune: false,
Force: false, Force: true,
InsecureSkipTLS: false, InsecureSkipTLS: false,
CABundle: []byte{}, CABundle: []byte{},
RequireRemoteRefs: []config.RefSpec{}, RequireRemoteRefs: []config.RefSpec{},
}) })
if err != nil { if err != nil {
return err return err
} }

37
roadmap.md
View File

@ -11,19 +11,36 @@
### Not in scope ### Not in scope
- [ ] Pooled runners - Pooled runners
- [ ] CLI with options - CLI with options
- [ ] Server app - Server app
- [ ] Git hosting providers - Git hosting providers
## Version 0.1 ## Version 0.1
- [ ] Allow instantiation of actions, kraken template repo etc. - [x] Setup a way to choose actions and predicates
- [ ] Create predicate handle - [x] Allow instantiation of actions, kraken template repo etc.
- [ ] Think about some sort of isolation - [x] Implement docker action
- [x] Create pr for gitea provider
- [x] Providing query results
- [ ] Create CLI to trigger action - [ ] Create CLI to trigger action
- [ ] Setup pool of runners
- [ ] Run authenticated on servers ### Not in scope
- [ ] Create queuing system
## Version 1.0 ## Version 1.0
- [ ] Write README
- [ ] Make configurable ssh user
- [ ] Make configurable gpg keyset
- [ ] Make configurable git provider
- [ ] Create templating function
- [ ] Add way to see progress of runners
- [ ] Implement global .kraken store for easy access
- [ ] Move builders to start instead of every time
## Version 1.x
- Think about some sort of isolation
- Run authenticated on servers
- Create queuing system
- Setup pool of runners

15
scripts/run.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/bash
set -e
run_server="cuddle_cli x run_server"
$run_server &
sleep 1s
cuddle_cli x run_client
sleep 5s
kill %1

8
scripts/run_client.sh Executable file
View File

@ -0,0 +1,8 @@
#!/bin/bash
set -e
current_branch=$(git branch --show-current)
go run cmd/kraken/kraken.go process --actions-repo "git@git.front.kjuulh.io:kjuulh/kraken.git" --branch "$current_branch" --path "_examples/actions/write_a_readme"
go run cmd/kraken/kraken.go process --actions-repo "git@git.front.kjuulh.io:kjuulh/kraken.git" --branch "$current_branch" --path "_examples/queries/scrape_readme"

7
scripts/run_server.sh Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
set -e
export $(cat .env | xargs)
go run cmd/server/server.go start