feature/query-results #12

Merged
kjuulh merged 5 commits from feature/query-results into v0.1 2022-09-18 16:11:23 +02:00
7 changed files with 172 additions and 9 deletions

View File

@ -0,0 +1,11 @@
apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1
name: write-a-readme
select:
repositories:
- git@git.front.kjuulh.io:kjuulh/kraken-test.git
# providers:
# - gitea: https://git.front.kjuulh.io
# organisation: "cibus"
queries:
- type: grep
query: "# README"

View File

@ -22,8 +22,8 @@ func CreateKrakenProcessCmd() *cobra.Command {
Path string `json:"path"` Path string `json:"path"`
}{ }{
Repository: "git@git.front.kjuulh.io:kjuulh/kraken.git", Repository: "git@git.front.kjuulh.io:kjuulh/kraken.git",
Branch: "feature/docker-action", Branch: "feature/query-results",
Path: "_examples/actions/docker_action/", Path: "_examples/queries/scrape_readme/",
}) })
if err != nil { if err != nil {
panic(err) panic(err)

View File

@ -5,6 +5,7 @@ import (
"errors" "errors"
"git.front.kjuulh.io/kjuulh/kraken/internal/actions/builders" "git.front.kjuulh.io/kjuulh/kraken/internal/actions/builders"
"git.front.kjuulh.io/kjuulh/kraken/internal/actions/querier"
"git.front.kjuulh.io/kjuulh/kraken/internal/schema" "git.front.kjuulh.io/kjuulh/kraken/internal/schema"
"git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage"
"go.uber.org/zap" "go.uber.org/zap"
@ -49,3 +50,28 @@ func (a *Action) Execute(ctx context.Context, area *storage.Area) error {
return nil return nil
} }
func (a *Action) Query(ctx context.Context, area *storage.Area) ([]string, bool, error) {
for _, query := range a.Schema.Queries {
switch query.Type {
case "grep":
exe, err := querier.NewRipGrep(zap.L()).Build(ctx, a.SchemaPath, query.Query)
if err != nil {
return nil, false, err
}
output, found, err := exe(ctx, area.Path)
if err != nil {
return nil, false, err
}
zap.L().Debug("Execution done")
return output, found, nil
default:
return nil, false, errors.New("could not determine query type")
}
}
return nil, false, nil
}

View File

@ -0,0 +1,106 @@
package querier
import (
"context"
"fmt"
"io"
"os/exec"
"strings"
"go.uber.org/zap"
"go.uber.org/zap/zapio"
)
type RipGrep struct {
logger *zap.Logger
}
func NewRipGrep(logger *zap.Logger) *RipGrep {
return &RipGrep{logger: logger}
}
type RipGrepCommand func(ctx context.Context, victimPath string) ([]string, bool, error)
func (g *RipGrep) Build(ctx context.Context, modulePath, query string) (RipGrepCommand, error) {
g.logger.Debug("Pulling docker image", zap.String("actiondir", modulePath), zap.String("query", query))
pullDockerImage := "docker pull mbologna/docker-ripgrep"
g.logger.Debug("Running command", zap.String("command", pullDockerImage))
cmd := exec.CommandContext(
ctx,
"/bin/bash",
"-c",
pullDockerImage,
)
debugwriter := &zapio.Writer{
Log: g.logger,
Level: zap.DebugLevel,
}
defer debugwriter.Close()
cmd.Stdout = debugwriter
cmd.Stderr = debugwriter
err := cmd.Start()
if err != nil {
return nil, err
}
err = cmd.Wait()
if err != nil {
return nil, err
}
g.logger.Debug("Docker image pulled")
return func(ctx context.Context, victimPath string) ([]string, bool, error) {
g.logger.Debug("Executing script", zap.String("victim", victimPath))
runRipGrepCmd := fmt.Sprintf("docker run --rm -v %s/:/data:ro mbologna/docker-ripgrep rg -i '%s' || true", victimPath, query)
g.logger.Debug("Execute ripgrep query", zap.String("command", runRipGrepCmd))
cmd := exec.CommandContext(
ctx,
"/bin/bash",
"-c",
runRipGrepCmd,
)
runDockerWriter := &zapio.Writer{
Log: g.logger,
Level: zap.DebugLevel,
}
defer runDockerWriter.Close()
builder := &strings.Builder{}
combinedWriter := io.MultiWriter(runDockerWriter, builder)
cmd.Stdout = combinedWriter
cmd.Stderr = combinedWriter
err = cmd.Start()
if err != nil {
return nil, false, err
}
err = cmd.Wait()
if err != nil {
return nil, false, err
}
contents := strings.Split(builder.String(), "\n")
validatedOutput := make([]string, 0)
for _, c := range contents {
if !strings.Contains(c, "WARNING:") {
validatedOutput = append(validatedOutput, c)
}
}
found := len(validatedOutput) > 0
return validatedOutput, found, nil
}, nil
}

View File

@ -112,17 +112,32 @@ func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, action
return err return err
} }
err = action.Execute(ctx, area) if len(action.Schema.Queries) > 0 {
if err != nil { result, found, err := action.Query(ctx, area)
return err if err != nil {
return err
}
if found {
pr.logger.Info("Query result", zap.Strings("result", result))
// TODO: Append to real result, and return together
}
} }
err = pr.commit(ctx, area, repo, repoUrl) if len(action.Schema.Actions) > 0 {
if err != nil { err = action.Execute(ctx, area)
return err if err != nil {
return err
}
err = pr.commit(ctx, area, repo, repoUrl)
if err != nil {
return err
}
} }
pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl))
return nil return nil
} }

View File

@ -16,6 +16,10 @@ type KrakenSchema struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Entry string `yaml:"entry"` Entry string `yaml:"entry"`
} `yaml:"actions"` } `yaml:"actions"`
Queries []struct {
Type string `yaml:"type"`
Query string `yaml:"query"`
} `yaml:"queries"`
} }
func Unmarshal(raw string) (*KrakenSchema, error) { func Unmarshal(raw string) (*KrakenSchema, error) {

View File

@ -22,7 +22,7 @@
- [x] Allow instantiation of actions, kraken template repo etc. - [x] Allow instantiation of actions, kraken template repo etc.
- [x] Implement docker action - [x] Implement docker action
- [x] Create pr for gitea provider - [x] Create pr for gitea provider
- [ ] Providing query results - [x] Providing query results
- [ ] Create CLI to trigger action - [ ] Create CLI to trigger action
### Not in scope ### Not in scope
@ -34,6 +34,7 @@
- [ ] Make configurable gpg keyset - [ ] Make configurable gpg keyset
- [ ] Make configurable git provider - [ ] Make configurable git provider
- [ ] Create templating function - [ ] Create templating function
- [ ] Add way to see progress of runners
## Version 1.x ## Version 1.x