From 3ef688a00d2a59fcf2fdf212a062035b26995880 Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Sun, 18 Sep 2022 16:11:22 +0200 Subject: [PATCH] feature/query-results (#12) Co-authored-by: kjuulh Reviewed-on: https://git.front.kjuulh.io/kjuulh/kraken/pulls/12 --- _examples/queries/scrape_readme/kraken.yml | 11 +++ cmd/kraken/commands/process.go | 4 +- internal/actions/action.go | 26 +++++ internal/actions/querier/ripgrep.go | 106 +++++++++++++++++++++ internal/commands/process_repos.go | 27 ++++-- internal/schema/kraken.go | 4 + roadmap.md | 3 +- 7 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 _examples/queries/scrape_readme/kraken.yml create mode 100644 internal/actions/querier/ripgrep.go diff --git a/_examples/queries/scrape_readme/kraken.yml b/_examples/queries/scrape_readme/kraken.yml new file mode 100644 index 0000000..35aa3a3 --- /dev/null +++ b/_examples/queries/scrape_readme/kraken.yml @@ -0,0 +1,11 @@ +apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1 +name: write-a-readme +select: + repositories: + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + # providers: + # - gitea: https://git.front.kjuulh.io + # organisation: "cibus" +queries: + - type: grep + query: "# README" diff --git a/cmd/kraken/commands/process.go b/cmd/kraken/commands/process.go index 1ffd16e..91bd822 100644 --- a/cmd/kraken/commands/process.go +++ b/cmd/kraken/commands/process.go @@ -22,8 +22,8 @@ func CreateKrakenProcessCmd() *cobra.Command { Path string `json:"path"` }{ Repository: "git@git.front.kjuulh.io:kjuulh/kraken.git", - Branch: "feature/docker-action", - Path: "_examples/actions/docker_action/", + Branch: "feature/query-results", + Path: "_examples/queries/scrape_readme/", }) if err != nil { panic(err) diff --git a/internal/actions/action.go b/internal/actions/action.go index bc9fdb4..b36155d 100644 --- a/internal/actions/action.go +++ b/internal/actions/action.go @@ -5,6 +5,7 @@ import ( "errors" "git.front.kjuulh.io/kjuulh/kraken/internal/actions/builders" + "git.front.kjuulh.io/kjuulh/kraken/internal/actions/querier" "git.front.kjuulh.io/kjuulh/kraken/internal/schema" "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" "go.uber.org/zap" @@ -49,3 +50,28 @@ func (a *Action) Execute(ctx context.Context, area *storage.Area) error { return nil } + +func (a *Action) Query(ctx context.Context, area *storage.Area) ([]string, bool, error) { + for _, query := range a.Schema.Queries { + switch query.Type { + case "grep": + exe, err := querier.NewRipGrep(zap.L()).Build(ctx, a.SchemaPath, query.Query) + if err != nil { + return nil, false, err + } + output, found, err := exe(ctx, area.Path) + if err != nil { + return nil, false, err + } + + zap.L().Debug("Execution done") + + return output, found, nil + + default: + return nil, false, errors.New("could not determine query type") + } + } + + return nil, false, nil +} diff --git a/internal/actions/querier/ripgrep.go b/internal/actions/querier/ripgrep.go new file mode 100644 index 0000000..99a415b --- /dev/null +++ b/internal/actions/querier/ripgrep.go @@ -0,0 +1,106 @@ +package querier + +import ( + "context" + "fmt" + "io" + "os/exec" + "strings" + + "go.uber.org/zap" + "go.uber.org/zap/zapio" +) + +type RipGrep struct { + logger *zap.Logger +} + +func NewRipGrep(logger *zap.Logger) *RipGrep { + return &RipGrep{logger: logger} +} + +type RipGrepCommand func(ctx context.Context, victimPath string) ([]string, bool, error) + +func (g *RipGrep) Build(ctx context.Context, modulePath, query string) (RipGrepCommand, error) { + g.logger.Debug("Pulling docker image", zap.String("actiondir", modulePath), zap.String("query", query)) + + pullDockerImage := "docker pull mbologna/docker-ripgrep" + g.logger.Debug("Running command", zap.String("command", pullDockerImage)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + pullDockerImage, + ) + + debugwriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer debugwriter.Close() + + cmd.Stdout = debugwriter + cmd.Stderr = debugwriter + err := cmd.Start() + if err != nil { + return nil, err + } + + err = cmd.Wait() + if err != nil { + return nil, err + } + + g.logger.Debug("Docker image pulled") + + return func(ctx context.Context, victimPath string) ([]string, bool, error) { + g.logger.Debug("Executing script", zap.String("victim", victimPath)) + + runRipGrepCmd := fmt.Sprintf("docker run --rm -v %s/:/data:ro mbologna/docker-ripgrep rg -i '%s' || true", victimPath, query) + + g.logger.Debug("Execute ripgrep query", zap.String("command", runRipGrepCmd)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + runRipGrepCmd, + ) + + runDockerWriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer runDockerWriter.Close() + + builder := &strings.Builder{} + combinedWriter := io.MultiWriter(runDockerWriter, builder) + + cmd.Stdout = combinedWriter + cmd.Stderr = combinedWriter + + err = cmd.Start() + if err != nil { + return nil, false, err + } + + err = cmd.Wait() + if err != nil { + return nil, false, err + } + + contents := strings.Split(builder.String(), "\n") + validatedOutput := make([]string, 0) + + for _, c := range contents { + if !strings.Contains(c, "WARNING:") { + validatedOutput = append(validatedOutput, c) + } + } + + found := len(validatedOutput) > 0 + + return validatedOutput, found, nil + }, nil +} diff --git a/internal/commands/process_repos.go b/internal/commands/process_repos.go index f801240..02c7c27 100644 --- a/internal/commands/process_repos.go +++ b/internal/commands/process_repos.go @@ -112,17 +112,32 @@ func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, action return err } - err = action.Execute(ctx, area) - if err != nil { - return err + if len(action.Schema.Queries) > 0 { + result, found, err := action.Query(ctx, area) + if err != nil { + return err + } + + if found { + pr.logger.Info("Query result", zap.Strings("result", result)) + // TODO: Append to real result, and return together + } } - err = pr.commit(ctx, area, repo, repoUrl) - if err != nil { - return err + if len(action.Schema.Actions) > 0 { + err = action.Execute(ctx, area) + if err != nil { + return err + } + + err = pr.commit(ctx, area, repo, repoUrl) + if err != nil { + return err + } } pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) + return nil } diff --git a/internal/schema/kraken.go b/internal/schema/kraken.go index fb7d75e..b1c76f5 100644 --- a/internal/schema/kraken.go +++ b/internal/schema/kraken.go @@ -16,6 +16,10 @@ type KrakenSchema struct { Type string `yaml:"type"` Entry string `yaml:"entry"` } `yaml:"actions"` + Queries []struct { + Type string `yaml:"type"` + Query string `yaml:"query"` + } `yaml:"queries"` } func Unmarshal(raw string) (*KrakenSchema, error) { diff --git a/roadmap.md b/roadmap.md index 5414be2..cff2a5c 100644 --- a/roadmap.md +++ b/roadmap.md @@ -22,7 +22,7 @@ - [x] Allow instantiation of actions, kraken template repo etc. - [x] Implement docker action - [x] Create pr for gitea provider -- [ ] Providing query results +- [x] Providing query results - [ ] Create CLI to trigger action ### Not in scope @@ -34,6 +34,7 @@ - [ ] Make configurable gpg keyset - [ ] Make configurable git provider - [ ] Create templating function +- [ ] Add way to see progress of runners ## Version 1.x