From 56b0ca3f0bfc27cdbefe5c0be5fe5ff98a724576 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Sun, 18 Sep 2022 15:20:15 +0200 Subject: [PATCH 1/5] Added example --- _examples/queries/scrape_readme/kraken.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 _examples/queries/scrape_readme/kraken.yml diff --git a/_examples/queries/scrape_readme/kraken.yml b/_examples/queries/scrape_readme/kraken.yml new file mode 100644 index 0000000..35aa3a3 --- /dev/null +++ b/_examples/queries/scrape_readme/kraken.yml @@ -0,0 +1,11 @@ +apiVersion: git.front.kjuulh.io/kjuulh/kraken/blob/main/schema/v1 +name: write-a-readme +select: + repositories: + - git@git.front.kjuulh.io:kjuulh/kraken-test.git + # providers: + # - gitea: https://git.front.kjuulh.io + # organisation: "cibus" +queries: + - type: grep + query: "# README" -- 2.45.2 From 7a27e7287671a01cb2a9f39eb77f09442a35b70f Mon Sep 17 00:00:00 2001 From: kjuulh Date: Sun, 18 Sep 2022 15:48:58 +0200 Subject: [PATCH 2/5] with queryier --- cmd/kraken/commands/process.go | 4 +- internal/actions/action.go | 26 ++++++++ internal/actions/querier/ripgrep.go | 95 +++++++++++++++++++++++++++++ internal/commands/process_repos.go | 24 ++++++-- internal/schema/kraken.go | 4 ++ roadmap.md | 1 + 6 files changed, 146 insertions(+), 8 deletions(-) create mode 100644 internal/actions/querier/ripgrep.go diff --git a/cmd/kraken/commands/process.go b/cmd/kraken/commands/process.go index 1ffd16e..83aa0eb 100644 --- a/cmd/kraken/commands/process.go +++ b/cmd/kraken/commands/process.go @@ -22,8 +22,8 @@ func CreateKrakenProcessCmd() *cobra.Command { Path string `json:"path"` }{ Repository: "git@git.front.kjuulh.io:kjuulh/kraken.git", - Branch: "feature/docker-action", - Path: "_examples/actions/docker_action/", + Branch: "feature/query-results", + Path: "_examples/queries/scrabe_readme/", }) if err != nil { panic(err) diff --git a/internal/actions/action.go b/internal/actions/action.go index bc9fdb4..a535ba0 100644 --- a/internal/actions/action.go +++ b/internal/actions/action.go @@ -5,6 +5,7 @@ import ( "errors" "git.front.kjuulh.io/kjuulh/kraken/internal/actions/builders" + "git.front.kjuulh.io/kjuulh/kraken/internal/actions/querier" "git.front.kjuulh.io/kjuulh/kraken/internal/schema" "git.front.kjuulh.io/kjuulh/kraken/internal/services/storage" "go.uber.org/zap" @@ -49,3 +50,28 @@ func (a *Action) Execute(ctx context.Context, area *storage.Area) error { return nil } + +func (a *Action) Query(ctx context.Context, area *storage.Area) ([]string, error) { + for _, query := range a.Schema.Queries { + switch query.Type { + case "grep": + exe, err := querier.NewRipGrep(zap.L()).Build(ctx, a.SchemaPath, query.Query) + if err != nil { + return nil, err + } + output, err := exe(ctx, area.Path) + if err != nil { + return nil, err + } + + zap.L().Debug("Execution done") + + return output, nil + + default: + return nil, errors.New("could not determine query type") + } + } + + return nil, nil +} diff --git a/internal/actions/querier/ripgrep.go b/internal/actions/querier/ripgrep.go new file mode 100644 index 0000000..671d6d5 --- /dev/null +++ b/internal/actions/querier/ripgrep.go @@ -0,0 +1,95 @@ +package querier + +import ( + "context" + "fmt" + "io" + "os/exec" + "strings" + + "go.uber.org/zap" + "go.uber.org/zap/zapio" +) + +type RipGrep struct { + logger *zap.Logger +} + +func NewRipGrep(logger *zap.Logger) *RipGrep { + return &RipGrep{logger: logger} +} + +type RipGrepCommand func(ctx context.Context, victimPath string) ([]string, error) + +func (g *RipGrep) Build(ctx context.Context, modulePath, query string) (RipGrepCommand, error) { + g.logger.Debug("Pulling docker image", zap.String("actiondir", modulePath), zap.String("query", query)) + + pullDockerImage := "docker pull mbologna/docker-ripgrep" + g.logger.Debug("Running command", zap.String("command", pullDockerImage)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + pullDockerImage, + ) + + debugwriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer debugwriter.Close() + + cmd.Stdout = debugwriter + cmd.Stderr = debugwriter + err := cmd.Start() + if err != nil { + return nil, err + } + + err = cmd.Wait() + if err != nil { + return nil, err + } + + g.logger.Debug("Docker image pulled") + + return func(ctx context.Context, victimPath string) ([]string, error) { + g.logger.Debug("Executing script", zap.String("victim", victimPath)) + + cmd := exec.CommandContext( + ctx, + "/bin/bash", + "-c", + fmt.Sprintf("docker run --rm -v %s/:/data mbologna/docker-ripgrep rg -i %s", victimPath, query), + ) + + runDockerWriter := &zapio.Writer{ + Log: g.logger, + Level: zap.DebugLevel, + } + defer runDockerWriter.Close() + + builder := &strings.Builder{} + + combinedWriter := io.MultiWriter(runDockerWriter, builder) + + cmd.Stdout = combinedWriter + cmd.Stderr = runDockerWriter + + err = cmd.Start() + if err != nil { + return nil, err + } + + err = cmd.Wait() + + if err != nil { + return nil, err + } + + contents := strings.Split(builder.String(), "\n") + + return contents, nil + }, nil +} diff --git a/internal/commands/process_repos.go b/internal/commands/process_repos.go index f801240..cbf6f6b 100644 --- a/internal/commands/process_repos.go +++ b/internal/commands/process_repos.go @@ -112,17 +112,29 @@ func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, action return err } - err = action.Execute(ctx, area) - if err != nil { - return err + if len(action.Schema.Queries) > 0 { + result, err := action.Query(ctx, area) + if err != nil { + return err + } + + pr.logger.Info("Query result", zap.Strings("result", result)) } - err = pr.commit(ctx, area, repo, repoUrl) - if err != nil { - return err + if len(action.Schema.Actions) > 0 { + err = action.Execute(ctx, area) + if err != nil { + return err + } + + err = pr.commit(ctx, area, repo, repoUrl) + if err != nil { + return err + } } pr.logger.Debug("processing done", zap.String("path", area.Path), zap.String("repoUrl", repoUrl)) + return nil } diff --git a/internal/schema/kraken.go b/internal/schema/kraken.go index fb7d75e..b1c76f5 100644 --- a/internal/schema/kraken.go +++ b/internal/schema/kraken.go @@ -16,6 +16,10 @@ type KrakenSchema struct { Type string `yaml:"type"` Entry string `yaml:"entry"` } `yaml:"actions"` + Queries []struct { + Type string `yaml:"type"` + Query string `yaml:"query"` + } `yaml:"queries"` } func Unmarshal(raw string) (*KrakenSchema, error) { diff --git a/roadmap.md b/roadmap.md index 5414be2..59573fd 100644 --- a/roadmap.md +++ b/roadmap.md @@ -34,6 +34,7 @@ - [ ] Make configurable gpg keyset - [ ] Make configurable git provider - [ ] Create templating function +- [ ] Add way to see progress of runners ## Version 1.x -- 2.45.2 From cf2418c72b4fe419fec2756db9cdd28eac52e9aa Mon Sep 17 00:00:00 2001 From: kjuulh Date: Sun, 18 Sep 2022 15:49:29 +0200 Subject: [PATCH 3/5] with correct path --- cmd/kraken/commands/process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/kraken/commands/process.go b/cmd/kraken/commands/process.go index 83aa0eb..91bd822 100644 --- a/cmd/kraken/commands/process.go +++ b/cmd/kraken/commands/process.go @@ -23,7 +23,7 @@ func CreateKrakenProcessCmd() *cobra.Command { }{ Repository: "git@git.front.kjuulh.io:kjuulh/kraken.git", Branch: "feature/query-results", - Path: "_examples/queries/scrabe_readme/", + Path: "_examples/queries/scrape_readme/", }) if err != nil { panic(err) -- 2.45.2 From e35672c26259b5cef5dad4df80ecf9282a7c772f Mon Sep 17 00:00:00 2001 From: kjuulh Date: Sun, 18 Sep 2022 16:09:54 +0200 Subject: [PATCH 4/5] we can now query --- internal/actions/action.go | 14 +++++++------- internal/actions/querier/ripgrep.go | 29 ++++++++++++++++++++--------- internal/commands/process_repos.go | 7 +++++-- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/internal/actions/action.go b/internal/actions/action.go index a535ba0..b36155d 100644 --- a/internal/actions/action.go +++ b/internal/actions/action.go @@ -51,27 +51,27 @@ func (a *Action) Execute(ctx context.Context, area *storage.Area) error { return nil } -func (a *Action) Query(ctx context.Context, area *storage.Area) ([]string, error) { +func (a *Action) Query(ctx context.Context, area *storage.Area) ([]string, bool, error) { for _, query := range a.Schema.Queries { switch query.Type { case "grep": exe, err := querier.NewRipGrep(zap.L()).Build(ctx, a.SchemaPath, query.Query) if err != nil { - return nil, err + return nil, false, err } - output, err := exe(ctx, area.Path) + output, found, err := exe(ctx, area.Path) if err != nil { - return nil, err + return nil, false, err } zap.L().Debug("Execution done") - return output, nil + return output, found, nil default: - return nil, errors.New("could not determine query type") + return nil, false, errors.New("could not determine query type") } } - return nil, nil + return nil, false, nil } diff --git a/internal/actions/querier/ripgrep.go b/internal/actions/querier/ripgrep.go index 671d6d5..99a415b 100644 --- a/internal/actions/querier/ripgrep.go +++ b/internal/actions/querier/ripgrep.go @@ -19,7 +19,7 @@ func NewRipGrep(logger *zap.Logger) *RipGrep { return &RipGrep{logger: logger} } -type RipGrepCommand func(ctx context.Context, victimPath string) ([]string, error) +type RipGrepCommand func(ctx context.Context, victimPath string) ([]string, bool, error) func (g *RipGrep) Build(ctx context.Context, modulePath, query string) (RipGrepCommand, error) { g.logger.Debug("Pulling docker image", zap.String("actiondir", modulePath), zap.String("query", query)) @@ -54,14 +54,18 @@ func (g *RipGrep) Build(ctx context.Context, modulePath, query string) (RipGrepC g.logger.Debug("Docker image pulled") - return func(ctx context.Context, victimPath string) ([]string, error) { + return func(ctx context.Context, victimPath string) ([]string, bool, error) { g.logger.Debug("Executing script", zap.String("victim", victimPath)) + runRipGrepCmd := fmt.Sprintf("docker run --rm -v %s/:/data:ro mbologna/docker-ripgrep rg -i '%s' || true", victimPath, query) + + g.logger.Debug("Execute ripgrep query", zap.String("command", runRipGrepCmd)) + cmd := exec.CommandContext( ctx, "/bin/bash", "-c", - fmt.Sprintf("docker run --rm -v %s/:/data mbologna/docker-ripgrep rg -i %s", victimPath, query), + runRipGrepCmd, ) runDockerWriter := &zapio.Writer{ @@ -71,25 +75,32 @@ func (g *RipGrep) Build(ctx context.Context, modulePath, query string) (RipGrepC defer runDockerWriter.Close() builder := &strings.Builder{} - combinedWriter := io.MultiWriter(runDockerWriter, builder) cmd.Stdout = combinedWriter - cmd.Stderr = runDockerWriter + cmd.Stderr = combinedWriter err = cmd.Start() if err != nil { - return nil, err + return nil, false, err } err = cmd.Wait() - if err != nil { - return nil, err + return nil, false, err } contents := strings.Split(builder.String(), "\n") + validatedOutput := make([]string, 0) - return contents, nil + for _, c := range contents { + if !strings.Contains(c, "WARNING:") { + validatedOutput = append(validatedOutput, c) + } + } + + found := len(validatedOutput) > 0 + + return validatedOutput, found, nil }, nil } diff --git a/internal/commands/process_repos.go b/internal/commands/process_repos.go index cbf6f6b..02c7c27 100644 --- a/internal/commands/process_repos.go +++ b/internal/commands/process_repos.go @@ -113,12 +113,15 @@ func (pr *ProcessRepos) processRepo(ctx context.Context, repoUrl string, action } if len(action.Schema.Queries) > 0 { - result, err := action.Query(ctx, area) + result, found, err := action.Query(ctx, area) if err != nil { return err } - pr.logger.Info("Query result", zap.Strings("result", result)) + if found { + pr.logger.Info("Query result", zap.Strings("result", result)) + // TODO: Append to real result, and return together + } } if len(action.Schema.Actions) > 0 { -- 2.45.2 From f081d813a19aec0c6aef87f935671acf378298ea Mon Sep 17 00:00:00 2001 From: kjuulh Date: Sun, 18 Sep 2022 16:10:49 +0200 Subject: [PATCH 5/5] with query results --- roadmap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roadmap.md b/roadmap.md index 59573fd..cff2a5c 100644 --- a/roadmap.md +++ b/roadmap.md @@ -22,7 +22,7 @@ - [x] Allow instantiation of actions, kraken template repo etc. - [x] Implement docker action - [x] Create pr for gitea provider -- [ ] Providing query results +- [x] Providing query results - [ ] Create CLI to trigger action ### Not in scope -- 2.45.2