From c7653dc09c5f4f86fa0f36d43f6851ce179ef52c Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Sun, 8 Aug 2021 07:36:33 +0200 Subject: [PATCH] Update package manager dependency parsing Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/arg.go | 58 +++++++++ cmd/dagger/cmd/mod/arg_test.go | 100 +++++++++++++++ cmd/dagger/cmd/mod/file.go | 219 +++++++++++++------------------- cmd/dagger/cmd/mod/file_test.go | 85 ------------- cmd/dagger/cmd/mod/get.go | 36 +++--- cmd/dagger/cmd/mod/repo.go | 4 +- cmd/dagger/cmd/mod/repo_test.go | 32 ++--- 7 files changed, 282 insertions(+), 252 deletions(-) create mode 100644 cmd/dagger/cmd/mod/arg.go create mode 100644 cmd/dagger/cmd/mod/arg_test.go diff --git a/cmd/dagger/cmd/mod/arg.go b/cmd/dagger/cmd/mod/arg.go new file mode 100644 index 00000000..8ab9e0d2 --- /dev/null +++ b/cmd/dagger/cmd/mod/arg.go @@ -0,0 +1,58 @@ +package mod + +import ( + "fmt" + "path" + "regexp" + "strings" +) + +func parseArgument(arg string) (*require, error) { + if strings.HasPrefix(arg, "github.com") { + return parseGithubRepoName(arg) + } else if strings.HasPrefix(arg, "alpha.dagger.io") { + return parseDaggerRepoName(arg) + } else { + return nil, fmt.Errorf("repo name does not match suported providers") + } +} + +var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`) + +func parseGithubRepoName(arg string) (*require, error) { + repoMatches := githubRepoNameRegex.FindStringSubmatch(arg) + + if len(repoMatches) < 4 { + return nil, fmt.Errorf("issue when parsing github repo") + } + + return &require{ + prefix: "https://", + repo: repoMatches[1], + path: repoMatches[2], + version: repoMatches[3], + + cloneRepo: repoMatches[1], + clonePath: repoMatches[2], + }, nil +} + +var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`) + +func parseDaggerRepoName(arg string) (*require, error) { + repoMatches := daggerRepoNameRegex.FindStringSubmatch(arg) + + if len(repoMatches) < 3 { + return nil, fmt.Errorf("issue when parsing dagger repo") + } + + return &require{ + prefix: "https://", + repo: "alpha.dagger.io", + path: repoMatches[1], + version: repoMatches[2], + + cloneRepo: "github.com/dagger/dagger", + clonePath: path.Join("/stdlib", repoMatches[1]), + }, nil +} diff --git a/cmd/dagger/cmd/mod/arg_test.go b/cmd/dagger/cmd/mod/arg_test.go new file mode 100644 index 00000000..4aa1d3fc --- /dev/null +++ b/cmd/dagger/cmd/mod/arg_test.go @@ -0,0 +1,100 @@ +package mod + +import "testing" + +func TestParseArgument(t *testing.T) { + cases := []struct { + name string + in string + want *require + hasError bool + }{ + { + name: "Random", + in: "abcd/bla@:/xyz", + hasError: true, + }, + { + name: "Dagger repo", + in: "github.com/dagger/dagger", + want: &require{ + repo: "github.com/dagger/dagger", + path: "", + version: "", + }, + }, + { + name: "Dagger repo with path", + in: "github.com/dagger/dagger/stdlib", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib", + version: "", + }, + }, + { + name: "Dagger repo with longer path", + in: "github.com/dagger/dagger/stdlib/test/test", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib/test/test", + version: "", + }, + }, + { + name: "Dagger repo with path and version", + in: "github.com/dagger/dagger/stdlib@v0.1", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib", + version: "v0.1", + }, + }, + { + name: "Dagger repo with longer path and version", + in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib/test/test", + version: "v0.0.1", + }, + }, + { + name: "Alpha Dagger repo", + in: "alpha.dagger.io/gcp/gke@v0.1.0-alpha.20", + want: &require{ + repo: "alpha.dagger.io", + path: "/gcp/gke", + version: "v0.1.0-alpha.20", + + cloneRepo: "github.com/dagger/dagger", + clonePath: "/stdlib/gcp/gke", + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := parseArgument(c.in) + if err != nil && c.hasError { + return + } + + if err != nil { + t.Fatal(err) + } + + if got.repo != c.want.repo { + t.Errorf("repos differ: want %s, got %s", c.want.repo, got.repo) + } + + if got.path != c.want.path { + t.Errorf("paths differ: want %s, got %s", c.want.path, got.path) + } + + if got.version != c.want.version { + t.Errorf("versions differ: want %s, got %s", c.want.version, got.version) + } + }) + } +} diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index 6debd3d1..9f3d3ab4 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -12,7 +12,9 @@ import ( "strings" ) -const FilePath = "./cue.mod/dagger.mod.cue" +const filePath = "./cue.mod/dagger.mod.cue" +const destBasePath = "./cue.mod/pkg" +const tmpBasePath = "./cue.mod/tmp" // A file is the parsed, interpreted form of a cue.mod file. type file struct { @@ -20,6 +22,84 @@ type file struct { require []*require } +func readModFile(workspacePath string) (*file, error) { + f, err := os.Open(path.Join(workspacePath, filePath)) + if err != nil { + return nil, err + } + + modFile, err := read(f) + if err != nil { + return nil, err + } + + return modFile, nil +} + +func read(f io.Reader) (*file, error) { + b, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + lines, err := nonEmptyLines(b) + if err != nil { + return nil, err + } + + if len(lines) == 0 { + return nil, fmt.Errorf("mod file is empty, missing module name") + } + + var module string + if split := strings.Split(lines[0], " "); len(split) > 1 { + module = strings.Trim(split[1], "\"") + } + + var requires []*require + for i := 1; i < len(lines); i++ { + split := strings.Split(lines[i], " ") + r, err := parseArgument(split[0]) + if err != nil { + return nil, err + } + + r.version = split[1] + + requires = append(requires, r) + } + + return &file{ + module: module, + require: requires, + }, nil +} + +var spaceRgx = regexp.MustCompile(`\s+`) + +func nonEmptyLines(b []byte) ([]string, error) { + s := strings.ReplaceAll(string(b), "\r\n", "\n") + split := strings.Split(s, "\n") + + lines := make([]string, 0, len(split)) + for _, l := range split { + trimmed := strings.TrimSpace(l) + if trimmed == "" { + continue + } + + trimmed = spaceRgx.ReplaceAllString(trimmed, " ") + + lines = append(lines, trimmed) + } + + return lines, nil +} + +func writeModFile(workspacePath string, f *file) error { + return ioutil.WriteFile(path.Join(workspacePath, filePath), f.contents().Bytes(), 0600) +} + func (f *file) contents() *bytes.Buffer { var b bytes.Buffer @@ -45,148 +125,20 @@ type require struct { repo string path string version string + + cloneRepo string + clonePath string } func (r *require) cloneURL() string { - return fmt.Sprintf("%s%s", r.prefix, r.repo) + return fmt.Sprintf("%s%s", r.prefix, r.cloneRepo) } func (r *require) fullPath() string { return path.Join(r.repo, r.path) } -func read(f io.Reader) (*file, error) { - b, err := ioutil.ReadAll(f) - if err != nil { - return nil, err - } - - lines, err := nonEmptyLines(b) - if err != nil { - return nil, err - } - - if len(lines) == 0 { - return nil, fmt.Errorf("mod file is empty") - } - - var module string - if split := strings.Split(lines[0], " "); len(split) > 1 { - module = strings.Trim(split[1], "\"") - } - - var requires []*require - for i := 1; i < len(lines); i++ { - split := strings.Split(lines[i], " ") - r, err := parseArgument(split[0]) - if err != nil { - return nil, err - } - - r.version = split[1] - - requires = append(requires, r) - } - - return &file{ - module: module, - require: requires, - }, nil -} - -func nonEmptyLines(b []byte) ([]string, error) { - s := strings.ReplaceAll(string(b), "\r\n", "\n") - split := strings.Split(s, "\n") - - spaceRgx, err := regexp.Compile(`\s+`) - if err != nil { - return nil, err - } - - var lines []string - for _, l := range split { - trimmed := strings.TrimSpace(l) - if trimmed == "" { - continue - } - - trimmed = spaceRgx.ReplaceAllString(trimmed, " ") - - lines = append(lines, trimmed) - } - - return lines, nil -} - -func parseArgument(arg string) (*require, error) { - if strings.HasPrefix(arg, "alpha.dagger.io") { - arg = strings.Replace(arg, "alpha.dagger.io", "github.com/dagger/dagger/stdlib", 1) - } - - name, suffix, err := parseGithubRepoName(arg) - if err != nil { - return nil, err - } - - repoPath, version := parseGithubRepoVersion(suffix) - - return &require{ - prefix: "https://", - repo: name, - path: repoPath, - version: version, - }, nil -} - -func parseGithubRepoName(arg string) (string, string, error) { - repoRegex, err := regexp.Compile("(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)(.*)") - if err != nil { - return "", "", err - } - - repoMatches := repoRegex.FindStringSubmatch(arg) - - if len(repoMatches) == 0 { - return "", "", fmt.Errorf("repo name does not match suported providers") - } - - // returns 2 elements: repo name and path+version - return repoMatches[1], repoMatches[2], nil -} - -func parseGithubRepoVersion(repoSuffix string) (string, string) { - if repoSuffix == "" { - return "", "" - } - - i := strings.LastIndexAny(repoSuffix, "@:") - if i == -1 { - return repoSuffix, "" - } - - return repoSuffix[:i], repoSuffix[i+1:] -} - -func readModFile() (*file, error) { - f, err := os.Open(FilePath) - if err != nil { - return nil, err - } - - modFile, err := read(f) - if err != nil { - return nil, err - } - - return modFile, nil -} - -func writeModFile(f *file) error { - return ioutil.WriteFile(FilePath, f.contents().Bytes(), 0600) -} - func move(r *require, sourceRepoPath, destBasePath string) error { - fmt.Println("move") destPath := path.Join(destBasePath, r.fullPath()) if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { return err @@ -199,7 +151,6 @@ func move(r *require, sourceRepoPath, destBasePath string) error { } func replace(r *require, sourceRepoPath, destBasePath string) error { - fmt.Println("replace") if err := os.RemoveAll(path.Join(destBasePath, r.fullPath())); err != nil { return err } diff --git a/cmd/dagger/cmd/mod/file_test.go b/cmd/dagger/cmd/mod/file_test.go index f8ff9cb5..a94175f4 100644 --- a/cmd/dagger/cmd/mod/file_test.go +++ b/cmd/dagger/cmd/mod/file_test.go @@ -5,91 +5,6 @@ import ( "testing" ) -func TestParseArgument(t *testing.T) { - cases := []struct { - name string - in string - want *require - hasError bool - }{ - { - name: "Random", - in: "abcd/bla@:/xyz", - hasError: true, - }, - { - name: "Dagger repo", - in: "github.com/dagger/dagger", - want: &require{ - repo: "github.com/dagger/dagger", - path: "", - version: "", - }, - }, - { - name: "Dagger repo with path", - in: "github.com/dagger/dagger/stdlib", - want: &require{ - repo: "github.com/dagger/dagger", - path: "/stdlib", - version: "", - }, - }, - { - name: "Dagger repo with longer path", - in: "github.com/dagger/dagger/stdlib/test/test", - want: &require{ - repo: "github.com/dagger/dagger", - path: "/stdlib/test/test", - version: "", - }, - }, - { - name: "Dagger repo with path and version", - in: "github.com/dagger/dagger/stdlib@v0.1", - want: &require{ - repo: "github.com/dagger/dagger", - path: "/stdlib", - version: "v0.1", - }, - }, - { - name: "Dagger repo with longer path and version", - in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1", - want: &require{ - repo: "github.com/dagger/dagger", - path: "/stdlib/test/test", - version: "v0.0.1", - }, - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - got, err := parseArgument(c.in) - if err != nil && c.hasError { - return - } - - if err != nil { - t.Fatal(err) - } - - if got.repo != c.want.repo { - t.Errorf("repos differ: want %s, got %s", c.want.repo, got.repo) - } - - if got.path != c.want.path { - t.Errorf("paths differ: want %s, got %s", c.want.path, got.path) - } - - if got.version != c.want.version { - t.Errorf("versions differ: want %s, got %s", c.want.version, got.version) - } - }) - } -} - func TestReadFile(t *testing.T) { cases := []struct { name string diff --git a/cmd/dagger/cmd/mod/get.go b/cmd/dagger/cmd/mod/get.go index 89fd32f3..669f0e8a 100644 --- a/cmd/dagger/cmd/mod/get.go +++ b/cmd/dagger/cmd/mod/get.go @@ -10,11 +10,12 @@ import ( "github.com/spf13/viper" "go.dagger.io/dagger/cmd/dagger/cmd/common" "go.dagger.io/dagger/cmd/dagger/logger" + "go.dagger.io/dagger/telemetry" ) var getCmd = &cobra.Command{ Use: "get [packages]", - Short: "download and install packages and dependencies", + Short: "download and install dependencies", Args: cobra.MaximumNArgs(1), PreRun: func(cmd *cobra.Command, args []string) { // Fix Viper bug for duplicate flags: @@ -28,12 +29,17 @@ var getCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - doneCh := common.TrackCommand(ctx, cmd) - if len(args) == 0 { lg.Fatal().Msg("need to specify package name in command argument") } + workspace := common.CurrentWorkspace(ctx) + st := common.CurrentEnvironmentState(ctx, workspace) + doneCh := common.TrackWorkspaceCommand(ctx, cmd, workspace, st, &telemetry.Property{ + Name: "packages", + Value: args, + }) + // parse packages to install var packages []*require for _, arg := range args { @@ -47,21 +53,20 @@ var getCmd = &cobra.Command{ } // read mod file in the current dir - modFile, err := readModFile() + modFile, err := readModFile(workspace.Path) if err != nil { lg.Fatal().Err(err).Msgf("error loading module file") } // download packages - destBasePath := "./cue.mod/pkg" for _, p := range packages { - if err := processRequire(p, modFile, destBasePath); err != nil { + if err := processRequire(p, modFile); err != nil { lg.Error().Err(err).Msg("error processing package") } } // write to mod file in the current dir - if err = writeModFile(modFile); err != nil { + if err = writeModFile(workspace.Path, modFile); err != nil { lg.Error().Err(err).Msg("error writing to mod file") } @@ -69,13 +74,14 @@ var getCmd = &cobra.Command{ }, } -func processRequire(req *require, modFile *file, destBasePath string) error { - tmpRepoPath := path.Join("./cue.mod/tmp", req.repo) - if err := os.MkdirAll(tmpRepoPath, 0755); err != nil { +func processRequire(req *require, modFile *file) error { + tmpPath := path.Join(tmpBasePath, req.repo) + if err := os.MkdirAll(tmpPath, 0755); err != nil { return fmt.Errorf("error creating tmp dir for cloning package") } + defer os.RemoveAll(tmpPath) - r, err := clone(req, tmpRepoPath) + r, err := clone(req, tmpPath) if err != nil { return fmt.Errorf("error downloading package %s: %w", req, err) } @@ -84,11 +90,11 @@ func processRequire(req *require, modFile *file, destBasePath string) error { // requirement is new, so we should move the files and add it to the module.cue if existing == nil { - if err := move(req, tmpRepoPath, destBasePath); err != nil { + if err := move(req, tmpPath, destBasePath); err != nil { return err } modFile.require = append(modFile.require, req) - return os.RemoveAll(tmpRepoPath) + return nil } c, err := compareVersions(existing.version, req.version) @@ -98,7 +104,7 @@ func processRequire(req *require, modFile *file, destBasePath string) error { // the existing requirement is newer so we skip installation if c > 0 { - return os.RemoveAll(tmpRepoPath) + return nil } // the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing @@ -108,7 +114,7 @@ func processRequire(req *require, modFile *file, destBasePath string) error { return err } - return replace(req, tmpRepoPath, destBasePath) + return replace(req, tmpPath, destBasePath) } func compareVersions(reqV1, reqV2 string) (int, error) { diff --git a/cmd/dagger/cmd/mod/repo.go b/cmd/dagger/cmd/mod/repo.go index 646db894..f9f5e1a9 100644 --- a/cmd/dagger/cmd/mod/repo.go +++ b/cmd/dagger/cmd/mod/repo.go @@ -40,8 +40,8 @@ func clone(require *require, dir string) (*repo, error) { return nil, err } - if _, err := os.Stat(path.Join(dir, require.path, FilePath)); err != nil { - return nil, fmt.Errorf("repo does not contain %s", FilePath) + if _, err := os.Stat(path.Join(dir, require.clonePath, filePath)); err != nil { + return nil, fmt.Errorf("repo does not contain %s", filePath) } return rr, nil diff --git a/cmd/dagger/cmd/mod/repo_test.go b/cmd/dagger/cmd/mod/repo_test.go index 52ec18c0..b4210aef 100644 --- a/cmd/dagger/cmd/mod/repo_test.go +++ b/cmd/dagger/cmd/mod/repo_test.go @@ -14,28 +14,28 @@ func TestClone(t *testing.T) { { name: "resolving shorter hash version", require: require{ - prefix: "https://", - repo: "github.com/tjovicic/gcpcloudrun-cue", - path: "", - version: "d530f2ea2099", + prefix: "https://", + cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", + clonePath: "", + version: "d530f2ea2099", }, }, { name: "resolving branch name", require: require{ - prefix: "https://", - repo: "github.com/tjovicic/gcpcloudrun-cue", - path: "", - version: "main", + prefix: "https://", + cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", + clonePath: "", + version: "main", }, }, { name: "resolving tag", require: require{ - prefix: "https://", - repo: "github.com/tjovicic/gcpcloudrun-cue", - path: "", - version: "v0.3", + prefix: "https://", + cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", + clonePath: "", + version: "v0.3", }, }, } @@ -65,10 +65,10 @@ func TestListTags(t *testing.T) { defer os.Remove(tmpDir) r, err := clone(&require{ - prefix: "https://", - repo: "github.com/tjovicic/gcpcloudrun-cue", - path: "", - version: "", + prefix: "https://", + cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", + clonePath: "", + version: "", }, tmpDir) if err != nil { t.Error(err)