From 0010609f4de38153f154d0979bb3cea2fdfcc1aa Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Fri, 30 Jul 2021 08:02:03 +0200 Subject: [PATCH 01/15] First version of package manager Signed-off-by: Tihomir Jovicic --- client/client.go | 2 +- cmd/dagger/cmd/mod/file.go | 205 ++++++++++++++++++++++++++++++++ cmd/dagger/cmd/mod/file_test.go | 152 +++++++++++++++++++++++ cmd/dagger/cmd/mod/get.go | 135 +++++++++++++++++++++ cmd/dagger/cmd/mod/get_test.go | 7 ++ cmd/dagger/cmd/mod/repo.go | 104 ++++++++++++++++ cmd/dagger/cmd/mod/repo_test.go | 85 +++++++++++++ cmd/dagger/cmd/mod/root.go | 17 +++ cmd/dagger/cmd/root.go | 2 + util/buildkitd/buildkitd.go | 14 ++- 10 files changed, 716 insertions(+), 7 deletions(-) create mode 100644 cmd/dagger/cmd/mod/file.go create mode 100644 cmd/dagger/cmd/mod/file_test.go create mode 100644 cmd/dagger/cmd/mod/get.go create mode 100644 cmd/dagger/cmd/mod/get_test.go create mode 100644 cmd/dagger/cmd/mod/repo.go create mode 100644 cmd/dagger/cmd/mod/repo_test.go create mode 100644 cmd/dagger/cmd/mod/root.go diff --git a/client/client.go b/client/client.go index 8267849e..965b7e8e 100644 --- a/client/client.go +++ b/client/client.go @@ -31,7 +31,7 @@ import ( "go.dagger.io/dagger/state" ) -// A dagger client +// Client is a dagger client type Client struct { c *bk.Client noCache bool diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go new file mode 100644 index 00000000..29474951 --- /dev/null +++ b/cmd/dagger/cmd/mod/file.go @@ -0,0 +1,205 @@ +package mod + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "regexp" + "strings" +) + +// A file is the parsed, interpreted form of a cue.mod file. +type file struct { + module string + require []*require +} + +func (f *file) contents() *bytes.Buffer { + var b bytes.Buffer + + b.WriteString(fmt.Sprintf("module: %s\n\n", f.module)) + for _, r := range f.require { + b.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version)) + } + + return &b +} + +func (f *file) search(r *require) *require { + for _, existing := range f.require { + if existing.fullPath() == r.fullPath() { + return existing + } + } + return nil +} + +type require struct { + prefix string + repo string + path string + version string +} + +func (r *require) cloneUrl() string { + return fmt.Sprintf("%s%s", r.prefix, r.repo) +} + +func (r *require) fullPath() string { + return path.Join(r.repo, r.path) +} + +func read(f io.Reader) (*file, error) { + b, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + lines, err := nonEmptyLines(b) + if err != nil { + return nil, err + } + + var module string + if split := strings.Split(lines[0], " "); len(split) > 1 { + module = strings.Trim(split[1], "\"") + } + + var requires []*require + for i := 1; i < len(lines); i++ { + split := strings.Split(lines[i], " ") + r, err := parseArgument(split[0]) + if err != nil { + return nil, err + } + + r.version = split[1] + + requires = append(requires, r) + } + + return &file{ + module: module, + require: requires, + }, nil +} + +func nonEmptyLines(b []byte) ([]string, error) { + s := strings.ReplaceAll(string(b), "\r\n", "\n") + split := strings.Split(s, "\n") + + spaceRgx, err := regexp.Compile(`\s+`) + if err != nil { + return nil, err + } + + var lines []string + for _, l := range split { + trimmed := strings.TrimSpace(l) + if trimmed == "" { + continue + } + + trimmed = spaceRgx.ReplaceAllString(trimmed, " ") + + lines = append(lines, trimmed) + } + + return lines, nil +} + +func parseArgument(arg string) (*require, error) { + if strings.HasPrefix(arg, "alpha.dagger.io") { + arg = strings.Replace(arg, "alpha.dagger.io", "github.com/dagger/dagger/stdlib", 1) + } + + name, suffix, err := parseGithubRepoName(arg) + if err != nil { + return nil, err + } + + repoPath, version, err := parseGithubRepoVersion(suffix) + if err != nil { + return nil, err + } + + return &require{ + prefix: "https://", + repo: name, + path: repoPath, + version: version, + }, nil +} + +func parseGithubRepoName(arg string) (string, string, error) { + repoRegex, err := regexp.Compile("(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)(.*)") + if err != nil { + return "", "", err + } + + repoMatches := repoRegex.FindStringSubmatch(arg) + + if len(repoMatches) == 0 { + return "", "", fmt.Errorf("repo name does not match suported providers") + } + + // returns 2 elements: repo name and path+version + return repoMatches[1], repoMatches[2], nil +} + +func parseGithubRepoVersion(repoSuffix string) (string, string, error) { + if repoSuffix == "" { + return "", "", nil + } + + i := strings.LastIndexAny(repoSuffix, "@:") + if i == -1 { + return repoSuffix, "", nil + } + + return repoSuffix[:i], repoSuffix[i+1:], nil +} + +func readModFile() (*file, error) { + f, err := os.Open("./cue.mod/module.cue") + if err != nil { + return nil, err + } + + modFile, err := read(f) + if err != nil { + return nil, err + } + + return modFile, nil +} + +func writeModFile(f *file) error { + return ioutil.WriteFile("./cue.mod/module.cue", f.contents().Bytes(), 0644) +} + +func move(r *require, sourceRepoPath, destBasePath string) error { + fmt.Println("move") + destPath := path.Join(destBasePath, r.fullPath()) + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return err + } + if err := os.Rename(path.Join(sourceRepoPath, r.path), destPath); err != nil { + return err + } + + return nil +} + +func replace(r *require, sourceRepoPath, destBasePath string) error { + fmt.Println("replace") + if err := os.RemoveAll(path.Join(destBasePath, r.fullPath())); err != nil { + return err + } + + return move(r, sourceRepoPath, destBasePath) +} diff --git a/cmd/dagger/cmd/mod/file_test.go b/cmd/dagger/cmd/mod/file_test.go new file mode 100644 index 00000000..f8ff9cb5 --- /dev/null +++ b/cmd/dagger/cmd/mod/file_test.go @@ -0,0 +1,152 @@ +package mod + +import ( + "strings" + "testing" +) + +func TestParseArgument(t *testing.T) { + cases := []struct { + name string + in string + want *require + hasError bool + }{ + { + name: "Random", + in: "abcd/bla@:/xyz", + hasError: true, + }, + { + name: "Dagger repo", + in: "github.com/dagger/dagger", + want: &require{ + repo: "github.com/dagger/dagger", + path: "", + version: "", + }, + }, + { + name: "Dagger repo with path", + in: "github.com/dagger/dagger/stdlib", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib", + version: "", + }, + }, + { + name: "Dagger repo with longer path", + in: "github.com/dagger/dagger/stdlib/test/test", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib/test/test", + version: "", + }, + }, + { + name: "Dagger repo with path and version", + in: "github.com/dagger/dagger/stdlib@v0.1", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib", + version: "v0.1", + }, + }, + { + name: "Dagger repo with longer path and version", + in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib/test/test", + version: "v0.0.1", + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := parseArgument(c.in) + if err != nil && c.hasError { + return + } + + if err != nil { + t.Fatal(err) + } + + if got.repo != c.want.repo { + t.Errorf("repos differ: want %s, got %s", c.want.repo, got.repo) + } + + if got.path != c.want.path { + t.Errorf("paths differ: want %s, got %s", c.want.path, got.path) + } + + if got.version != c.want.version { + t.Errorf("versions differ: want %s, got %s", c.want.version, got.version) + } + }) + } +} + +func TestReadFile(t *testing.T) { + cases := []struct { + name string + input string + want *file + }{ + { + name: "module file without dependencies", + input: ` + module: "alpha.dagger.io" + `, + want: &file{ + module: "alpha.dagger.io", + }, + }, + { + name: "module file with valid dependencies", + input: ` + module: "alpha.dagger.io" + + github.com/tjovicic/test xyz + github.com/bla/bla abc + `, + want: &file{ + module: "alpha.dagger.io", + require: []*require{ + { + prefix: "https://", + repo: "github.com/tjovicic/test", + path: "", + version: "xyz", + }, + { + prefix: "https://", + repo: "github.com/bla/bla", + path: "", + version: "abc", + }, + }, + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := read(strings.NewReader(c.input)) + if err != nil { + t.Error(err) + } + + if got.module != c.want.module { + t.Errorf("module names differ: want %s, got %s", c.want.module, got.module) + } + + if len(got.require) != len(c.want.require) { + t.Errorf("requires length differs: want %d, got %d", len(c.want.require), len(got.require)) + } + }) + } +} diff --git a/cmd/dagger/cmd/mod/get.go b/cmd/dagger/cmd/mod/get.go new file mode 100644 index 00000000..89fd32f3 --- /dev/null +++ b/cmd/dagger/cmd/mod/get.go @@ -0,0 +1,135 @@ +package mod + +import ( + "fmt" + "os" + "path" + + "github.com/hashicorp/go-version" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "go.dagger.io/dagger/cmd/dagger/cmd/common" + "go.dagger.io/dagger/cmd/dagger/logger" +) + +var getCmd = &cobra.Command{ + Use: "get [packages]", + Short: "download and install packages and dependencies", + Args: cobra.MaximumNArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + // Fix Viper bug for duplicate flags: + // https://github.com/spf13/viper/issues/233 + if err := viper.BindPFlags(cmd.Flags()); err != nil { + panic(err) + } + }, + + Run: func(cmd *cobra.Command, args []string) { + lg := logger.New() + ctx := lg.WithContext(cmd.Context()) + + doneCh := common.TrackCommand(ctx, cmd) + + if len(args) == 0 { + lg.Fatal().Msg("need to specify package name in command argument") + } + + // parse packages to install + var packages []*require + for _, arg := range args { + p, err := parseArgument(arg) + if err != nil { + lg.Error().Err(err).Msgf("error parsing package %s", arg) + continue + } + + packages = append(packages, p) + } + + // read mod file in the current dir + modFile, err := readModFile() + if err != nil { + lg.Fatal().Err(err).Msgf("error loading module file") + } + + // download packages + destBasePath := "./cue.mod/pkg" + for _, p := range packages { + if err := processRequire(p, modFile, destBasePath); err != nil { + lg.Error().Err(err).Msg("error processing package") + } + } + + // write to mod file in the current dir + if err = writeModFile(modFile); err != nil { + lg.Error().Err(err).Msg("error writing to mod file") + } + + <-doneCh + }, +} + +func processRequire(req *require, modFile *file, destBasePath string) error { + tmpRepoPath := path.Join("./cue.mod/tmp", req.repo) + if err := os.MkdirAll(tmpRepoPath, 0755); err != nil { + return fmt.Errorf("error creating tmp dir for cloning package") + } + + r, err := clone(req, tmpRepoPath) + if err != nil { + return fmt.Errorf("error downloading package %s: %w", req, err) + } + + existing := modFile.search(req) + + // requirement is new, so we should move the files and add it to the module.cue + if existing == nil { + if err := move(req, tmpRepoPath, destBasePath); err != nil { + return err + } + modFile.require = append(modFile.require, req) + return os.RemoveAll(tmpRepoPath) + } + + c, err := compareVersions(existing.version, req.version) + if err != nil { + return err + } + + // the existing requirement is newer so we skip installation + if c > 0 { + return os.RemoveAll(tmpRepoPath) + } + + // the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing + // requirement and replace the code in the /pkg folder + existing.version = req.version + if err = r.checkout(req.version); err != nil { + return err + } + + return replace(req, tmpRepoPath, destBasePath) +} + +func compareVersions(reqV1, reqV2 string) (int, error) { + v1, err := version.NewVersion(reqV1) + if err != nil { + return 0, err + } + + v2, err := version.NewVersion(reqV2) + if err != nil { + return 0, err + } + if v1.LessThan(v2) { + return -1, nil + } + + return 1, nil +} + +func init() { + if err := viper.BindPFlags(getCmd.Flags()); err != nil { + panic(err) + } +} diff --git a/cmd/dagger/cmd/mod/get_test.go b/cmd/dagger/cmd/mod/get_test.go new file mode 100644 index 00000000..d0b96e4e --- /dev/null +++ b/cmd/dagger/cmd/mod/get_test.go @@ -0,0 +1,7 @@ +package mod + +import "testing" + +func TestUpdateModFile(t *testing.T) { + +} diff --git a/cmd/dagger/cmd/mod/repo.go b/cmd/dagger/cmd/mod/repo.go new file mode 100644 index 00000000..4970c914 --- /dev/null +++ b/cmd/dagger/cmd/mod/repo.go @@ -0,0 +1,104 @@ +package mod + +import ( + "fmt" + "os" + "path" + "sort" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/hashicorp/go-version" +) + +type repo struct { + localPath string + contents *git.Repository +} + +func clone(require *require, dir string) (*repo, error) { + r, err := git.PlainClone(dir, false, &git.CloneOptions{ + URL: require.cloneUrl(), + }) + if err != nil { + return nil, err + } + + rr := &repo{ + localPath: dir, + contents: r, + } + + if require.version == "" { + require.version, err = rr.latestTag() + if err != nil { + return nil, err + } + } + + if err := rr.checkout(require.version); err != nil { + return nil, err + } + + if _, err := os.Stat(path.Join(dir, require.path, "cue.mod", "module.cue")); err != nil { + return nil, fmt.Errorf("repo does not contain cue.mod/module.cue") + } + + return rr, nil +} + +func (r *repo) checkout(version string) error { + h, err := r.contents.ResolveRevision(plumbing.Revision(version)) + if err != nil { + return err + } + + w, err := r.contents.Worktree() + if err != nil { + return err + } + + err = w.Checkout(&git.CheckoutOptions{ + Hash: *h, + }) + if err != nil { + return err + } + + return nil +} + +func (r *repo) listTags() ([]string, error) { + iter, err := r.contents.Tags() + if err != nil { + return nil, err + } + + var tags []string + if err := iter.ForEach(func(ref *plumbing.Reference) error { + tags = append(tags, ref.Name().Short()) + return nil + }); err != nil { + return nil, err + } + + return tags, nil +} + +func (r *repo) latestTag() (string, error) { + versionsRaw, err := r.listTags() + if err != nil { + return "", err + } + + versions := make([]*version.Version, len(versionsRaw)) + for i, raw := range versionsRaw { + v, _ := version.NewVersion(raw) + versions[i] = v + } + + // After this, the versions are properly sorted + sort.Sort(version.Collection(versions)) + + return versions[len(versions)-1].Original(), nil +} diff --git a/cmd/dagger/cmd/mod/repo_test.go b/cmd/dagger/cmd/mod/repo_test.go new file mode 100644 index 00000000..94078345 --- /dev/null +++ b/cmd/dagger/cmd/mod/repo_test.go @@ -0,0 +1,85 @@ +package mod + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestClone(t *testing.T) { + cases := []struct { + name string + require require + }{ + { + name: "resolving shorter hash version", + require: require{ + prefix: "https://", + repo: "github.com/tjovicic/gcpcloudrun-cue", + path: "", + version: "5839b7b432b8b0c", + }, + }, + { + name: "resolving branch name", + require: require{ + prefix: "https://", + repo: "github.com/tjovicic/gcpcloudrun-cue", + path: "", + version: "main", + }, + }, + { + name: "resolving tag", + require: require{ + prefix: "https://", + repo: "github.com/tjovicic/gcpcloudrun-cue", + path: "", + version: "v0.1", + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "clone") + if err != nil { + t.Fatal("error creating tmp dir") + } + + defer os.Remove(tmpDir) + + _, err = clone(&c.require, tmpDir) + if err != nil { + t.Error(err) + } + }) + } +} + +func TestListTags(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "clone") + if err != nil { + t.Fatal("error creating tmp dir") + } + defer os.Remove(tmpDir) + + r, err := clone(&require{ + prefix: "https://", + repo: "github.com/cuelang/cue", + path: "", + version: "", + }, tmpDir) + if err != nil { + t.Error(err) + } + + tags, err := r.listTags() + if err != nil { + t.Error(err) + } + + if len(tags) == 0 { + t.Errorf("could not list repo tags") + } +} diff --git a/cmd/dagger/cmd/mod/root.go b/cmd/dagger/cmd/mod/root.go new file mode 100644 index 00000000..1f12ca18 --- /dev/null +++ b/cmd/dagger/cmd/mod/root.go @@ -0,0 +1,17 @@ +package mod + +import "github.com/spf13/cobra" + + +// Cmd exposes the top-level command +var Cmd = &cobra.Command{ + Use: "mod", + Short: "Manage an environment's dependencies", +} + +func init() { + Cmd.AddCommand( + getCmd, + ) +} + diff --git a/cmd/dagger/cmd/root.go b/cmd/dagger/cmd/root.go index d3ade03b..705630c5 100644 --- a/cmd/dagger/cmd/root.go +++ b/cmd/dagger/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "go.dagger.io/dagger/cmd/dagger/cmd/mod" "os" "strings" @@ -59,6 +60,7 @@ func init() { output.Cmd, versionCmd, docCmd, + mod.Cmd, ) if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil { diff --git a/util/buildkitd/buildkitd.go b/util/buildkitd/buildkitd.go index 6a111d30..64563774 100644 --- a/util/buildkitd/buildkitd.go +++ b/util/buildkitd/buildkitd.go @@ -29,22 +29,24 @@ const ( func init() { bi, ok := debug.ReadBuildInfo() if !ok { - panic("unable to retrieve build info") + return } + for _, d := range bi.Deps { if d.Path == "github.com/moby/buildkit" { vendoredVersion = d.Version break } } - if vendoredVersion == "" { - panic("failed to solve vendored buildkit version") - } } func Start(ctx context.Context) (string, error) { lg := log.Ctx(ctx) + if vendoredVersion == "" { + return "", fmt.Errorf("vendored version is empty") + } + // Attempt to detect the current buildkit version currentVersion, err := getBuildkitVersion(ctx) if err != nil { @@ -66,7 +68,7 @@ func Start(ctx context.Context) (string, error) { Info(). Str("version", vendoredVersion). Msg("upgrading buildkit") - if err := remvoveBuildkit(ctx); err != nil { + if err := removeBuildkit(ctx); err != nil { return "", err } } else { @@ -183,7 +185,7 @@ func waitBuildkit(ctx context.Context) error { return errors.New("buildkit failed to respond") } -func remvoveBuildkit(ctx context.Context) error { +func removeBuildkit(ctx context.Context) error { lg := log. Ctx(ctx) From 92e6e693f5280abfd6e6858ea062248da514b355 Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Mon, 2 Aug 2021 09:35:33 +0200 Subject: [PATCH 02/15] fix linting issues Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/file.go | 17 +++++++---------- cmd/dagger/cmd/mod/repo.go | 2 +- cmd/dagger/cmd/mod/root.go | 2 -- cmd/dagger/cmd/root.go | 2 +- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index 29474951..33be4a68 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -45,7 +45,7 @@ type require struct { version string } -func (r *require) cloneUrl() string { +func (r *require) cloneURL() string { return fmt.Sprintf("%s%s", r.prefix, r.repo) } @@ -122,10 +122,7 @@ func parseArgument(arg string) (*require, error) { return nil, err } - repoPath, version, err := parseGithubRepoVersion(suffix) - if err != nil { - return nil, err - } + repoPath, version := parseGithubRepoVersion(suffix) return &require{ prefix: "https://", @@ -151,17 +148,17 @@ func parseGithubRepoName(arg string) (string, string, error) { return repoMatches[1], repoMatches[2], nil } -func parseGithubRepoVersion(repoSuffix string) (string, string, error) { +func parseGithubRepoVersion(repoSuffix string) (string, string) { if repoSuffix == "" { - return "", "", nil + return "", "" } i := strings.LastIndexAny(repoSuffix, "@:") if i == -1 { - return repoSuffix, "", nil + return repoSuffix, "" } - return repoSuffix[:i], repoSuffix[i+1:], nil + return repoSuffix[:i], repoSuffix[i+1:] } func readModFile() (*file, error) { @@ -179,7 +176,7 @@ func readModFile() (*file, error) { } func writeModFile(f *file) error { - return ioutil.WriteFile("./cue.mod/module.cue", f.contents().Bytes(), 0644) + return ioutil.WriteFile("./cue.mod/module.cue", f.contents().Bytes(), 0600) } func move(r *require, sourceRepoPath, destBasePath string) error { diff --git a/cmd/dagger/cmd/mod/repo.go b/cmd/dagger/cmd/mod/repo.go index 4970c914..36630001 100644 --- a/cmd/dagger/cmd/mod/repo.go +++ b/cmd/dagger/cmd/mod/repo.go @@ -18,7 +18,7 @@ type repo struct { func clone(require *require, dir string) (*repo, error) { r, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: require.cloneUrl(), + URL: require.cloneURL(), }) if err != nil { return nil, err diff --git a/cmd/dagger/cmd/mod/root.go b/cmd/dagger/cmd/mod/root.go index 1f12ca18..87d934ab 100644 --- a/cmd/dagger/cmd/mod/root.go +++ b/cmd/dagger/cmd/mod/root.go @@ -2,7 +2,6 @@ package mod import "github.com/spf13/cobra" - // Cmd exposes the top-level command var Cmd = &cobra.Command{ Use: "mod", @@ -14,4 +13,3 @@ func init() { getCmd, ) } - diff --git a/cmd/dagger/cmd/root.go b/cmd/dagger/cmd/root.go index 705630c5..988bc481 100644 --- a/cmd/dagger/cmd/root.go +++ b/cmd/dagger/cmd/root.go @@ -1,7 +1,6 @@ package cmd import ( - "go.dagger.io/dagger/cmd/dagger/cmd/mod" "os" "strings" @@ -9,6 +8,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "go.dagger.io/dagger/cmd/dagger/cmd/input" + "go.dagger.io/dagger/cmd/dagger/cmd/mod" "go.dagger.io/dagger/cmd/dagger/cmd/output" "go.dagger.io/dagger/cmd/dagger/logger" "go.dagger.io/dagger/keychain" From e5c8d6eee0ce55ed74e5a1588856d6487d5be609 Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Mon, 2 Aug 2021 11:17:42 +0200 Subject: [PATCH 03/15] replace module.cue with dagger.mod.cue Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/file.go | 6 ++++-- cmd/dagger/cmd/mod/get_test.go | 7 ------- cmd/dagger/cmd/mod/repo.go | 4 ++-- cmd/dagger/cmd/mod/repo_test.go | 4 ++-- 4 files changed, 8 insertions(+), 13 deletions(-) delete mode 100644 cmd/dagger/cmd/mod/get_test.go diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index 33be4a68..15e28a7a 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -12,6 +12,8 @@ import ( "strings" ) +const FilePath = "./cue.mod/dagger.mod.cue" + // A file is the parsed, interpreted form of a cue.mod file. type file struct { module string @@ -162,7 +164,7 @@ func parseGithubRepoVersion(repoSuffix string) (string, string) { } func readModFile() (*file, error) { - f, err := os.Open("./cue.mod/module.cue") + f, err := os.Open(FilePath) if err != nil { return nil, err } @@ -176,7 +178,7 @@ func readModFile() (*file, error) { } func writeModFile(f *file) error { - return ioutil.WriteFile("./cue.mod/module.cue", f.contents().Bytes(), 0600) + return ioutil.WriteFile(FilePath, f.contents().Bytes(), 0600) } func move(r *require, sourceRepoPath, destBasePath string) error { diff --git a/cmd/dagger/cmd/mod/get_test.go b/cmd/dagger/cmd/mod/get_test.go deleted file mode 100644 index d0b96e4e..00000000 --- a/cmd/dagger/cmd/mod/get_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package mod - -import "testing" - -func TestUpdateModFile(t *testing.T) { - -} diff --git a/cmd/dagger/cmd/mod/repo.go b/cmd/dagger/cmd/mod/repo.go index 36630001..646db894 100644 --- a/cmd/dagger/cmd/mod/repo.go +++ b/cmd/dagger/cmd/mod/repo.go @@ -40,8 +40,8 @@ func clone(require *require, dir string) (*repo, error) { return nil, err } - if _, err := os.Stat(path.Join(dir, require.path, "cue.mod", "module.cue")); err != nil { - return nil, fmt.Errorf("repo does not contain cue.mod/module.cue") + if _, err := os.Stat(path.Join(dir, require.path, FilePath)); err != nil { + return nil, fmt.Errorf("repo does not contain %s", FilePath) } return rr, nil diff --git a/cmd/dagger/cmd/mod/repo_test.go b/cmd/dagger/cmd/mod/repo_test.go index 94078345..779dac72 100644 --- a/cmd/dagger/cmd/mod/repo_test.go +++ b/cmd/dagger/cmd/mod/repo_test.go @@ -17,7 +17,7 @@ func TestClone(t *testing.T) { prefix: "https://", repo: "github.com/tjovicic/gcpcloudrun-cue", path: "", - version: "5839b7b432b8b0c", + version: "d530f2ea2099", }, }, { @@ -35,7 +35,7 @@ func TestClone(t *testing.T) { prefix: "https://", repo: "github.com/tjovicic/gcpcloudrun-cue", path: "", - version: "v0.1", + version: "v0.3", }, }, } From be913b0e908775349f6540a128df5f52a561a4fb Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Wed, 4 Aug 2021 08:20:58 +0200 Subject: [PATCH 04/15] update mod list tags tests Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/file.go | 4 ++++ cmd/dagger/cmd/mod/repo_test.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index 15e28a7a..6debd3d1 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -66,6 +66,10 @@ func read(f io.Reader) (*file, error) { return nil, err } + if len(lines) == 0 { + return nil, fmt.Errorf("mod file is empty") + } + var module string if split := strings.Split(lines[0], " "); len(split) > 1 { module = strings.Trim(split[1], "\"") diff --git a/cmd/dagger/cmd/mod/repo_test.go b/cmd/dagger/cmd/mod/repo_test.go index 779dac72..52ec18c0 100644 --- a/cmd/dagger/cmd/mod/repo_test.go +++ b/cmd/dagger/cmd/mod/repo_test.go @@ -66,7 +66,7 @@ func TestListTags(t *testing.T) { r, err := clone(&require{ prefix: "https://", - repo: "github.com/cuelang/cue", + repo: "github.com/tjovicic/gcpcloudrun-cue", path: "", version: "", }, tmpDir) From c7653dc09c5f4f86fa0f36d43f6851ce179ef52c Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Sun, 8 Aug 2021 07:36:33 +0200 Subject: [PATCH 05/15] Update package manager dependency parsing Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/arg.go | 58 +++++++++ cmd/dagger/cmd/mod/arg_test.go | 100 +++++++++++++++ cmd/dagger/cmd/mod/file.go | 219 +++++++++++++------------------- cmd/dagger/cmd/mod/file_test.go | 85 ------------- cmd/dagger/cmd/mod/get.go | 36 +++--- cmd/dagger/cmd/mod/repo.go | 4 +- cmd/dagger/cmd/mod/repo_test.go | 32 ++--- 7 files changed, 282 insertions(+), 252 deletions(-) create mode 100644 cmd/dagger/cmd/mod/arg.go create mode 100644 cmd/dagger/cmd/mod/arg_test.go diff --git a/cmd/dagger/cmd/mod/arg.go b/cmd/dagger/cmd/mod/arg.go new file mode 100644 index 00000000..8ab9e0d2 --- /dev/null +++ b/cmd/dagger/cmd/mod/arg.go @@ -0,0 +1,58 @@ +package mod + +import ( + "fmt" + "path" + "regexp" + "strings" +) + +func parseArgument(arg string) (*require, error) { + if strings.HasPrefix(arg, "github.com") { + return parseGithubRepoName(arg) + } else if strings.HasPrefix(arg, "alpha.dagger.io") { + return parseDaggerRepoName(arg) + } else { + return nil, fmt.Errorf("repo name does not match suported providers") + } +} + +var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`) + +func parseGithubRepoName(arg string) (*require, error) { + repoMatches := githubRepoNameRegex.FindStringSubmatch(arg) + + if len(repoMatches) < 4 { + return nil, fmt.Errorf("issue when parsing github repo") + } + + return &require{ + prefix: "https://", + repo: repoMatches[1], + path: repoMatches[2], + version: repoMatches[3], + + cloneRepo: repoMatches[1], + clonePath: repoMatches[2], + }, nil +} + +var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`) + +func parseDaggerRepoName(arg string) (*require, error) { + repoMatches := daggerRepoNameRegex.FindStringSubmatch(arg) + + if len(repoMatches) < 3 { + return nil, fmt.Errorf("issue when parsing dagger repo") + } + + return &require{ + prefix: "https://", + repo: "alpha.dagger.io", + path: repoMatches[1], + version: repoMatches[2], + + cloneRepo: "github.com/dagger/dagger", + clonePath: path.Join("/stdlib", repoMatches[1]), + }, nil +} diff --git a/cmd/dagger/cmd/mod/arg_test.go b/cmd/dagger/cmd/mod/arg_test.go new file mode 100644 index 00000000..4aa1d3fc --- /dev/null +++ b/cmd/dagger/cmd/mod/arg_test.go @@ -0,0 +1,100 @@ +package mod + +import "testing" + +func TestParseArgument(t *testing.T) { + cases := []struct { + name string + in string + want *require + hasError bool + }{ + { + name: "Random", + in: "abcd/bla@:/xyz", + hasError: true, + }, + { + name: "Dagger repo", + in: "github.com/dagger/dagger", + want: &require{ + repo: "github.com/dagger/dagger", + path: "", + version: "", + }, + }, + { + name: "Dagger repo with path", + in: "github.com/dagger/dagger/stdlib", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib", + version: "", + }, + }, + { + name: "Dagger repo with longer path", + in: "github.com/dagger/dagger/stdlib/test/test", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib/test/test", + version: "", + }, + }, + { + name: "Dagger repo with path and version", + in: "github.com/dagger/dagger/stdlib@v0.1", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib", + version: "v0.1", + }, + }, + { + name: "Dagger repo with longer path and version", + in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib/test/test", + version: "v0.0.1", + }, + }, + { + name: "Alpha Dagger repo", + in: "alpha.dagger.io/gcp/gke@v0.1.0-alpha.20", + want: &require{ + repo: "alpha.dagger.io", + path: "/gcp/gke", + version: "v0.1.0-alpha.20", + + cloneRepo: "github.com/dagger/dagger", + clonePath: "/stdlib/gcp/gke", + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := parseArgument(c.in) + if err != nil && c.hasError { + return + } + + if err != nil { + t.Fatal(err) + } + + if got.repo != c.want.repo { + t.Errorf("repos differ: want %s, got %s", c.want.repo, got.repo) + } + + if got.path != c.want.path { + t.Errorf("paths differ: want %s, got %s", c.want.path, got.path) + } + + if got.version != c.want.version { + t.Errorf("versions differ: want %s, got %s", c.want.version, got.version) + } + }) + } +} diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index 6debd3d1..9f3d3ab4 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -12,7 +12,9 @@ import ( "strings" ) -const FilePath = "./cue.mod/dagger.mod.cue" +const filePath = "./cue.mod/dagger.mod.cue" +const destBasePath = "./cue.mod/pkg" +const tmpBasePath = "./cue.mod/tmp" // A file is the parsed, interpreted form of a cue.mod file. type file struct { @@ -20,6 +22,84 @@ type file struct { require []*require } +func readModFile(workspacePath string) (*file, error) { + f, err := os.Open(path.Join(workspacePath, filePath)) + if err != nil { + return nil, err + } + + modFile, err := read(f) + if err != nil { + return nil, err + } + + return modFile, nil +} + +func read(f io.Reader) (*file, error) { + b, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + lines, err := nonEmptyLines(b) + if err != nil { + return nil, err + } + + if len(lines) == 0 { + return nil, fmt.Errorf("mod file is empty, missing module name") + } + + var module string + if split := strings.Split(lines[0], " "); len(split) > 1 { + module = strings.Trim(split[1], "\"") + } + + var requires []*require + for i := 1; i < len(lines); i++ { + split := strings.Split(lines[i], " ") + r, err := parseArgument(split[0]) + if err != nil { + return nil, err + } + + r.version = split[1] + + requires = append(requires, r) + } + + return &file{ + module: module, + require: requires, + }, nil +} + +var spaceRgx = regexp.MustCompile(`\s+`) + +func nonEmptyLines(b []byte) ([]string, error) { + s := strings.ReplaceAll(string(b), "\r\n", "\n") + split := strings.Split(s, "\n") + + lines := make([]string, 0, len(split)) + for _, l := range split { + trimmed := strings.TrimSpace(l) + if trimmed == "" { + continue + } + + trimmed = spaceRgx.ReplaceAllString(trimmed, " ") + + lines = append(lines, trimmed) + } + + return lines, nil +} + +func writeModFile(workspacePath string, f *file) error { + return ioutil.WriteFile(path.Join(workspacePath, filePath), f.contents().Bytes(), 0600) +} + func (f *file) contents() *bytes.Buffer { var b bytes.Buffer @@ -45,148 +125,20 @@ type require struct { repo string path string version string + + cloneRepo string + clonePath string } func (r *require) cloneURL() string { - return fmt.Sprintf("%s%s", r.prefix, r.repo) + return fmt.Sprintf("%s%s", r.prefix, r.cloneRepo) } func (r *require) fullPath() string { return path.Join(r.repo, r.path) } -func read(f io.Reader) (*file, error) { - b, err := ioutil.ReadAll(f) - if err != nil { - return nil, err - } - - lines, err := nonEmptyLines(b) - if err != nil { - return nil, err - } - - if len(lines) == 0 { - return nil, fmt.Errorf("mod file is empty") - } - - var module string - if split := strings.Split(lines[0], " "); len(split) > 1 { - module = strings.Trim(split[1], "\"") - } - - var requires []*require - for i := 1; i < len(lines); i++ { - split := strings.Split(lines[i], " ") - r, err := parseArgument(split[0]) - if err != nil { - return nil, err - } - - r.version = split[1] - - requires = append(requires, r) - } - - return &file{ - module: module, - require: requires, - }, nil -} - -func nonEmptyLines(b []byte) ([]string, error) { - s := strings.ReplaceAll(string(b), "\r\n", "\n") - split := strings.Split(s, "\n") - - spaceRgx, err := regexp.Compile(`\s+`) - if err != nil { - return nil, err - } - - var lines []string - for _, l := range split { - trimmed := strings.TrimSpace(l) - if trimmed == "" { - continue - } - - trimmed = spaceRgx.ReplaceAllString(trimmed, " ") - - lines = append(lines, trimmed) - } - - return lines, nil -} - -func parseArgument(arg string) (*require, error) { - if strings.HasPrefix(arg, "alpha.dagger.io") { - arg = strings.Replace(arg, "alpha.dagger.io", "github.com/dagger/dagger/stdlib", 1) - } - - name, suffix, err := parseGithubRepoName(arg) - if err != nil { - return nil, err - } - - repoPath, version := parseGithubRepoVersion(suffix) - - return &require{ - prefix: "https://", - repo: name, - path: repoPath, - version: version, - }, nil -} - -func parseGithubRepoName(arg string) (string, string, error) { - repoRegex, err := regexp.Compile("(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)(.*)") - if err != nil { - return "", "", err - } - - repoMatches := repoRegex.FindStringSubmatch(arg) - - if len(repoMatches) == 0 { - return "", "", fmt.Errorf("repo name does not match suported providers") - } - - // returns 2 elements: repo name and path+version - return repoMatches[1], repoMatches[2], nil -} - -func parseGithubRepoVersion(repoSuffix string) (string, string) { - if repoSuffix == "" { - return "", "" - } - - i := strings.LastIndexAny(repoSuffix, "@:") - if i == -1 { - return repoSuffix, "" - } - - return repoSuffix[:i], repoSuffix[i+1:] -} - -func readModFile() (*file, error) { - f, err := os.Open(FilePath) - if err != nil { - return nil, err - } - - modFile, err := read(f) - if err != nil { - return nil, err - } - - return modFile, nil -} - -func writeModFile(f *file) error { - return ioutil.WriteFile(FilePath, f.contents().Bytes(), 0600) -} - func move(r *require, sourceRepoPath, destBasePath string) error { - fmt.Println("move") destPath := path.Join(destBasePath, r.fullPath()) if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { return err @@ -199,7 +151,6 @@ func move(r *require, sourceRepoPath, destBasePath string) error { } func replace(r *require, sourceRepoPath, destBasePath string) error { - fmt.Println("replace") if err := os.RemoveAll(path.Join(destBasePath, r.fullPath())); err != nil { return err } diff --git a/cmd/dagger/cmd/mod/file_test.go b/cmd/dagger/cmd/mod/file_test.go index f8ff9cb5..a94175f4 100644 --- a/cmd/dagger/cmd/mod/file_test.go +++ b/cmd/dagger/cmd/mod/file_test.go @@ -5,91 +5,6 @@ import ( "testing" ) -func TestParseArgument(t *testing.T) { - cases := []struct { - name string - in string - want *require - hasError bool - }{ - { - name: "Random", - in: "abcd/bla@:/xyz", - hasError: true, - }, - { - name: "Dagger repo", - in: "github.com/dagger/dagger", - want: &require{ - repo: "github.com/dagger/dagger", - path: "", - version: "", - }, - }, - { - name: "Dagger repo with path", - in: "github.com/dagger/dagger/stdlib", - want: &require{ - repo: "github.com/dagger/dagger", - path: "/stdlib", - version: "", - }, - }, - { - name: "Dagger repo with longer path", - in: "github.com/dagger/dagger/stdlib/test/test", - want: &require{ - repo: "github.com/dagger/dagger", - path: "/stdlib/test/test", - version: "", - }, - }, - { - name: "Dagger repo with path and version", - in: "github.com/dagger/dagger/stdlib@v0.1", - want: &require{ - repo: "github.com/dagger/dagger", - path: "/stdlib", - version: "v0.1", - }, - }, - { - name: "Dagger repo with longer path and version", - in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1", - want: &require{ - repo: "github.com/dagger/dagger", - path: "/stdlib/test/test", - version: "v0.0.1", - }, - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - got, err := parseArgument(c.in) - if err != nil && c.hasError { - return - } - - if err != nil { - t.Fatal(err) - } - - if got.repo != c.want.repo { - t.Errorf("repos differ: want %s, got %s", c.want.repo, got.repo) - } - - if got.path != c.want.path { - t.Errorf("paths differ: want %s, got %s", c.want.path, got.path) - } - - if got.version != c.want.version { - t.Errorf("versions differ: want %s, got %s", c.want.version, got.version) - } - }) - } -} - func TestReadFile(t *testing.T) { cases := []struct { name string diff --git a/cmd/dagger/cmd/mod/get.go b/cmd/dagger/cmd/mod/get.go index 89fd32f3..669f0e8a 100644 --- a/cmd/dagger/cmd/mod/get.go +++ b/cmd/dagger/cmd/mod/get.go @@ -10,11 +10,12 @@ import ( "github.com/spf13/viper" "go.dagger.io/dagger/cmd/dagger/cmd/common" "go.dagger.io/dagger/cmd/dagger/logger" + "go.dagger.io/dagger/telemetry" ) var getCmd = &cobra.Command{ Use: "get [packages]", - Short: "download and install packages and dependencies", + Short: "download and install dependencies", Args: cobra.MaximumNArgs(1), PreRun: func(cmd *cobra.Command, args []string) { // Fix Viper bug for duplicate flags: @@ -28,12 +29,17 @@ var getCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - doneCh := common.TrackCommand(ctx, cmd) - if len(args) == 0 { lg.Fatal().Msg("need to specify package name in command argument") } + workspace := common.CurrentWorkspace(ctx) + st := common.CurrentEnvironmentState(ctx, workspace) + doneCh := common.TrackWorkspaceCommand(ctx, cmd, workspace, st, &telemetry.Property{ + Name: "packages", + Value: args, + }) + // parse packages to install var packages []*require for _, arg := range args { @@ -47,21 +53,20 @@ var getCmd = &cobra.Command{ } // read mod file in the current dir - modFile, err := readModFile() + modFile, err := readModFile(workspace.Path) if err != nil { lg.Fatal().Err(err).Msgf("error loading module file") } // download packages - destBasePath := "./cue.mod/pkg" for _, p := range packages { - if err := processRequire(p, modFile, destBasePath); err != nil { + if err := processRequire(p, modFile); err != nil { lg.Error().Err(err).Msg("error processing package") } } // write to mod file in the current dir - if err = writeModFile(modFile); err != nil { + if err = writeModFile(workspace.Path, modFile); err != nil { lg.Error().Err(err).Msg("error writing to mod file") } @@ -69,13 +74,14 @@ var getCmd = &cobra.Command{ }, } -func processRequire(req *require, modFile *file, destBasePath string) error { - tmpRepoPath := path.Join("./cue.mod/tmp", req.repo) - if err := os.MkdirAll(tmpRepoPath, 0755); err != nil { +func processRequire(req *require, modFile *file) error { + tmpPath := path.Join(tmpBasePath, req.repo) + if err := os.MkdirAll(tmpPath, 0755); err != nil { return fmt.Errorf("error creating tmp dir for cloning package") } + defer os.RemoveAll(tmpPath) - r, err := clone(req, tmpRepoPath) + r, err := clone(req, tmpPath) if err != nil { return fmt.Errorf("error downloading package %s: %w", req, err) } @@ -84,11 +90,11 @@ func processRequire(req *require, modFile *file, destBasePath string) error { // requirement is new, so we should move the files and add it to the module.cue if existing == nil { - if err := move(req, tmpRepoPath, destBasePath); err != nil { + if err := move(req, tmpPath, destBasePath); err != nil { return err } modFile.require = append(modFile.require, req) - return os.RemoveAll(tmpRepoPath) + return nil } c, err := compareVersions(existing.version, req.version) @@ -98,7 +104,7 @@ func processRequire(req *require, modFile *file, destBasePath string) error { // the existing requirement is newer so we skip installation if c > 0 { - return os.RemoveAll(tmpRepoPath) + return nil } // the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing @@ -108,7 +114,7 @@ func processRequire(req *require, modFile *file, destBasePath string) error { return err } - return replace(req, tmpRepoPath, destBasePath) + return replace(req, tmpPath, destBasePath) } func compareVersions(reqV1, reqV2 string) (int, error) { diff --git a/cmd/dagger/cmd/mod/repo.go b/cmd/dagger/cmd/mod/repo.go index 646db894..f9f5e1a9 100644 --- a/cmd/dagger/cmd/mod/repo.go +++ b/cmd/dagger/cmd/mod/repo.go @@ -40,8 +40,8 @@ func clone(require *require, dir string) (*repo, error) { return nil, err } - if _, err := os.Stat(path.Join(dir, require.path, FilePath)); err != nil { - return nil, fmt.Errorf("repo does not contain %s", FilePath) + if _, err := os.Stat(path.Join(dir, require.clonePath, filePath)); err != nil { + return nil, fmt.Errorf("repo does not contain %s", filePath) } return rr, nil diff --git a/cmd/dagger/cmd/mod/repo_test.go b/cmd/dagger/cmd/mod/repo_test.go index 52ec18c0..b4210aef 100644 --- a/cmd/dagger/cmd/mod/repo_test.go +++ b/cmd/dagger/cmd/mod/repo_test.go @@ -14,28 +14,28 @@ func TestClone(t *testing.T) { { name: "resolving shorter hash version", require: require{ - prefix: "https://", - repo: "github.com/tjovicic/gcpcloudrun-cue", - path: "", - version: "d530f2ea2099", + prefix: "https://", + cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", + clonePath: "", + version: "d530f2ea2099", }, }, { name: "resolving branch name", require: require{ - prefix: "https://", - repo: "github.com/tjovicic/gcpcloudrun-cue", - path: "", - version: "main", + prefix: "https://", + cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", + clonePath: "", + version: "main", }, }, { name: "resolving tag", require: require{ - prefix: "https://", - repo: "github.com/tjovicic/gcpcloudrun-cue", - path: "", - version: "v0.3", + prefix: "https://", + cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", + clonePath: "", + version: "v0.3", }, }, } @@ -65,10 +65,10 @@ func TestListTags(t *testing.T) { defer os.Remove(tmpDir) r, err := clone(&require{ - prefix: "https://", - repo: "github.com/tjovicic/gcpcloudrun-cue", - path: "", - version: "", + prefix: "https://", + cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", + clonePath: "", + version: "", }, tmpDir) if err != nil { t.Error(err) From 29ddcca32ef14a28f289a6d41c51240d8454c4d7 Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Mon, 9 Aug 2021 07:48:45 +0200 Subject: [PATCH 06/15] fix mod command lint errors Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/arg.go | 7 ++++--- cmd/dagger/cmd/mod/file.go | 9 +++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/dagger/cmd/mod/arg.go b/cmd/dagger/cmd/mod/arg.go index 8ab9e0d2..0efb2750 100644 --- a/cmd/dagger/cmd/mod/arg.go +++ b/cmd/dagger/cmd/mod/arg.go @@ -8,11 +8,12 @@ import ( ) func parseArgument(arg string) (*require, error) { - if strings.HasPrefix(arg, "github.com") { + switch { + case strings.HasPrefix(arg, "github.com"): return parseGithubRepoName(arg) - } else if strings.HasPrefix(arg, "alpha.dagger.io") { + case strings.HasPrefix(arg, "alpha.dagger.io"): return parseDaggerRepoName(arg) - } else { + default: return nil, fmt.Errorf("repo name does not match suported providers") } } diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index 9f3d3ab4..9fe164fd 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -42,10 +42,7 @@ func read(f io.Reader) (*file, error) { return nil, err } - lines, err := nonEmptyLines(b) - if err != nil { - return nil, err - } + lines := nonEmptyLines(b) if len(lines) == 0 { return nil, fmt.Errorf("mod file is empty, missing module name") @@ -77,7 +74,7 @@ func read(f io.Reader) (*file, error) { var spaceRgx = regexp.MustCompile(`\s+`) -func nonEmptyLines(b []byte) ([]string, error) { +func nonEmptyLines(b []byte) []string { s := strings.ReplaceAll(string(b), "\r\n", "\n") split := strings.Split(s, "\n") @@ -93,7 +90,7 @@ func nonEmptyLines(b []byte) ([]string, error) { lines = append(lines, trimmed) } - return lines, nil + return lines } func writeModFile(workspacePath string, f *file) error { From 4f8128abcb3e4e36e58db892beffc795cfea502f Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Mon, 9 Aug 2021 08:05:39 +0200 Subject: [PATCH 07/15] fix installing packages using workspace dir Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/get.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/dagger/cmd/mod/get.go b/cmd/dagger/cmd/mod/get.go index 669f0e8a..406e520e 100644 --- a/cmd/dagger/cmd/mod/get.go +++ b/cmd/dagger/cmd/mod/get.go @@ -60,7 +60,7 @@ var getCmd = &cobra.Command{ // download packages for _, p := range packages { - if err := processRequire(p, modFile); err != nil { + if err := processRequire(workspace.Path, p, modFile); err != nil { lg.Error().Err(err).Msg("error processing package") } } @@ -74,8 +74,8 @@ var getCmd = &cobra.Command{ }, } -func processRequire(req *require, modFile *file) error { - tmpPath := path.Join(tmpBasePath, req.repo) +func processRequire(workspacePath string, req *require, modFile *file) error { + tmpPath := path.Join(workspacePath, tmpBasePath, req.repo) if err := os.MkdirAll(tmpPath, 0755); err != nil { return fmt.Errorf("error creating tmp dir for cloning package") } @@ -87,10 +87,11 @@ func processRequire(req *require, modFile *file) error { } existing := modFile.search(req) + destPath := path.Join(workspacePath, destBasePath) // requirement is new, so we should move the files and add it to the module.cue if existing == nil { - if err := move(req, tmpPath, destBasePath); err != nil { + if err := move(req, tmpPath, destPath); err != nil { return err } modFile.require = append(modFile.require, req) @@ -114,7 +115,7 @@ func processRequire(req *require, modFile *file) error { return err } - return replace(req, tmpPath, destBasePath) + return replace(req, tmpPath, destPath) } func compareVersions(reqV1, reqV2 string) (int, error) { From 050d1fcb3f6f2319bccef50996d9e3a832f9d596 Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Tue, 10 Aug 2021 12:23:03 +0200 Subject: [PATCH 08/15] add support for cloning private repos in mod get command Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/arg.go | 2 -- cmd/dagger/cmd/mod/file.go | 5 ----- cmd/dagger/cmd/mod/file_test.go | 2 -- cmd/dagger/cmd/mod/get.go | 7 ++++++- cmd/dagger/cmd/mod/repo.go | 24 +++++++++++++++++++----- cmd/dagger/cmd/mod/repo_test.go | 18 ++++++++++++------ 6 files changed, 37 insertions(+), 21 deletions(-) diff --git a/cmd/dagger/cmd/mod/arg.go b/cmd/dagger/cmd/mod/arg.go index 0efb2750..fe372206 100644 --- a/cmd/dagger/cmd/mod/arg.go +++ b/cmd/dagger/cmd/mod/arg.go @@ -28,7 +28,6 @@ func parseGithubRepoName(arg string) (*require, error) { } return &require{ - prefix: "https://", repo: repoMatches[1], path: repoMatches[2], version: repoMatches[3], @@ -48,7 +47,6 @@ func parseDaggerRepoName(arg string) (*require, error) { } return &require{ - prefix: "https://", repo: "alpha.dagger.io", path: repoMatches[1], version: repoMatches[2], diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index 9fe164fd..d26727e8 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -118,7 +118,6 @@ func (f *file) search(r *require) *require { } type require struct { - prefix string repo string path string version string @@ -127,10 +126,6 @@ type require struct { clonePath string } -func (r *require) cloneURL() string { - return fmt.Sprintf("%s%s", r.prefix, r.cloneRepo) -} - func (r *require) fullPath() string { return path.Join(r.repo, r.path) } diff --git a/cmd/dagger/cmd/mod/file_test.go b/cmd/dagger/cmd/mod/file_test.go index a94175f4..27fd91f1 100644 --- a/cmd/dagger/cmd/mod/file_test.go +++ b/cmd/dagger/cmd/mod/file_test.go @@ -32,13 +32,11 @@ func TestReadFile(t *testing.T) { module: "alpha.dagger.io", require: []*require{ { - prefix: "https://", repo: "github.com/tjovicic/test", path: "", version: "xyz", }, { - prefix: "https://", repo: "github.com/bla/bla", path: "", version: "abc", diff --git a/cmd/dagger/cmd/mod/get.go b/cmd/dagger/cmd/mod/get.go index 406e520e..5ded1cf1 100644 --- a/cmd/dagger/cmd/mod/get.go +++ b/cmd/dagger/cmd/mod/get.go @@ -81,7 +81,9 @@ func processRequire(workspacePath string, req *require, modFile *file) error { } defer os.RemoveAll(tmpPath) - r, err := clone(req, tmpPath) + privateKeyFile := viper.GetString("private-key-file") + privateKeyPassword := viper.GetString("private-key-password") + r, err := clone(req, tmpPath, privateKeyFile, privateKeyPassword) if err != nil { return fmt.Errorf("error downloading package %s: %w", req, err) } @@ -136,6 +138,9 @@ func compareVersions(reqV1, reqV2 string) (int, error) { } func init() { + getCmd.Flags().String("private-key-file", "~/.ssh/id_rsa", "Private ssh key") + getCmd.Flags().String("private-key-password", "", "Private ssh key password") + if err := viper.BindPFlags(getCmd.Flags()); err != nil { panic(err) } diff --git a/cmd/dagger/cmd/mod/repo.go b/cmd/dagger/cmd/mod/repo.go index f9f5e1a9..e869dceb 100644 --- a/cmd/dagger/cmd/mod/repo.go +++ b/cmd/dagger/cmd/mod/repo.go @@ -5,6 +5,9 @@ import ( "os" "path" "sort" + "strings" + + "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" @@ -16,10 +19,22 @@ type repo struct { contents *git.Repository } -func clone(require *require, dir string) (*repo, error) { - r, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: require.cloneURL(), - }) +func clone(require *require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) { + o := git.CloneOptions{ + URL: fmt.Sprintf("https://%s", require.cloneRepo), + } + + if privateKeyFile != "" { + publicKeys, err := ssh.NewPublicKeysFromFile("git", privateKeyFile, privateKeyPassword) + if err != nil { + return nil, err + } + + o.Auth = publicKeys + o.URL = fmt.Sprintf("git@%s", strings.Replace(require.cloneRepo, "/", ":", 1)) + } + + r, err := git.PlainClone(dir, false, &o) if err != nil { return nil, err } @@ -97,7 +112,6 @@ func (r *repo) latestTag() (string, error) { versions[i] = v } - // After this, the versions are properly sorted sort.Sort(version.Collection(versions)) return versions[len(versions)-1].Original(), nil diff --git a/cmd/dagger/cmd/mod/repo_test.go b/cmd/dagger/cmd/mod/repo_test.go index b4210aef..e28c6516 100644 --- a/cmd/dagger/cmd/mod/repo_test.go +++ b/cmd/dagger/cmd/mod/repo_test.go @@ -14,7 +14,6 @@ func TestClone(t *testing.T) { { name: "resolving shorter hash version", require: require{ - prefix: "https://", cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", clonePath: "", version: "d530f2ea2099", @@ -23,7 +22,6 @@ func TestClone(t *testing.T) { { name: "resolving branch name", require: require{ - prefix: "https://", cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", clonePath: "", version: "main", @@ -32,12 +30,21 @@ func TestClone(t *testing.T) { { name: "resolving tag", require: require{ - prefix: "https://", cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", clonePath: "", version: "v0.3", }, }, + { + name: "alpha.dagger.io", + require: require{ + cloneRepo: "github.com/dagger/dagger", + clonePath: "", + version: "", + + repo: "alpha.dagger.io", + }, + }, } for _, c := range cases { @@ -49,7 +56,7 @@ func TestClone(t *testing.T) { defer os.Remove(tmpDir) - _, err = clone(&c.require, tmpDir) + _, err = clone(&c.require, tmpDir, "", "") if err != nil { t.Error(err) } @@ -65,11 +72,10 @@ func TestListTags(t *testing.T) { defer os.Remove(tmpDir) r, err := clone(&require{ - prefix: "https://", cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", clonePath: "", version: "", - }, tmpDir) + }, tmpDir, "", "") if err != nil { t.Error(err) } From 2102e78c1f0f1d359e3a36db5479d31a60ccd38f Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Thu, 12 Aug 2021 08:37:17 +0200 Subject: [PATCH 09/15] test pulling private git repos using mod get command Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/file.go | 4 +- cmd/dagger/cmd/mod/get.go | 53 ++++++++++++------- cmd/dagger/cmd/mod/repo.go | 12 ++++- cmd/dagger/cmd/mod/repo_test.go | 18 ++++--- .../cmd/mod/test-ssh-keys/id_ed25519_test | 7 +++ .../cmd/mod/test-ssh-keys/id_ed25519_test.pub | 1 + 6 files changed, 65 insertions(+), 30 deletions(-) create mode 100644 cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test create mode 100644 cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test.pub diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index d26727e8..6d905f09 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -22,7 +22,7 @@ type file struct { require []*require } -func readModFile(workspacePath string) (*file, error) { +func readPath(workspacePath string) (*file, error) { f, err := os.Open(path.Join(workspacePath, filePath)) if err != nil { return nil, err @@ -93,7 +93,7 @@ func nonEmptyLines(b []byte) []string { return lines } -func writeModFile(workspacePath string, f *file) error { +func (f *file) write(workspacePath string) error { return ioutil.WriteFile(path.Join(workspacePath, filePath), f.contents().Bytes(), 0600) } diff --git a/cmd/dagger/cmd/mod/get.go b/cmd/dagger/cmd/mod/get.go index 5ded1cf1..b1b73228 100644 --- a/cmd/dagger/cmd/mod/get.go +++ b/cmd/dagger/cmd/mod/get.go @@ -40,6 +40,12 @@ var getCmd = &cobra.Command{ Value: args, }) + // read mod file in the current dir + modFile, err := readPath(workspace.Path) + if err != nil { + lg.Fatal().Err(err).Msg("error loading module file") + } + // parse packages to install var packages []*require for _, arg := range args { @@ -52,72 +58,81 @@ var getCmd = &cobra.Command{ packages = append(packages, p) } - // read mod file in the current dir - modFile, err := readModFile(workspace.Path) - if err != nil { - lg.Fatal().Err(err).Msgf("error loading module file") - } - // download packages for _, p := range packages { - if err := processRequire(workspace.Path, p, modFile); err != nil { + isNew, err := processRequire(workspace.Path, p, modFile) + if err != nil { lg.Error().Err(err).Msg("error processing package") } + + if isNew { + lg.Info().Msgf("downloading %s:%v", p.repo, p.version) + } } // write to mod file in the current dir - if err = writeModFile(workspace.Path, modFile); err != nil { + if err = modFile.write(workspace.Path); err != nil { lg.Error().Err(err).Msg("error writing to mod file") } + lg.Info().Msg("checking for new versions...") + <-doneCh }, } -func processRequire(workspacePath string, req *require, modFile *file) error { +func processRequire(workspacePath string, req *require, modFile *file) (bool, error) { + var isNew bool + tmpPath := path.Join(workspacePath, tmpBasePath, req.repo) if err := os.MkdirAll(tmpPath, 0755); err != nil { - return fmt.Errorf("error creating tmp dir for cloning package") + return false, fmt.Errorf("error creating tmp dir for cloning package") } defer os.RemoveAll(tmpPath) + // clone the repo privateKeyFile := viper.GetString("private-key-file") privateKeyPassword := viper.GetString("private-key-password") r, err := clone(req, tmpPath, privateKeyFile, privateKeyPassword) if err != nil { - return fmt.Errorf("error downloading package %s: %w", req, err) + return isNew, fmt.Errorf("error downloading package %s: %w", req, err) } existing := modFile.search(req) destPath := path.Join(workspacePath, destBasePath) - // requirement is new, so we should move the files and add it to the module.cue + // requirement is new, so we should move the files and add it to the mod file if existing == nil { if err := move(req, tmpPath, destPath); err != nil { - return err + return isNew, err } modFile.require = append(modFile.require, req) - return nil + isNew = true + return isNew, nil } c, err := compareVersions(existing.version, req.version) if err != nil { - return err + return isNew, err } // the existing requirement is newer so we skip installation if c > 0 { - return nil + return isNew, nil } // the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing // requirement and replace the code in the /pkg folder existing.version = req.version if err = r.checkout(req.version); err != nil { - return err + return isNew, err } + if err = replace(req, tmpPath, destPath); err != nil { + return isNew, err + } + isNew = true - return replace(req, tmpPath, destPath) + return isNew, nil } func compareVersions(reqV1, reqV2 string) (int, error) { @@ -138,7 +153,7 @@ func compareVersions(reqV1, reqV2 string) (int, error) { } func init() { - getCmd.Flags().String("private-key-file", "~/.ssh/id_rsa", "Private ssh key") + getCmd.Flags().String("private-key-file", "", "Private ssh key") getCmd.Flags().String("private-key-password", "", "Private ssh key password") if err := viper.BindPFlags(getCmd.Flags()); err != nil { diff --git a/cmd/dagger/cmd/mod/repo.go b/cmd/dagger/cmd/mod/repo.go index e869dceb..a5b7f3f2 100644 --- a/cmd/dagger/cmd/mod/repo.go +++ b/cmd/dagger/cmd/mod/repo.go @@ -45,10 +45,16 @@ func clone(require *require, dir string, privateKeyFile, privateKeyPassword stri } if require.version == "" { - require.version, err = rr.latestTag() + latestTag, err := rr.latestTag() if err != nil { return nil, err } + + if latestTag == "" { + return nil, fmt.Errorf("no git tags found in the repo") + } + + require.version = latestTag } if err := rr.checkout(require.version); err != nil { @@ -114,5 +120,9 @@ func (r *repo) latestTag() (string, error) { sort.Sort(version.Collection(versions)) + if len(versions) == 0 { + return "", nil + } + return versions[len(versions)-1].Original(), nil } diff --git a/cmd/dagger/cmd/mod/repo_test.go b/cmd/dagger/cmd/mod/repo_test.go index e28c6516..7fdbb1ea 100644 --- a/cmd/dagger/cmd/mod/repo_test.go +++ b/cmd/dagger/cmd/mod/repo_test.go @@ -8,8 +8,10 @@ import ( func TestClone(t *testing.T) { cases := []struct { - name string - require require + name string + require require + privateKeyFile string + privateKeyPassword string }{ { name: "resolving shorter hash version", @@ -36,14 +38,14 @@ func TestClone(t *testing.T) { }, }, { - name: "alpha.dagger.io", + name: "Dagger private test repo", require: require{ - cloneRepo: "github.com/dagger/dagger", + cloneRepo: "github.com/dagger/test", clonePath: "", - version: "", - - repo: "alpha.dagger.io", + version: "v0.2", }, + privateKeyFile: "./test-ssh-keys/id_ed25519_test", + privateKeyPassword: "", }, } @@ -56,7 +58,7 @@ func TestClone(t *testing.T) { defer os.Remove(tmpDir) - _, err = clone(&c.require, tmpDir, "", "") + _, err = clone(&c.require, tmpDir, c.privateKeyFile, c.privateKeyPassword) if err != nil { t.Error(err) } diff --git a/cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test b/cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test new file mode 100644 index 00000000..58f9f087 --- /dev/null +++ b/cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACCpGsk8WLx7gXCXX1muGhKjlkqaqykF1X198WQMkBO2pwAAAKC5Ec8WuRHP +FgAAAAtzc2gtZWQyNTUxOQAAACCpGsk8WLx7gXCXX1muGhKjlkqaqykF1X198WQMkBO2pw +AAAEBXE9Uht+QHuyK7+yYcZFVWOJ3qkhUh/wn289nDKDPHKakayTxYvHuBcJdfWa4aEqOW +SpqrKQXVfX3xZAyQE7anAAAAGnRpaG9taXIuam92aWNpY0B0b3B0YWwuY29tAQID +-----END OPENSSH PRIVATE KEY----- diff --git a/cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test.pub b/cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test.pub new file mode 100644 index 00000000..28bde217 --- /dev/null +++ b/cmd/dagger/cmd/mod/test-ssh-keys/id_ed25519_test.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKkayTxYvHuBcJdfWa4aEqOWSpqrKQXVfX3xZAyQE7an tihomir.jovicic@toptal.com From 6a0b74416a5413a299fc0507a57378a75793eb22 Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Thu, 12 Aug 2021 10:47:59 +0200 Subject: [PATCH 10/15] support upgrading existing packages with empty mod get command Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/file.go | 77 +++++++++++++++++++++++++++++- cmd/dagger/cmd/mod/get.go | 96 +++++++++----------------------------- 2 files changed, 97 insertions(+), 76 deletions(-) diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index 6d905f09..13c869d8 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -10,6 +10,8 @@ import ( "path/filepath" "regexp" "strings" + + "github.com/spf13/viper" ) const filePath = "./cue.mod/dagger.mod.cue" @@ -20,6 +22,8 @@ const tmpBasePath = "./cue.mod/tmp" type file struct { module string require []*require + + workspacePath string } func readPath(workspacePath string) (*file, error) { @@ -33,6 +37,8 @@ func readPath(workspacePath string) (*file, error) { return nil, err } + modFile.workspacePath = workspacePath + return modFile, nil } @@ -93,8 +99,75 @@ func nonEmptyLines(b []byte) []string { return lines } -func (f *file) write(workspacePath string) error { - return ioutil.WriteFile(path.Join(workspacePath, filePath), f.contents().Bytes(), 0600) +func (f *file) processRequire(req *require, upgrade bool) (bool, error) { + var isNew bool + + tmpPath := path.Join(f.workspacePath, tmpBasePath, req.repo) + if err := os.MkdirAll(tmpPath, 0755); err != nil { + return false, fmt.Errorf("error creating tmp dir for cloning package") + } + defer os.RemoveAll(tmpPath) + + // clone the repo + privateKeyFile := viper.GetString("private-key-file") + privateKeyPassword := viper.GetString("private-key-password") + r, err := clone(req, tmpPath, privateKeyFile, privateKeyPassword) + if err != nil { + return isNew, fmt.Errorf("error downloading package %s: %w", req, err) + } + + existing := f.search(req) + destPath := path.Join(f.workspacePath, destBasePath) + + // requirement is new, so we should move the files and add it to the mod file + if existing == nil { + if err := move(req, tmpPath, destPath); err != nil { + return isNew, err + } + f.require = append(f.require, req) + isNew = true + return isNew, nil + } + + if upgrade { + latestTag, err := r.latestTag() + if err != nil { + return isNew, err + } + + if latestTag == "" { + return isNew, fmt.Errorf("repo does not have a tag") + } + + req.version = latestTag + } + + c, err := compareVersions(existing.version, req.version) + if err != nil { + return isNew, err + } + + // the existing requirement is newer so we skip installation + if c > 0 { + return isNew, nil + } + + // the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing + // requirement and replace the code in the /pkg folder + existing.version = req.version + if err = r.checkout(req.version); err != nil { + return isNew, err + } + if err = replace(req, tmpPath, destPath); err != nil { + return isNew, err + } + isNew = true + + return isNew, nil +} + +func (f *file) write() error { + return ioutil.WriteFile(path.Join(f.workspacePath, filePath), f.contents().Bytes(), 0600) } func (f *file) contents() *bytes.Buffer { diff --git a/cmd/dagger/cmd/mod/get.go b/cmd/dagger/cmd/mod/get.go index b1b73228..30638764 100644 --- a/cmd/dagger/cmd/mod/get.go +++ b/cmd/dagger/cmd/mod/get.go @@ -1,10 +1,6 @@ package mod import ( - "fmt" - "os" - "path" - "github.com/hashicorp/go-version" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -29,10 +25,6 @@ var getCmd = &cobra.Command{ lg := logger.New() ctx := lg.WithContext(cmd.Context()) - if len(args) == 0 { - lg.Fatal().Msg("need to specify package name in command argument") - } - workspace := common.CurrentWorkspace(ctx) st := common.CurrentEnvironmentState(ctx, workspace) doneCh := common.TrackWorkspaceCommand(ctx, cmd, workspace, st, &telemetry.Property{ @@ -48,21 +40,28 @@ var getCmd = &cobra.Command{ // parse packages to install var packages []*require - for _, arg := range args { - p, err := parseArgument(arg) - if err != nil { - lg.Error().Err(err).Msgf("error parsing package %s", arg) - continue - } + var upgrade bool - packages = append(packages, p) + if len(args) == 0 { + lg.Info().Msg("upgrading installed packages...") + packages = modFile.require + upgrade = true + } else { + for _, arg := range args { + p, err := parseArgument(arg) + if err != nil { + lg.Error().Err(err).Msgf("error parsing package %s", arg) + continue + } + packages = append(packages, p) + } } // download packages for _, p := range packages { - isNew, err := processRequire(workspace.Path, p, modFile) + isNew, err := modFile.processRequire(p, upgrade) if err != nil { - lg.Error().Err(err).Msg("error processing package") + lg.Error().Err(err).Msgf("error processing package %s", p.repo) } if isNew { @@ -71,70 +70,14 @@ var getCmd = &cobra.Command{ } // write to mod file in the current dir - if err = modFile.write(workspace.Path); err != nil { + if err = modFile.write(); err != nil { lg.Error().Err(err).Msg("error writing to mod file") } - lg.Info().Msg("checking for new versions...") - <-doneCh }, } -func processRequire(workspacePath string, req *require, modFile *file) (bool, error) { - var isNew bool - - tmpPath := path.Join(workspacePath, tmpBasePath, req.repo) - if err := os.MkdirAll(tmpPath, 0755); err != nil { - return false, fmt.Errorf("error creating tmp dir for cloning package") - } - defer os.RemoveAll(tmpPath) - - // clone the repo - privateKeyFile := viper.GetString("private-key-file") - privateKeyPassword := viper.GetString("private-key-password") - r, err := clone(req, tmpPath, privateKeyFile, privateKeyPassword) - if err != nil { - return isNew, fmt.Errorf("error downloading package %s: %w", req, err) - } - - existing := modFile.search(req) - destPath := path.Join(workspacePath, destBasePath) - - // requirement is new, so we should move the files and add it to the mod file - if existing == nil { - if err := move(req, tmpPath, destPath); err != nil { - return isNew, err - } - modFile.require = append(modFile.require, req) - isNew = true - return isNew, nil - } - - c, err := compareVersions(existing.version, req.version) - if err != nil { - return isNew, err - } - - // the existing requirement is newer so we skip installation - if c > 0 { - return isNew, nil - } - - // the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing - // requirement and replace the code in the /pkg folder - existing.version = req.version - if err = r.checkout(req.version); err != nil { - return isNew, err - } - if err = replace(req, tmpPath, destPath); err != nil { - return isNew, err - } - isNew = true - - return isNew, nil -} - func compareVersions(reqV1, reqV2 string) (int, error) { v1, err := version.NewVersion(reqV1) if err != nil { @@ -145,10 +88,15 @@ func compareVersions(reqV1, reqV2 string) (int, error) { if err != nil { return 0, err } + if v1.LessThan(v2) { return -1, nil } + if v1.Equal(v2) { + return 0, nil + } + return 1, nil } From a3d2f72ea36c4d8d7ee863db6d657064eb7d9aa4 Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Mon, 23 Aug 2021 06:29:36 +0200 Subject: [PATCH 11/15] create dagger.mod.cue file automatically Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index 13c869d8..aed7c1d1 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -51,7 +51,7 @@ func read(f io.Reader) (*file, error) { lines := nonEmptyLines(b) if len(lines) == 0 { - return nil, fmt.Errorf("mod file is empty, missing module name") + return nil, fmt.Errorf("cue.mod/dagger.mod.cue file is empty, missing module name") } var module string From 620a37715cff80cac0aa498f81b512c977fe8231 Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Mon, 23 Aug 2021 06:50:59 +0200 Subject: [PATCH 12/15] create dagger.mod.cue file automatically Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/file.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index aed7c1d1..d70393ff 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -2,8 +2,10 @@ package mod import ( "bytes" + "errors" "fmt" "io" + "io/fs" "io/ioutil" "os" "path" @@ -27,9 +29,18 @@ type file struct { } func readPath(workspacePath string) (*file, error) { - f, err := os.Open(path.Join(workspacePath, filePath)) + p := path.Join(workspacePath, filePath) + + f, err := os.Open(p) if err != nil { - return nil, err + if !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + + // dagger.mod.cue doesn't exist, let's create an empty file + if f, err = os.Create(p); err != nil { + return nil, err + } } modFile, err := read(f) From 8aaaacc9da4277f6105aef6b5a09c04d3b60ad6d Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Mon, 23 Aug 2021 12:30:19 +0200 Subject: [PATCH 13/15] Add package manager usage docs Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/file.go | 4 +- cmd/dagger/cmd/mod/repo_test.go | 22 +++--- docs/learn/1011-package-manager.md | 122 +++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 13 deletions(-) create mode 100644 docs/learn/1011-package-manager.md diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index d70393ff..21a69f08 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -158,8 +158,8 @@ func (f *file) processRequire(req *require, upgrade bool) (bool, error) { return isNew, err } - // the existing requirement is newer so we skip installation - if c > 0 { + // the existing requirement is newer or equal so we skip installation + if c >= 0 { return isNew, nil } diff --git a/cmd/dagger/cmd/mod/repo_test.go b/cmd/dagger/cmd/mod/repo_test.go index 7fdbb1ea..5831d6f5 100644 --- a/cmd/dagger/cmd/mod/repo_test.go +++ b/cmd/dagger/cmd/mod/repo_test.go @@ -16,25 +16,25 @@ func TestClone(t *testing.T) { { name: "resolving shorter hash version", require: require{ - cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", - clonePath: "", - version: "d530f2ea2099", + cloneRepo: "github.com/tjovicic/dagger-modules", + clonePath: "gcpcloudrun", + version: "f4a5110b86a43871", }, }, { name: "resolving branch name", require: require{ - cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", - clonePath: "", + cloneRepo: "github.com/tjovicic/dagger-modules", + clonePath: "gcpcloudrun", version: "main", }, }, { name: "resolving tag", require: require{ - cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", - clonePath: "", - version: "v0.3", + cloneRepo: "github.com/tjovicic/dagger-modules", + clonePath: "gcpcloudrun", + version: "v0.1", }, }, { @@ -74,12 +74,12 @@ func TestListTags(t *testing.T) { defer os.Remove(tmpDir) r, err := clone(&require{ - cloneRepo: "github.com/tjovicic/gcpcloudrun-cue", - clonePath: "", + cloneRepo: "github.com/tjovicic/dagger-modules", + clonePath: "gcpcloudrun", version: "", }, tmpDir, "", "") if err != nil { - t.Error(err) + t.Fatal(err) } tags, err := r.listTags() diff --git a/docs/learn/1011-package-manager.md b/docs/learn/1011-package-manager.md new file mode 100644 index 00000000..431faf5e --- /dev/null +++ b/docs/learn/1011-package-manager.md @@ -0,0 +1,122 @@ +--- +slug: /1011/package-manager/ +--- + +# Manage packages using the package manager + +This tutorial illustrates how to install and upgrade packages using Dagger package manager. + +## Installing a package + +### Initializing workspace + +Create an empty directory for your new Dagger workspace: + +```shell +mkdir workspace +cd workspace +``` + +As described in the previous tutorials, initialize your Dagger workspace: + +```shell +dagger init +dagger new test +``` + +That will create 2 directories: `.dagger` and `cue.mod` where our package will reside: + +```shell +. +├── cue.mod +│ ├── module.cue +│ ├── pkg +│ └── usr +├── .dagger +│ └── env +│ └── test +``` + +### Install + +In our example we will use `gcpcloudrun` module from [github](https://github.com/tjovicic/dagger-modules/blob/main/gcpcloudrun/source.cue) +Let's first add it to our `source.cue` file: + +```cue title="./source.cue" +package main + +import ( + "github.com/tjovicic/dagger-modules/gcpcloudrun" +) + +run: gcpcloudrun.#Run +``` + +To install it just run + +```shell +dagger mod get github.com/tjovicic/dagger-modules/gcpcloudrun@v0.1 +``` + +It should pull the `v0.1` version from Github, leave a copy in `cue.mod/pkg` and reflect the change in +`cue.mod/dagger.mod.cue` file: + +```shell +cue.mod/pkg/github.com/ +└── tjovicic + └── dagger-modules + └── gcpcloudrun + ├── cue.mod + ├── README.md + └── source.cue +``` + +```cue title="./cue.mod/dagger.mod.cue" +module: main + +github.com/tjovicic/dagger-modules/gcpcloudrun v0.1 +``` + +Querying the current setup with `dagger query` should return a valid result: + +```json +{ + "run": { + "creds": { + "username": "oauth2accesstoken" + }, + "deploy": { + "platform": "managed", + "port": "80" + }, + "push": { + "auth": { + "username": "oauth2accesstoken" + }, + "push": {} + } + } +} +``` + +### Upgrading + +Now that you've successfully installed a package, let's try to upgrade it. + +```shell +dagger mod get github.com/tjovicic/dagger-modules/gcpcloudrun@v0.2 +``` + +You should see similar output: + +```shell +12:25PM INF system | downloading github.com/tjovicic/dagger-modules:v0.2 +``` + +And `cue.mod/dagger.mod.cue` should reflect the new version: + +```cue title="./cue.mod/dagger.mod.cue" +module: main + +github.com/tjovicic/dagger-modules/gcpcloudrun v0.2 +``` From a6a6037d01491dab2c229ef13ab8dcba98252e9a Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Tue, 24 Aug 2021 10:16:29 +0200 Subject: [PATCH 14/15] rename dagger.mod.cue to dagger.mod and remove the need for module name Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/file.go | 18 +++--------------- cmd/dagger/cmd/mod/file_test.go | 16 ---------------- docs/learn/1011-package-manager.md | 12 ++++-------- 3 files changed, 7 insertions(+), 39 deletions(-) diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index 21a69f08..969f69e7 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -16,13 +16,12 @@ import ( "github.com/spf13/viper" ) -const filePath = "./cue.mod/dagger.mod.cue" +const filePath = "./cue.mod/dagger.mod" const destBasePath = "./cue.mod/pkg" const tmpBasePath = "./cue.mod/tmp" // A file is the parsed, interpreted form of a cue.mod file. type file struct { - module string require []*require workspacePath string @@ -61,18 +60,9 @@ func read(f io.Reader) (*file, error) { lines := nonEmptyLines(b) - if len(lines) == 0 { - return nil, fmt.Errorf("cue.mod/dagger.mod.cue file is empty, missing module name") - } - - var module string - if split := strings.Split(lines[0], " "); len(split) > 1 { - module = strings.Trim(split[1], "\"") - } - var requires []*require - for i := 1; i < len(lines); i++ { - split := strings.Split(lines[i], " ") + for _, line := range lines { + split := strings.Split(line, " ") r, err := parseArgument(split[0]) if err != nil { return nil, err @@ -84,7 +74,6 @@ func read(f io.Reader) (*file, error) { } return &file{ - module: module, require: requires, }, nil } @@ -184,7 +173,6 @@ func (f *file) write() error { func (f *file) contents() *bytes.Buffer { var b bytes.Buffer - b.WriteString(fmt.Sprintf("module: %s\n\n", f.module)) for _, r := range f.require { b.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version)) } diff --git a/cmd/dagger/cmd/mod/file_test.go b/cmd/dagger/cmd/mod/file_test.go index 27fd91f1..1b1c5a4d 100644 --- a/cmd/dagger/cmd/mod/file_test.go +++ b/cmd/dagger/cmd/mod/file_test.go @@ -11,25 +11,13 @@ func TestReadFile(t *testing.T) { input string want *file }{ - { - name: "module file without dependencies", - input: ` - module: "alpha.dagger.io" - `, - want: &file{ - module: "alpha.dagger.io", - }, - }, { name: "module file with valid dependencies", input: ` - module: "alpha.dagger.io" - github.com/tjovicic/test xyz github.com/bla/bla abc `, want: &file{ - module: "alpha.dagger.io", require: []*require{ { repo: "github.com/tjovicic/test", @@ -53,10 +41,6 @@ func TestReadFile(t *testing.T) { t.Error(err) } - if got.module != c.want.module { - t.Errorf("module names differ: want %s, got %s", c.want.module, got.module) - } - if len(got.require) != len(c.want.require) { t.Errorf("requires length differs: want %d, got %d", len(c.want.require), len(got.require)) } diff --git a/docs/learn/1011-package-manager.md b/docs/learn/1011-package-manager.md index 431faf5e..c0679930 100644 --- a/docs/learn/1011-package-manager.md +++ b/docs/learn/1011-package-manager.md @@ -52,13 +52,13 @@ import ( run: gcpcloudrun.#Run ``` -To install it just run +To install it just run ```shell dagger mod get github.com/tjovicic/dagger-modules/gcpcloudrun@v0.1 ``` -It should pull the `v0.1` version from Github, leave a copy in `cue.mod/pkg` and reflect the change in +It should pull the `v0.1` version from Github, leave a copy in `cue.mod/pkg` and reflect the change in `cue.mod/dagger.mod.cue` file: ```shell @@ -71,9 +71,7 @@ cue.mod/pkg/github.com/ └── source.cue ``` -```cue title="./cue.mod/dagger.mod.cue" -module: main - +```cue title="./cue.mod/dagger.mod" github.com/tjovicic/dagger-modules/gcpcloudrun v0.1 ``` @@ -115,8 +113,6 @@ You should see similar output: And `cue.mod/dagger.mod.cue` should reflect the new version: -```cue title="./cue.mod/dagger.mod.cue" -module: main - +```cue title="./cue.mod/dagger.mod" github.com/tjovicic/dagger-modules/gcpcloudrun v0.2 ``` From 90d902fa7710c61cb3271cf9cad554ec8c2424ec Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Tue, 24 Aug 2021 11:00:24 +0200 Subject: [PATCH 15/15] fix mod get command tests Signed-off-by: Tihomir Jovicic --- cmd/dagger/cmd/mod/file.go | 2 +- cmd/dagger/cmd/mod/repo_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go index 969f69e7..8ac5001a 100644 --- a/cmd/dagger/cmd/mod/file.go +++ b/cmd/dagger/cmd/mod/file.go @@ -60,7 +60,7 @@ func read(f io.Reader) (*file, error) { lines := nonEmptyLines(b) - var requires []*require + requires := make([]*require, 0, len(lines)) for _, line := range lines { split := strings.Split(line, " ") r, err := parseArgument(split[0]) diff --git a/cmd/dagger/cmd/mod/repo_test.go b/cmd/dagger/cmd/mod/repo_test.go index 5831d6f5..19375d4b 100644 --- a/cmd/dagger/cmd/mod/repo_test.go +++ b/cmd/dagger/cmd/mod/repo_test.go @@ -18,7 +18,7 @@ func TestClone(t *testing.T) { require: require{ cloneRepo: "github.com/tjovicic/dagger-modules", clonePath: "gcpcloudrun", - version: "f4a5110b86a43871", + version: "26a1d46d1b3c", }, }, { @@ -34,7 +34,7 @@ func TestClone(t *testing.T) { require: require{ cloneRepo: "github.com/tjovicic/dagger-modules", clonePath: "gcpcloudrun", - version: "v0.1", + version: "v0.3", }, }, { @@ -42,7 +42,7 @@ func TestClone(t *testing.T) { require: require{ cloneRepo: "github.com/dagger/test", clonePath: "", - version: "v0.2", + version: "main", }, privateKeyFile: "./test-ssh-keys/id_ed25519_test", privateKeyPassword: "",