From 0010609f4de38153f154d0979bb3cea2fdfcc1aa Mon Sep 17 00:00:00 2001 From: Tihomir Jovicic Date: Fri, 30 Jul 2021 08:02:03 +0200 Subject: [PATCH] First version of package manager Signed-off-by: Tihomir Jovicic --- client/client.go | 2 +- cmd/dagger/cmd/mod/file.go | 205 ++++++++++++++++++++++++++++++++ cmd/dagger/cmd/mod/file_test.go | 152 +++++++++++++++++++++++ cmd/dagger/cmd/mod/get.go | 135 +++++++++++++++++++++ cmd/dagger/cmd/mod/get_test.go | 7 ++ cmd/dagger/cmd/mod/repo.go | 104 ++++++++++++++++ cmd/dagger/cmd/mod/repo_test.go | 85 +++++++++++++ cmd/dagger/cmd/mod/root.go | 17 +++ cmd/dagger/cmd/root.go | 2 + util/buildkitd/buildkitd.go | 14 ++- 10 files changed, 716 insertions(+), 7 deletions(-) create mode 100644 cmd/dagger/cmd/mod/file.go create mode 100644 cmd/dagger/cmd/mod/file_test.go create mode 100644 cmd/dagger/cmd/mod/get.go create mode 100644 cmd/dagger/cmd/mod/get_test.go create mode 100644 cmd/dagger/cmd/mod/repo.go create mode 100644 cmd/dagger/cmd/mod/repo_test.go create mode 100644 cmd/dagger/cmd/mod/root.go diff --git a/client/client.go b/client/client.go index 8267849e..965b7e8e 100644 --- a/client/client.go +++ b/client/client.go @@ -31,7 +31,7 @@ import ( "go.dagger.io/dagger/state" ) -// A dagger client +// Client is a dagger client type Client struct { c *bk.Client noCache bool diff --git a/cmd/dagger/cmd/mod/file.go b/cmd/dagger/cmd/mod/file.go new file mode 100644 index 00000000..29474951 --- /dev/null +++ b/cmd/dagger/cmd/mod/file.go @@ -0,0 +1,205 @@ +package mod + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "regexp" + "strings" +) + +// A file is the parsed, interpreted form of a cue.mod file. +type file struct { + module string + require []*require +} + +func (f *file) contents() *bytes.Buffer { + var b bytes.Buffer + + b.WriteString(fmt.Sprintf("module: %s\n\n", f.module)) + for _, r := range f.require { + b.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version)) + } + + return &b +} + +func (f *file) search(r *require) *require { + for _, existing := range f.require { + if existing.fullPath() == r.fullPath() { + return existing + } + } + return nil +} + +type require struct { + prefix string + repo string + path string + version string +} + +func (r *require) cloneUrl() string { + return fmt.Sprintf("%s%s", r.prefix, r.repo) +} + +func (r *require) fullPath() string { + return path.Join(r.repo, r.path) +} + +func read(f io.Reader) (*file, error) { + b, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + lines, err := nonEmptyLines(b) + if err != nil { + return nil, err + } + + var module string + if split := strings.Split(lines[0], " "); len(split) > 1 { + module = strings.Trim(split[1], "\"") + } + + var requires []*require + for i := 1; i < len(lines); i++ { + split := strings.Split(lines[i], " ") + r, err := parseArgument(split[0]) + if err != nil { + return nil, err + } + + r.version = split[1] + + requires = append(requires, r) + } + + return &file{ + module: module, + require: requires, + }, nil +} + +func nonEmptyLines(b []byte) ([]string, error) { + s := strings.ReplaceAll(string(b), "\r\n", "\n") + split := strings.Split(s, "\n") + + spaceRgx, err := regexp.Compile(`\s+`) + if err != nil { + return nil, err + } + + var lines []string + for _, l := range split { + trimmed := strings.TrimSpace(l) + if trimmed == "" { + continue + } + + trimmed = spaceRgx.ReplaceAllString(trimmed, " ") + + lines = append(lines, trimmed) + } + + return lines, nil +} + +func parseArgument(arg string) (*require, error) { + if strings.HasPrefix(arg, "alpha.dagger.io") { + arg = strings.Replace(arg, "alpha.dagger.io", "github.com/dagger/dagger/stdlib", 1) + } + + name, suffix, err := parseGithubRepoName(arg) + if err != nil { + return nil, err + } + + repoPath, version, err := parseGithubRepoVersion(suffix) + if err != nil { + return nil, err + } + + return &require{ + prefix: "https://", + repo: name, + path: repoPath, + version: version, + }, nil +} + +func parseGithubRepoName(arg string) (string, string, error) { + repoRegex, err := regexp.Compile("(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)(.*)") + if err != nil { + return "", "", err + } + + repoMatches := repoRegex.FindStringSubmatch(arg) + + if len(repoMatches) == 0 { + return "", "", fmt.Errorf("repo name does not match suported providers") + } + + // returns 2 elements: repo name and path+version + return repoMatches[1], repoMatches[2], nil +} + +func parseGithubRepoVersion(repoSuffix string) (string, string, error) { + if repoSuffix == "" { + return "", "", nil + } + + i := strings.LastIndexAny(repoSuffix, "@:") + if i == -1 { + return repoSuffix, "", nil + } + + return repoSuffix[:i], repoSuffix[i+1:], nil +} + +func readModFile() (*file, error) { + f, err := os.Open("./cue.mod/module.cue") + if err != nil { + return nil, err + } + + modFile, err := read(f) + if err != nil { + return nil, err + } + + return modFile, nil +} + +func writeModFile(f *file) error { + return ioutil.WriteFile("./cue.mod/module.cue", f.contents().Bytes(), 0644) +} + +func move(r *require, sourceRepoPath, destBasePath string) error { + fmt.Println("move") + destPath := path.Join(destBasePath, r.fullPath()) + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return err + } + if err := os.Rename(path.Join(sourceRepoPath, r.path), destPath); err != nil { + return err + } + + return nil +} + +func replace(r *require, sourceRepoPath, destBasePath string) error { + fmt.Println("replace") + if err := os.RemoveAll(path.Join(destBasePath, r.fullPath())); err != nil { + return err + } + + return move(r, sourceRepoPath, destBasePath) +} diff --git a/cmd/dagger/cmd/mod/file_test.go b/cmd/dagger/cmd/mod/file_test.go new file mode 100644 index 00000000..f8ff9cb5 --- /dev/null +++ b/cmd/dagger/cmd/mod/file_test.go @@ -0,0 +1,152 @@ +package mod + +import ( + "strings" + "testing" +) + +func TestParseArgument(t *testing.T) { + cases := []struct { + name string + in string + want *require + hasError bool + }{ + { + name: "Random", + in: "abcd/bla@:/xyz", + hasError: true, + }, + { + name: "Dagger repo", + in: "github.com/dagger/dagger", + want: &require{ + repo: "github.com/dagger/dagger", + path: "", + version: "", + }, + }, + { + name: "Dagger repo with path", + in: "github.com/dagger/dagger/stdlib", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib", + version: "", + }, + }, + { + name: "Dagger repo with longer path", + in: "github.com/dagger/dagger/stdlib/test/test", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib/test/test", + version: "", + }, + }, + { + name: "Dagger repo with path and version", + in: "github.com/dagger/dagger/stdlib@v0.1", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib", + version: "v0.1", + }, + }, + { + name: "Dagger repo with longer path and version", + in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1", + want: &require{ + repo: "github.com/dagger/dagger", + path: "/stdlib/test/test", + version: "v0.0.1", + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := parseArgument(c.in) + if err != nil && c.hasError { + return + } + + if err != nil { + t.Fatal(err) + } + + if got.repo != c.want.repo { + t.Errorf("repos differ: want %s, got %s", c.want.repo, got.repo) + } + + if got.path != c.want.path { + t.Errorf("paths differ: want %s, got %s", c.want.path, got.path) + } + + if got.version != c.want.version { + t.Errorf("versions differ: want %s, got %s", c.want.version, got.version) + } + }) + } +} + +func TestReadFile(t *testing.T) { + cases := []struct { + name string + input string + want *file + }{ + { + name: "module file without dependencies", + input: ` + module: "alpha.dagger.io" + `, + want: &file{ + module: "alpha.dagger.io", + }, + }, + { + name: "module file with valid dependencies", + input: ` + module: "alpha.dagger.io" + + github.com/tjovicic/test xyz + github.com/bla/bla abc + `, + want: &file{ + module: "alpha.dagger.io", + require: []*require{ + { + prefix: "https://", + repo: "github.com/tjovicic/test", + path: "", + version: "xyz", + }, + { + prefix: "https://", + repo: "github.com/bla/bla", + path: "", + version: "abc", + }, + }, + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := read(strings.NewReader(c.input)) + if err != nil { + t.Error(err) + } + + if got.module != c.want.module { + t.Errorf("module names differ: want %s, got %s", c.want.module, got.module) + } + + if len(got.require) != len(c.want.require) { + t.Errorf("requires length differs: want %d, got %d", len(c.want.require), len(got.require)) + } + }) + } +} diff --git a/cmd/dagger/cmd/mod/get.go b/cmd/dagger/cmd/mod/get.go new file mode 100644 index 00000000..89fd32f3 --- /dev/null +++ b/cmd/dagger/cmd/mod/get.go @@ -0,0 +1,135 @@ +package mod + +import ( + "fmt" + "os" + "path" + + "github.com/hashicorp/go-version" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "go.dagger.io/dagger/cmd/dagger/cmd/common" + "go.dagger.io/dagger/cmd/dagger/logger" +) + +var getCmd = &cobra.Command{ + Use: "get [packages]", + Short: "download and install packages and dependencies", + Args: cobra.MaximumNArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + // Fix Viper bug for duplicate flags: + // https://github.com/spf13/viper/issues/233 + if err := viper.BindPFlags(cmd.Flags()); err != nil { + panic(err) + } + }, + + Run: func(cmd *cobra.Command, args []string) { + lg := logger.New() + ctx := lg.WithContext(cmd.Context()) + + doneCh := common.TrackCommand(ctx, cmd) + + if len(args) == 0 { + lg.Fatal().Msg("need to specify package name in command argument") + } + + // parse packages to install + var packages []*require + for _, arg := range args { + p, err := parseArgument(arg) + if err != nil { + lg.Error().Err(err).Msgf("error parsing package %s", arg) + continue + } + + packages = append(packages, p) + } + + // read mod file in the current dir + modFile, err := readModFile() + if err != nil { + lg.Fatal().Err(err).Msgf("error loading module file") + } + + // download packages + destBasePath := "./cue.mod/pkg" + for _, p := range packages { + if err := processRequire(p, modFile, destBasePath); err != nil { + lg.Error().Err(err).Msg("error processing package") + } + } + + // write to mod file in the current dir + if err = writeModFile(modFile); err != nil { + lg.Error().Err(err).Msg("error writing to mod file") + } + + <-doneCh + }, +} + +func processRequire(req *require, modFile *file, destBasePath string) error { + tmpRepoPath := path.Join("./cue.mod/tmp", req.repo) + if err := os.MkdirAll(tmpRepoPath, 0755); err != nil { + return fmt.Errorf("error creating tmp dir for cloning package") + } + + r, err := clone(req, tmpRepoPath) + if err != nil { + return fmt.Errorf("error downloading package %s: %w", req, err) + } + + existing := modFile.search(req) + + // requirement is new, so we should move the files and add it to the module.cue + if existing == nil { + if err := move(req, tmpRepoPath, destBasePath); err != nil { + return err + } + modFile.require = append(modFile.require, req) + return os.RemoveAll(tmpRepoPath) + } + + c, err := compareVersions(existing.version, req.version) + if err != nil { + return err + } + + // the existing requirement is newer so we skip installation + if c > 0 { + return os.RemoveAll(tmpRepoPath) + } + + // the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing + // requirement and replace the code in the /pkg folder + existing.version = req.version + if err = r.checkout(req.version); err != nil { + return err + } + + return replace(req, tmpRepoPath, destBasePath) +} + +func compareVersions(reqV1, reqV2 string) (int, error) { + v1, err := version.NewVersion(reqV1) + if err != nil { + return 0, err + } + + v2, err := version.NewVersion(reqV2) + if err != nil { + return 0, err + } + if v1.LessThan(v2) { + return -1, nil + } + + return 1, nil +} + +func init() { + if err := viper.BindPFlags(getCmd.Flags()); err != nil { + panic(err) + } +} diff --git a/cmd/dagger/cmd/mod/get_test.go b/cmd/dagger/cmd/mod/get_test.go new file mode 100644 index 00000000..d0b96e4e --- /dev/null +++ b/cmd/dagger/cmd/mod/get_test.go @@ -0,0 +1,7 @@ +package mod + +import "testing" + +func TestUpdateModFile(t *testing.T) { + +} diff --git a/cmd/dagger/cmd/mod/repo.go b/cmd/dagger/cmd/mod/repo.go new file mode 100644 index 00000000..4970c914 --- /dev/null +++ b/cmd/dagger/cmd/mod/repo.go @@ -0,0 +1,104 @@ +package mod + +import ( + "fmt" + "os" + "path" + "sort" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/hashicorp/go-version" +) + +type repo struct { + localPath string + contents *git.Repository +} + +func clone(require *require, dir string) (*repo, error) { + r, err := git.PlainClone(dir, false, &git.CloneOptions{ + URL: require.cloneUrl(), + }) + if err != nil { + return nil, err + } + + rr := &repo{ + localPath: dir, + contents: r, + } + + if require.version == "" { + require.version, err = rr.latestTag() + if err != nil { + return nil, err + } + } + + if err := rr.checkout(require.version); err != nil { + return nil, err + } + + if _, err := os.Stat(path.Join(dir, require.path, "cue.mod", "module.cue")); err != nil { + return nil, fmt.Errorf("repo does not contain cue.mod/module.cue") + } + + return rr, nil +} + +func (r *repo) checkout(version string) error { + h, err := r.contents.ResolveRevision(plumbing.Revision(version)) + if err != nil { + return err + } + + w, err := r.contents.Worktree() + if err != nil { + return err + } + + err = w.Checkout(&git.CheckoutOptions{ + Hash: *h, + }) + if err != nil { + return err + } + + return nil +} + +func (r *repo) listTags() ([]string, error) { + iter, err := r.contents.Tags() + if err != nil { + return nil, err + } + + var tags []string + if err := iter.ForEach(func(ref *plumbing.Reference) error { + tags = append(tags, ref.Name().Short()) + return nil + }); err != nil { + return nil, err + } + + return tags, nil +} + +func (r *repo) latestTag() (string, error) { + versionsRaw, err := r.listTags() + if err != nil { + return "", err + } + + versions := make([]*version.Version, len(versionsRaw)) + for i, raw := range versionsRaw { + v, _ := version.NewVersion(raw) + versions[i] = v + } + + // After this, the versions are properly sorted + sort.Sort(version.Collection(versions)) + + return versions[len(versions)-1].Original(), nil +} diff --git a/cmd/dagger/cmd/mod/repo_test.go b/cmd/dagger/cmd/mod/repo_test.go new file mode 100644 index 00000000..94078345 --- /dev/null +++ b/cmd/dagger/cmd/mod/repo_test.go @@ -0,0 +1,85 @@ +package mod + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestClone(t *testing.T) { + cases := []struct { + name string + require require + }{ + { + name: "resolving shorter hash version", + require: require{ + prefix: "https://", + repo: "github.com/tjovicic/gcpcloudrun-cue", + path: "", + version: "5839b7b432b8b0c", + }, + }, + { + name: "resolving branch name", + require: require{ + prefix: "https://", + repo: "github.com/tjovicic/gcpcloudrun-cue", + path: "", + version: "main", + }, + }, + { + name: "resolving tag", + require: require{ + prefix: "https://", + repo: "github.com/tjovicic/gcpcloudrun-cue", + path: "", + version: "v0.1", + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "clone") + if err != nil { + t.Fatal("error creating tmp dir") + } + + defer os.Remove(tmpDir) + + _, err = clone(&c.require, tmpDir) + if err != nil { + t.Error(err) + } + }) + } +} + +func TestListTags(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "clone") + if err != nil { + t.Fatal("error creating tmp dir") + } + defer os.Remove(tmpDir) + + r, err := clone(&require{ + prefix: "https://", + repo: "github.com/cuelang/cue", + path: "", + version: "", + }, tmpDir) + if err != nil { + t.Error(err) + } + + tags, err := r.listTags() + if err != nil { + t.Error(err) + } + + if len(tags) == 0 { + t.Errorf("could not list repo tags") + } +} diff --git a/cmd/dagger/cmd/mod/root.go b/cmd/dagger/cmd/mod/root.go new file mode 100644 index 00000000..1f12ca18 --- /dev/null +++ b/cmd/dagger/cmd/mod/root.go @@ -0,0 +1,17 @@ +package mod + +import "github.com/spf13/cobra" + + +// Cmd exposes the top-level command +var Cmd = &cobra.Command{ + Use: "mod", + Short: "Manage an environment's dependencies", +} + +func init() { + Cmd.AddCommand( + getCmd, + ) +} + diff --git a/cmd/dagger/cmd/root.go b/cmd/dagger/cmd/root.go index d3ade03b..705630c5 100644 --- a/cmd/dagger/cmd/root.go +++ b/cmd/dagger/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "go.dagger.io/dagger/cmd/dagger/cmd/mod" "os" "strings" @@ -59,6 +60,7 @@ func init() { output.Cmd, versionCmd, docCmd, + mod.Cmd, ) if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil { diff --git a/util/buildkitd/buildkitd.go b/util/buildkitd/buildkitd.go index 6a111d30..64563774 100644 --- a/util/buildkitd/buildkitd.go +++ b/util/buildkitd/buildkitd.go @@ -29,22 +29,24 @@ const ( func init() { bi, ok := debug.ReadBuildInfo() if !ok { - panic("unable to retrieve build info") + return } + for _, d := range bi.Deps { if d.Path == "github.com/moby/buildkit" { vendoredVersion = d.Version break } } - if vendoredVersion == "" { - panic("failed to solve vendored buildkit version") - } } func Start(ctx context.Context) (string, error) { lg := log.Ctx(ctx) + if vendoredVersion == "" { + return "", fmt.Errorf("vendored version is empty") + } + // Attempt to detect the current buildkit version currentVersion, err := getBuildkitVersion(ctx) if err != nil { @@ -66,7 +68,7 @@ func Start(ctx context.Context) (string, error) { Info(). Str("version", vendoredVersion). Msg("upgrading buildkit") - if err := remvoveBuildkit(ctx); err != nil { + if err := removeBuildkit(ctx); err != nil { return "", err } } else { @@ -183,7 +185,7 @@ func waitBuildkit(ctx context.Context) error { return errors.New("buildkit failed to respond") } -func remvoveBuildkit(ctx context.Context) error { +func removeBuildkit(ctx context.Context) error { lg := log. Ctx(ctx)