First version of package manager

Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>

client/client.go

@@ -31,7 +31,7 @@ import (
 	"go.dagger.io/dagger/state"
 )
 
-// A dagger client
+// Client is a dagger client
 type Client struct {
 	c       *bk.Client
 	noCache bool

cmd/dagger/cmd/mod/file.go

@@ -0,0 +1,205 @@
+package mod
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// A file is the parsed, interpreted form of a cue.mod file.
+type file struct {
+	module  string
+	require []*require
+}
+
+func (f *file) contents() *bytes.Buffer {
+	var b bytes.Buffer
+
+	b.WriteString(fmt.Sprintf("module: %s\n\n", f.module))
+	for _, r := range f.require {
+		b.WriteString(fmt.Sprintf("%s %s\n", r.fullPath(), r.version))
+	}
+
+	return &b
+}
+
+func (f *file) search(r *require) *require {
+	for _, existing := range f.require {
+		if existing.fullPath() == r.fullPath() {
+			return existing
+		}
+	}
+	return nil
+}
+
+type require struct {
+	prefix  string
+	repo    string
+	path    string
+	version string
+}
+
+func (r *require) cloneUrl() string {
+	return fmt.Sprintf("%s%s", r.prefix, r.repo)
+}
+
+func (r *require) fullPath() string {
+	return path.Join(r.repo, r.path)
+}
+
+func read(f io.Reader) (*file, error) {
+	b, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+
+	lines, err := nonEmptyLines(b)
+	if err != nil {
+		return nil, err
+	}
+
+	var module string
+	if split := strings.Split(lines[0], " "); len(split) > 1 {
+		module = strings.Trim(split[1], "\"")
+	}
+
+	var requires []*require
+	for i := 1; i < len(lines); i++ {
+		split := strings.Split(lines[i], " ")
+		r, err := parseArgument(split[0])
+		if err != nil {
+			return nil, err
+		}
+
+		r.version = split[1]
+
+		requires = append(requires, r)
+	}
+
+	return &file{
+		module:  module,
+		require: requires,
+	}, nil
+}
+
+func nonEmptyLines(b []byte) ([]string, error) {
+	s := strings.ReplaceAll(string(b), "\r\n", "\n")
+	split := strings.Split(s, "\n")
+
+	spaceRgx, err := regexp.Compile(`\s+`)
+	if err != nil {
+		return nil, err
+	}
+
+	var lines []string
+	for _, l := range split {
+		trimmed := strings.TrimSpace(l)
+		if trimmed == "" {
+			continue
+		}
+
+		trimmed = spaceRgx.ReplaceAllString(trimmed, " ")
+
+		lines = append(lines, trimmed)
+	}
+
+	return lines, nil
+}
+
+func parseArgument(arg string) (*require, error) {
+	if strings.HasPrefix(arg, "alpha.dagger.io") {
+		arg = strings.Replace(arg, "alpha.dagger.io", "github.com/dagger/dagger/stdlib", 1)
+	}
+
+	name, suffix, err := parseGithubRepoName(arg)
+	if err != nil {
+		return nil, err
+	}
+
+	repoPath, version, err := parseGithubRepoVersion(suffix)
+	if err != nil {
+		return nil, err
+	}
+
+	return &require{
+		prefix:  "https://",
+		repo:    name,
+		path:    repoPath,
+		version: version,
+	}, nil
+}
+
+func parseGithubRepoName(arg string) (string, string, error) {
+	repoRegex, err := regexp.Compile("(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)(.*)")
+	if err != nil {
+		return "", "", err
+	}
+
+	repoMatches := repoRegex.FindStringSubmatch(arg)
+
+	if len(repoMatches) == 0 {
+		return "", "", fmt.Errorf("repo name does not match suported providers")
+	}
+
+	// returns 2 elements: repo name and path+version
+	return repoMatches[1], repoMatches[2], nil
+}
+
+func parseGithubRepoVersion(repoSuffix string) (string, string, error) {
+	if repoSuffix == "" {
+		return "", "", nil
+	}
+
+	i := strings.LastIndexAny(repoSuffix, "@:")
+	if i == -1 {
+		return repoSuffix, "", nil
+	}
+
+	return repoSuffix[:i], repoSuffix[i+1:], nil
+}
+
+func readModFile() (*file, error) {
+	f, err := os.Open("./cue.mod/module.cue")
+	if err != nil {
+		return nil, err
+	}
+
+	modFile, err := read(f)
+	if err != nil {
+		return nil, err
+	}
+
+	return modFile, nil
+}
+
+func writeModFile(f *file) error {
+	return ioutil.WriteFile("./cue.mod/module.cue", f.contents().Bytes(), 0644)
+}
+
+func move(r *require, sourceRepoPath, destBasePath string) error {
+	fmt.Println("move")
+	destPath := path.Join(destBasePath, r.fullPath())
+	if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
+		return err
+	}
+	if err := os.Rename(path.Join(sourceRepoPath, r.path), destPath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func replace(r *require, sourceRepoPath, destBasePath string) error {
+	fmt.Println("replace")
+	if err := os.RemoveAll(path.Join(destBasePath, r.fullPath())); err != nil {
+		return err
+	}
+
+	return move(r, sourceRepoPath, destBasePath)
+}

cmd/dagger/cmd/mod/file_test.go

@@ -0,0 +1,152 @@
+package mod
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseArgument(t *testing.T) {
+	cases := []struct {
+		name     string
+		in       string
+		want     *require
+		hasError bool
+	}{
+		{
+			name:     "Random",
+			in:       "abcd/bla@:/xyz",
+			hasError: true,
+		},
+		{
+			name: "Dagger repo",
+			in:   "github.com/dagger/dagger",
+			want: &require{
+				repo:    "github.com/dagger/dagger",
+				path:    "",
+				version: "",
+			},
+		},
+		{
+			name: "Dagger repo with path",
+			in:   "github.com/dagger/dagger/stdlib",
+			want: &require{
+				repo:    "github.com/dagger/dagger",
+				path:    "/stdlib",
+				version: "",
+			},
+		},
+		{
+			name: "Dagger repo with longer path",
+			in:   "github.com/dagger/dagger/stdlib/test/test",
+			want: &require{
+				repo:    "github.com/dagger/dagger",
+				path:    "/stdlib/test/test",
+				version: "",
+			},
+		},
+		{
+			name: "Dagger repo with path and version",
+			in:   "github.com/dagger/dagger/stdlib@v0.1",
+			want: &require{
+				repo:    "github.com/dagger/dagger",
+				path:    "/stdlib",
+				version: "v0.1",
+			},
+		},
+		{
+			name: "Dagger repo with longer path and version",
+			in:   "github.com/dagger/dagger/stdlib/test/test@v0.0.1",
+			want: &require{
+				repo:    "github.com/dagger/dagger",
+				path:    "/stdlib/test/test",
+				version: "v0.0.1",
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got, err := parseArgument(c.in)
+			if err != nil && c.hasError {
+				return
+			}
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if got.repo != c.want.repo {
+				t.Errorf("repos differ: want %s, got %s", c.want.repo, got.repo)
+			}
+
+			if got.path != c.want.path {
+				t.Errorf("paths differ: want %s, got %s", c.want.path, got.path)
+			}
+
+			if got.version != c.want.version {
+				t.Errorf("versions differ: want %s, got %s", c.want.version, got.version)
+			}
+		})
+	}
+}
+
+func TestReadFile(t *testing.T) {
+	cases := []struct {
+		name  string
+		input string
+		want  *file
+	}{
+		{
+			name: "module file without dependencies",
+			input: `
+				module: "alpha.dagger.io"
+			`,
+			want: &file{
+				module: "alpha.dagger.io",
+			},
+		},
+		{
+			name: "module file with valid dependencies",
+			input: `
+				module: "alpha.dagger.io"
+	
+				github.com/tjovicic/test xyz
+				github.com/bla/bla abc
+			`,
+			want: &file{
+				module: "alpha.dagger.io",
+				require: []*require{
+					{
+						prefix:  "https://",
+						repo:    "github.com/tjovicic/test",
+						path:    "",
+						version: "xyz",
+					},
+					{
+						prefix:  "https://",
+						repo:    "github.com/bla/bla",
+						path:    "",
+						version: "abc",
+					},
+				},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got, err := read(strings.NewReader(c.input))
+			if err != nil {
+				t.Error(err)
+			}
+
+			if got.module != c.want.module {
+				t.Errorf("module names differ: want %s, got %s", c.want.module, got.module)
+			}
+
+			if len(got.require) != len(c.want.require) {
+				t.Errorf("requires length differs: want %d, got %d", len(c.want.require), len(got.require))
+			}
+		})
+	}
+}

cmd/dagger/cmd/mod/get.go

@@ -0,0 +1,135 @@
+package mod
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/hashicorp/go-version"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"go.dagger.io/dagger/cmd/dagger/cmd/common"
+	"go.dagger.io/dagger/cmd/dagger/logger"
+)
+
+var getCmd = &cobra.Command{
+	Use:   "get [packages]",
+	Short: "download and install packages and dependencies",
+	Args:  cobra.MaximumNArgs(1),
+	PreRun: func(cmd *cobra.Command, args []string) {
+		// Fix Viper bug for duplicate flags:
+		// https://github.com/spf13/viper/issues/233
+		if err := viper.BindPFlags(cmd.Flags()); err != nil {
+			panic(err)
+		}
+	},
+
+	Run: func(cmd *cobra.Command, args []string) {
+		lg := logger.New()
+		ctx := lg.WithContext(cmd.Context())
+
+		doneCh := common.TrackCommand(ctx, cmd)
+
+		if len(args) == 0 {
+			lg.Fatal().Msg("need to specify package name in command argument")
+		}
+
+		// parse packages to install
+		var packages []*require
+		for _, arg := range args {
+			p, err := parseArgument(arg)
+			if err != nil {
+				lg.Error().Err(err).Msgf("error parsing package %s", arg)
+				continue
+			}
+
+			packages = append(packages, p)
+		}
+
+		// read mod file in the current dir
+		modFile, err := readModFile()
+		if err != nil {
+			lg.Fatal().Err(err).Msgf("error loading module file")
+		}
+
+		// download packages
+		destBasePath := "./cue.mod/pkg"
+		for _, p := range packages {
+			if err := processRequire(p, modFile, destBasePath); err != nil {
+				lg.Error().Err(err).Msg("error processing package")
+			}
+		}
+
+		// write to mod file in the current dir
+		if err = writeModFile(modFile); err != nil {
+			lg.Error().Err(err).Msg("error writing to mod file")
+		}
+
+		<-doneCh
+	},
+}
+
+func processRequire(req *require, modFile *file, destBasePath string) error {
+	tmpRepoPath := path.Join("./cue.mod/tmp", req.repo)
+	if err := os.MkdirAll(tmpRepoPath, 0755); err != nil {
+		return fmt.Errorf("error creating tmp dir for cloning package")
+	}
+
+	r, err := clone(req, tmpRepoPath)
+	if err != nil {
+		return fmt.Errorf("error downloading package %s: %w", req, err)
+	}
+
+	existing := modFile.search(req)
+
+	// requirement is new, so we should move the files and add it to the module.cue
+	if existing == nil {
+		if err := move(req, tmpRepoPath, destBasePath); err != nil {
+			return err
+		}
+		modFile.require = append(modFile.require, req)
+		return os.RemoveAll(tmpRepoPath)
+	}
+
+	c, err := compareVersions(existing.version, req.version)
+	if err != nil {
+		return err
+	}
+
+	// the existing requirement is newer so we skip installation
+	if c > 0 {
+		return os.RemoveAll(tmpRepoPath)
+	}
+
+	// the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing
+	// requirement and replace the code in the /pkg folder
+	existing.version = req.version
+	if err = r.checkout(req.version); err != nil {
+		return err
+	}
+
+	return replace(req, tmpRepoPath, destBasePath)
+}
+
+func compareVersions(reqV1, reqV2 string) (int, error) {
+	v1, err := version.NewVersion(reqV1)
+	if err != nil {
+		return 0, err
+	}
+
+	v2, err := version.NewVersion(reqV2)
+	if err != nil {
+		return 0, err
+	}
+	if v1.LessThan(v2) {
+		return -1, nil
+	}
+
+	return 1, nil
+}
+
+func init() {
+	if err := viper.BindPFlags(getCmd.Flags()); err != nil {
+		panic(err)
+	}
+}

cmd/dagger/cmd/mod/get_test.go

@@ -0,0 +1,7 @@
+package mod
+
+import "testing"
+
+func TestUpdateModFile(t *testing.T) {
+
+}

cmd/dagger/cmd/mod/repo.go

@@ -0,0 +1,104 @@
+package mod
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"sort"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/hashicorp/go-version"
+)
+
+type repo struct {
+	localPath string
+	contents  *git.Repository
+}
+
+func clone(require *require, dir string) (*repo, error) {
+	r, err := git.PlainClone(dir, false, &git.CloneOptions{
+		URL: require.cloneUrl(),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	rr := &repo{
+		localPath: dir,
+		contents:  r,
+	}
+
+	if require.version == "" {
+		require.version, err = rr.latestTag()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if err := rr.checkout(require.version); err != nil {
+		return nil, err
+	}
+
+	if _, err := os.Stat(path.Join(dir, require.path, "cue.mod", "module.cue")); err != nil {
+		return nil, fmt.Errorf("repo does not contain cue.mod/module.cue")
+	}
+
+	return rr, nil
+}
+
+func (r *repo) checkout(version string) error {
+	h, err := r.contents.ResolveRevision(plumbing.Revision(version))
+	if err != nil {
+		return err
+	}
+
+	w, err := r.contents.Worktree()
+	if err != nil {
+		return err
+	}
+
+	err = w.Checkout(&git.CheckoutOptions{
+		Hash: *h,
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *repo) listTags() ([]string, error) {
+	iter, err := r.contents.Tags()
+	if err != nil {
+		return nil, err
+	}
+
+	var tags []string
+	if err := iter.ForEach(func(ref *plumbing.Reference) error {
+		tags = append(tags, ref.Name().Short())
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return tags, nil
+}
+
+func (r *repo) latestTag() (string, error) {
+	versionsRaw, err := r.listTags()
+	if err != nil {
+		return "", err
+	}
+
+	versions := make([]*version.Version, len(versionsRaw))
+	for i, raw := range versionsRaw {
+		v, _ := version.NewVersion(raw)
+		versions[i] = v
+	}
+
+	// After this, the versions are properly sorted
+	sort.Sort(version.Collection(versions))
+
+	return versions[len(versions)-1].Original(), nil
+}

cmd/dagger/cmd/mod/repo_test.go

@@ -0,0 +1,85 @@
+package mod
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestClone(t *testing.T) {
+	cases := []struct {
+		name    string
+		require require
+	}{
+		{
+			name: "resolving shorter hash version",
+			require: require{
+				prefix:  "https://",
+				repo:    "github.com/tjovicic/gcpcloudrun-cue",
+				path:    "",
+				version: "5839b7b432b8b0c",
+			},
+		},
+		{
+			name: "resolving branch name",
+			require: require{
+				prefix:  "https://",
+				repo:    "github.com/tjovicic/gcpcloudrun-cue",
+				path:    "",
+				version: "main",
+			},
+		},
+		{
+			name: "resolving tag",
+			require: require{
+				prefix:  "https://",
+				repo:    "github.com/tjovicic/gcpcloudrun-cue",
+				path:    "",
+				version: "v0.1",
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			tmpDir, err := ioutil.TempDir("", "clone")
+			if err != nil {
+				t.Fatal("error creating tmp dir")
+			}
+
+			defer os.Remove(tmpDir)
+
+			_, err = clone(&c.require, tmpDir)
+			if err != nil {
+				t.Error(err)
+			}
+		})
+	}
+}
+
+func TestListTags(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "clone")
+	if err != nil {
+		t.Fatal("error creating tmp dir")
+	}
+	defer os.Remove(tmpDir)
+
+	r, err := clone(&require{
+		prefix:  "https://",
+		repo:    "github.com/cuelang/cue",
+		path:    "",
+		version: "",
+	}, tmpDir)
+	if err != nil {
+		t.Error(err)
+	}
+
+	tags, err := r.listTags()
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(tags) == 0 {
+		t.Errorf("could not list repo tags")
+	}
+}

cmd/dagger/cmd/mod/root.go

@@ -0,0 +1,17 @@
+package mod
+
+import "github.com/spf13/cobra"
+
+
+// Cmd exposes the top-level command
+var Cmd = &cobra.Command{
+	Use:   "mod",
+	Short: "Manage an environment's dependencies",
+}
+
+func init() {
+	Cmd.AddCommand(
+		getCmd,
+	)
+}
+

cmd/dagger/cmd/root.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"go.dagger.io/dagger/cmd/dagger/cmd/mod"
 	"os"
 	"strings"
 
@@ -59,6 +60,7 @@ func init() {
 		output.Cmd,
 		versionCmd,
 		docCmd,
+		mod.Cmd,
 	)
 
 	if err := viper.BindPFlags(rootCmd.PersistentFlags()); err != nil {

util/buildkitd/buildkitd.go

@@ -29,22 +29,24 @@ const (
 func init() {
 	bi, ok := debug.ReadBuildInfo()
 	if !ok {
-		panic("unable to retrieve build info")
+		return
 	}
+
 	for _, d := range bi.Deps {
 		if d.Path == "github.com/moby/buildkit" {
 			vendoredVersion = d.Version
 			break
 		}
 	}
-	if vendoredVersion == "" {
-		panic("failed to solve vendored buildkit version")
-	}
 }
 
 func Start(ctx context.Context) (string, error) {
 	lg := log.Ctx(ctx)
 
+	if vendoredVersion == "" {
+		return "", fmt.Errorf("vendored version is empty")
+	}
+
 	// Attempt to detect the current buildkit version
 	currentVersion, err := getBuildkitVersion(ctx)
 	if err != nil {
@@ -66,7 +68,7 @@ func Start(ctx context.Context) (string, error) {
 				Info().
 				Str("version", vendoredVersion).
 				Msg("upgrading buildkit")
-			if err := remvoveBuildkit(ctx); err != nil {
+			if err := removeBuildkit(ctx); err != nil {
 				return "", err
 			}
 		} else {
@@ -183,7 +185,7 @@ func waitBuildkit(ctx context.Context) error {
 	return errors.New("buildkit failed to respond")
 }
 
-func remvoveBuildkit(ctx context.Context) error {
+func removeBuildkit(ctx context.Context) error {
 	lg := log.
 		Ctx(ctx)