Merge pull request #1076 from samalba/remote-stdlib

Support version constraints on remote modules

cmd/dagger/cmd/mod/get.go

@@ -38,13 +38,13 @@ var getCmd = &cobra.Command{
 		var err error
 		if update && len(args) == 0 {
 			lg.Info().Msg("updating all installed packages...")
-			processedRequires, err = mod.UpdateInstalled(project.Path)
+			processedRequires, err = mod.UpdateInstalled(ctx, project.Path)
 		} else if update && len(args) > 0 {
 			lg.Info().Msg("updating specified packages...")
-			processedRequires, err = mod.UpdateAll(project.Path, args)
+			processedRequires, err = mod.UpdateAll(ctx, project.Path, args)
 		} else if !update && len(args) > 0 {
 			lg.Info().Msg("installing specified packages...")
-			processedRequires, err = mod.InstallAll(project.Path, args)
+			processedRequires, err = mod.InstallAll(ctx, project.Path, args)
 		} else {
 			lg.Fatal().Msg("unrecognized update/install operation")
 		}

mod/file.go

@@ -2,6 +2,7 @@ package mod
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -15,11 +16,13 @@ import (
 	"github.com/spf13/viper"
 )
 
-const modFilePath = "./cue.mod/dagger.mod"
+const (
-const sumFilePath = "./cue.mod/dagger.sum"
+	modFilePath  = "./cue.mod/dagger.mod"
-const lockFilePath = "./cue.mod/dagger.lock"
+	sumFilePath  = "./cue.mod/dagger.sum"
-const destBasePath = "./cue.mod/pkg"
+	lockFilePath = "./cue.mod/dagger.lock"
-const tmpBasePath = "./cue.mod/tmp"
+	destBasePath = "./cue.mod/pkg"
+	tmpBasePath  = "./cue.mod/tmp"
+)
 
 // file is the parsed, interpreted form of dagger.mod file.
 type file struct {
@@ -98,7 +101,8 @@ func read(fMod, fSum io.Reader) (*file, error) {
 			return nil, fmt.Errorf("repos in mod and sum line don't match: %s and %s", modSplit[0], sumSplit[0])
 		}
 
-		require, err := newRequire(modSplit[0])
+		// FIXME: if we want to add support for version constraints in the mod file, it would be here
+		require, err := newRequire(modSplit[0], "")
 		if err != nil {
 			return nil, err
 		}
@@ -126,20 +130,19 @@ func nonEmptyLines(b []byte) []string {
 		}
 
 		trimmed = spaceRgx.ReplaceAllString(trimmed, " ")
-
 		lines = append(lines, trimmed)
 	}
 
 	return lines
 }
 
-func (f *file) install(req *Require) error {
+func (f *file) install(ctx context.Context, req *Require) error {
 	// cleaning up possible leftovers
 	tmpPath := path.Join(f.workspacePath, tmpBasePath, req.fullPath())
 	defer os.RemoveAll(tmpPath)
 
 	// clone to a tmp directory
-	r, err := clone(req, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
+	r, err := clone(ctx, req, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
 	if err != nil {
 		return fmt.Errorf("error downloading package %s: %w", req, err)
 	}
@@ -168,7 +171,7 @@ func (f *file) install(req *Require) error {
 	// checkout the cloned repo to that tag, change the version in the existing requirement and
 	// replace the code in the /pkg folder
 	existing.version = req.version
-	if err = r.checkout(req.version); err != nil {
+	if err = r.checkout(ctx, req.version); err != nil {
 		return err
 	}
 
@@ -186,7 +189,7 @@ func (f *file) install(req *Require) error {
 	return nil
 }
 
-func (f *file) updateToLatest(req *Require) (*Require, error) {
+func (f *file) updateToLatest(ctx context.Context, req *Require) (*Require, error) {
 	// check if it doesn't exist
 	existing := f.searchInstalledRequire(req)
 	if existing == nil {
@@ -198,13 +201,13 @@ func (f *file) updateToLatest(req *Require) (*Require, error) {
 	defer os.RemoveAll(tmpPath)
 
 	// clone to a tmp directory
-	gitRepo, err := clone(existing, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
+	gitRepo, err := clone(ctx, existing, tmpPath, viper.GetString("private-key-file"), viper.GetString("private-key-password"))
 	if err != nil {
 		return nil, fmt.Errorf("error downloading package %s: %w", existing, err)
 	}
 
 	// checkout the latest tag
-	latestTag, err := gitRepo.latestTag()
+	latestTag, err := gitRepo.latestTag(ctx, req.versionConstraint)
 	if err != nil {
 		return nil, err
 	}
@@ -219,7 +222,7 @@ func (f *file) updateToLatest(req *Require) (*Require, error) {
 	}
 
 	existing.version = latestTag
-	if err = gitRepo.checkout(existing.version); err != nil {
+	if err = gitRepo.checkout(ctx, existing.version); err != nil {
 		return nil, err
 	}
 

mod/mod.go

@@ -1,19 +1,32 @@
 package mod
 
 import (
+	"context"
 	"path"
+	"strings"
 
 	"github.com/gofrs/flock"
+	"github.com/rs/zerolog/log"
 )
 
-func InstallStdlib(workspace string) error {
+const (
-	_, err := Install(workspace, "alpha.dagger.io@v0.1")
+	UniverseVersionConstraint = ">= 0.1.0, < 0.2"
+)
 
-	return err
+func isUniverse(repoName string) bool {
+	return strings.HasPrefix(strings.ToLower(repoName), "alpha.dagger.io")
 }
 
-func Install(workspace, repoName string) (*Require, error) {
+func Install(ctx context.Context, workspace, repoName, versionConstraint string) (*Require, error) {
-	require, err := newRequire(repoName)
+	lg := log.Ctx(ctx)
+
+	if isUniverse(repoName) {
+		// override versionConstraint to lock the version of universe we vendor
+		versionConstraint = UniverseVersionConstraint
+	}
+
+	lg.Debug().Str("name", repoName).Msg("installing module")
+	require, err := newRequire(repoName, versionConstraint)
 	if err != nil {
 		return nil, err
 	}
@@ -28,7 +41,7 @@ func Install(workspace, repoName string) (*Require, error) {
 		return nil, err
 	}
 
-	err = modfile.install(require)
+	err = modfile.install(ctx, require)
 	if err != nil {
 		return nil, err
 	}
@@ -44,14 +57,14 @@ func Install(workspace, repoName string) (*Require, error) {
 	return require, nil
 }
 
-func InstallAll(workspace string, repoNames []string) ([]*Require, error) {
+func InstallAll(ctx context.Context, workspace string, repoNames []string) ([]*Require, error) {
 	installedRequires := make([]*Require, 0, len(repoNames))
 	var err error
 
 	for _, repoName := range repoNames {
 		var require *Require
 
-		if require, err = Install(workspace, repoName); err != nil {
+		if require, err = Install(ctx, workspace, repoName, ""); err != nil {
 			continue
 		}
 
@@ -61,8 +74,16 @@ func InstallAll(workspace string, repoNames []string) ([]*Require, error) {
 	return installedRequires, err
 }
 
-func Update(workspace, repoName string) (*Require, error) {
+func Update(ctx context.Context, workspace, repoName, versionConstraint string) (*Require, error) {
-	require, err := newRequire(repoName)
+	lg := log.Ctx(ctx)
+
+	if isUniverse(repoName) {
+		// override versionConstraint to lock the version of universe we vendor
+		versionConstraint = UniverseVersionConstraint
+	}
+
+	lg.Debug().Str("name", repoName).Msg("updating module")
+	require, err := newRequire(repoName, versionConstraint)
 	if err != nil {
 		return nil, err
 	}
@@ -77,7 +98,7 @@ func Update(workspace, repoName string) (*Require, error) {
 		return nil, err
 	}
 
-	updatedRequire, err := modfile.updateToLatest(require)
+	updatedRequire, err := modfile.updateToLatest(ctx, require)
 	if err != nil {
 		return nil, err
 	}
@@ -93,14 +114,14 @@ func Update(workspace, repoName string) (*Require, error) {
 	return updatedRequire, nil
 }
 
-func UpdateAll(workspace string, repoNames []string) ([]*Require, error) {
+func UpdateAll(ctx context.Context, workspace string, repoNames []string) ([]*Require, error) {
 	updatedRequires := make([]*Require, 0, len(repoNames))
 	var err error
 
 	for _, repoName := range repoNames {
 		var require *Require
 
-		if require, err = Update(workspace, repoName); err != nil {
+		if require, err = Update(ctx, workspace, repoName, ""); err != nil {
 			continue
 		}
 
@@ -110,7 +131,7 @@ func UpdateAll(workspace string, repoNames []string) ([]*Require, error) {
 	return updatedRequires, err
 }
 
-func UpdateInstalled(workspace string) ([]*Require, error) {
+func UpdateInstalled(ctx context.Context, workspace string) ([]*Require, error) {
 	modfile, err := readPath(workspace)
 	if err != nil {
 		return nil, err
@@ -122,7 +143,7 @@ func UpdateInstalled(workspace string) ([]*Require, error) {
 		repoNames = append(repoNames, require.String())
 	}
 
-	return UpdateAll(workspace, repoNames)
+	return UpdateAll(ctx, workspace, repoNames)
 }
 
 func Ensure(workspace string) error {

mod/repo.go

@@ -1,12 +1,14 @@
 package mod
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"sort"
 	"strings"
 
 	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
+	"github.com/rs/zerolog/log"
 
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
@@ -15,9 +17,10 @@ import (
 
 type repo struct {
 	contents *git.Repository
+	require  *Require
 }
 
-func clone(require *Require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) {
+func clone(ctx context.Context, require *Require, dir string, privateKeyFile, privateKeyPassword string) (*repo, error) {
 	if err := os.RemoveAll(dir); err != nil {
 		return nil, fmt.Errorf("error cleaning up tmp directory")
 	}
@@ -46,10 +49,11 @@ func clone(require *Require, dir string, privateKeyFile, privateKeyPassword stri
 
 	rr := &repo{
 		contents: r,
+		require:  require,
 	}
 
 	if require.version == "" {
-		latestTag, err := rr.latestTag()
+		latestTag, err := rr.latestTag(ctx, require.versionConstraint)
 		if err != nil {
 			return nil, err
 		}
@@ -57,19 +61,23 @@ func clone(require *Require, dir string, privateKeyFile, privateKeyPassword stri
 		require.version = latestTag
 	}
 
-	if err := rr.checkout(require.version); err != nil {
+	if err := rr.checkout(ctx, require.version); err != nil {
 		return nil, err
 	}
 
 	return rr, nil
 }
 
-func (r *repo) checkout(version string) error {
+func (r *repo) checkout(ctx context.Context, version string) error {
+	lg := log.Ctx(ctx)
+
 	h, err := r.contents.ResolveRevision(plumbing.Revision(version))
 	if err != nil {
 		return err
 	}
 
+	lg.Debug().Str("repository", r.require.repo).Str("version", version).Str("commit", h.String()).Msg("checkout repo")
+
 	w, err := r.contents.Worktree()
 	if err != nil {
 		return err
@@ -85,25 +93,61 @@ func (r *repo) checkout(version string) error {
 	return nil
 }
 
-func (r *repo) listTags() ([]string, error) {
+func (r *repo) listTagVersions(ctx context.Context, versionConstraint string) ([]string, error) {
+	lg := log.Ctx(ctx).With().
+		Str("repository", r.require.repo).
+		Str("versionConstraint", versionConstraint).
+		Logger()
+
+	if versionConstraint == "" {
+		versionConstraint = ">= 0"
+	}
+
+	constraint, err := version.NewConstraint(versionConstraint)
+	if err != nil {
+		return nil, err
+	}
+
 	iter, err := r.contents.Tags()
 	if err != nil {
 		return nil, err
 	}
 
 	var tags []string
-	if err := iter.ForEach(func(ref *plumbing.Reference) error {
+	err = iter.ForEach(func(ref *plumbing.Reference) error {
-		tags = append(tags, ref.Name().Short())
+		tagV := ref.Name().Short()
+
+		if !strings.HasPrefix(tagV, "v") {
+			lg.Debug().Str("tag", tagV).Msg("tag version ignored, wrong format")
+			return nil
+		}
+
+		v, err := version.NewVersion(tagV)
+		if err != nil {
+			lg.Debug().Str("tag", tagV).Err(err).Msg("tag version ignored, parsing error")
+			return nil
+		}
+
+		if constraint.Check(v) {
+			// Add tag if it matches the version constraint
+			tags = append(tags, ref.Name().Short())
+			lg.Debug().Str("tag", tagV).Msg("version added")
+		} else {
+			lg.Debug().Str("tag", tagV).Msg("tag version ignored, does not satisfy constraint")
+		}
+
 		return nil
-	}); err != nil {
+	})
+
+	if err != nil {
 		return nil, err
 	}
 
 	return tags, nil
 }
 
-func (r *repo) latestTag() (string, error) {
+func (r *repo) latestTag(ctx context.Context, versionConstraint string) (string, error) {
-	versionsRaw, err := r.listTags()
+	versionsRaw, err := r.listTagVersions(ctx, versionConstraint)
 	if err != nil {
 		return "", err
 	}
@@ -115,10 +159,11 @@ func (r *repo) latestTag() (string, error) {
 	}
 
 	if len(versions) == 0 {
-		return "", fmt.Errorf("repo doesn't have any tags")
+		return "", fmt.Errorf("repo doesn't have any tags matching the required version")
 	}
 
-	sort.Sort(version.Collection(versions))
+	sort.Sort(sort.Reverse(version.Collection(versions)))
+	version := versions[0].Original()
 
-	return versions[len(versions)-1].Original(), nil
+	return version, nil
 }

mod/repo_test.go

@@ -1,6 +1,7 @@
 package mod
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -50,7 +51,7 @@ func TestClone(t *testing.T) {
 
 			defer os.Remove(tmpDir)
 
-			_, err = clone(&c.require, tmpDir, c.privateKeyFile, c.privateKeyPassword)
+			_, err = clone(context.TODO(), &c.require, tmpDir, c.privateKeyFile, c.privateKeyPassword)
 			if err != nil {
 				t.Error(err)
 			}
@@ -65,7 +66,9 @@ func TestListTags(t *testing.T) {
 	}
 	defer os.Remove(tmpDir)
 
-	r, err := clone(&Require{
+	ctx := context.TODO()
+
+	r, err := clone(ctx, &Require{
 		cloneRepo: "github.com/dagger/universe",
 		clonePath: "stdlib",
 		version:   "",
@@ -74,7 +77,7 @@ func TestListTags(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	tags, err := r.listTags()
+	tags, err := r.listTagVersions(ctx, "")
 	if err != nil {
 		t.Error(err)
 	}
@@ -83,3 +86,38 @@ func TestListTags(t *testing.T) {
 		t.Errorf("could not list repo tags")
 	}
 }
+
+func TestVersionConstraint(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "clone")
+	if err != nil {
+		t.Fatal("error creating tmp dir")
+	}
+	defer os.Remove(tmpDir)
+
+	ctx := context.TODO()
+
+	r, err := clone(ctx, &Require{
+		cloneRepo: "github.com/dagger/universe",
+		clonePath: "stdlib",
+		version:   "",
+	}, tmpDir, "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tagVersion, err := r.latestTag(ctx, "<= 0.1.0")
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Make sure we select the right version based on constraint
+	if tagVersion != "v0.1.0" {
+		t.Errorf("wrong version: expected 0.1.0, got %v", tagVersion)
+	}
+
+	// Make sure an invalid constraint (version out of range) returns an error
+	_, err = r.latestTag(ctx, "> 99999")
+	if err == nil {
+		t.Error("selected wrong version based on constraint")
+	}
+}

mod/require.go

@@ -15,16 +15,17 @@ type Require struct {
 	cloneRepo string
 	clonePath string
 
-	version  string
+	version           string
-	checksum string
+	versionConstraint string
+	checksum          string
 }
 
-func newRequire(repoName string) (*Require, error) {
+func newRequire(repoName, versionConstraint string) (*Require, error) {
 	switch {
 	case strings.HasPrefix(repoName, "github.com"):
-		return parseGithubRepoName(repoName)
+		return parseGithubRepoName(repoName, versionConstraint)
 	case strings.HasPrefix(repoName, "alpha.dagger.io"):
-		return parseDaggerRepoName(repoName)
+		return parseDaggerRepoName(repoName, versionConstraint)
 	default:
 		return nil, fmt.Errorf("repo name does not match suported providers")
 	}
@@ -32,7 +33,7 @@ func newRequire(repoName string) (*Require, error) {
 
 var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
 
-func parseGithubRepoName(repoName string) (*Require, error) {
+func parseGithubRepoName(repoName, versionConstraint string) (*Require, error) {
 	repoMatches := githubRepoNameRegex.FindStringSubmatch(repoName)
 
 	if len(repoMatches) < 4 {
@@ -40,9 +41,10 @@ func parseGithubRepoName(repoName string) (*Require, error) {
 	}
 
 	return &Require{
-		repo:    repoMatches[1],
+		repo:              repoMatches[1],
-		path:    repoMatches[2],
+		path:              repoMatches[2],
-		version: repoMatches[3],
+		version:           repoMatches[3],
+		versionConstraint: versionConstraint,
 
 		cloneRepo: repoMatches[1],
 		clonePath: repoMatches[2],
@@ -51,7 +53,7 @@ func parseGithubRepoName(repoName string) (*Require, error) {
 
 var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
 
-func parseDaggerRepoName(repoName string) (*Require, error) {
+func parseDaggerRepoName(repoName, versionConstraint string) (*Require, error) {
 	repoMatches := daggerRepoNameRegex.FindStringSubmatch(repoName)
 
 	if len(repoMatches) < 3 {
@@ -59,9 +61,10 @@ func parseDaggerRepoName(repoName string) (*Require, error) {
 	}
 
 	return &Require{
-		repo:    "alpha.dagger.io",
+		repo:              "alpha.dagger.io",
-		path:    repoMatches[1],
+		path:              repoMatches[1],
-		version: repoMatches[2],
+		version:           repoMatches[2],
+		versionConstraint: versionConstraint,
 
 		cloneRepo: "github.com/dagger/universe",
 		clonePath: path.Join("/stdlib", repoMatches[1]),

mod/require_test.go

@@ -98,7 +98,7 @@ func TestParseArgument(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			got, err := newRequire(c.in)
+			got, err := newRequire(c.in, "")
 			if err != nil && c.hasError {
 				return
 			}

state/project.go

@@ -396,9 +396,9 @@ func vendorUniverse(ctx context.Context, p string) error {
 	}
 
 	log.Ctx(ctx).Debug().Str("mod", p).Msg("vendoring universe")
-	// FIXME(samalba): disabled install remote stdlib temporarily
-	// if err := mod.InstallStdlib(p); err != nil {
 	if err := stdlib.Vendor(ctx, p); err != nil {
+		// FIXME(samalba): disabled install remote stdlib temporarily
+		// if _, err := mod.Install(ctx, p, "alpha.dagger.io", ""); err != nil {
 		return err
 	}