kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Update package manager dependency parsing

Browse Source 
Signed-off-by: Tihomir Jovicic <tihomir.jovicic.develop@gmail.com>
This commit is contained in:
Tihomir Jovicic 2021-08-08 07:36:33 +02:00
parent be913b0e90
commit c7653dc09c
7 changed files with 282 additions and 252 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
cmd/dagger/cmd/mod/arg.go Normal file
View File

@ -0,0 +1,58 @@
package mod
import (
"fmt"
"path"
"regexp"
"strings"
)
func parseArgument(arg string) (*require, error) {
if strings.HasPrefix(arg, "github.com") {
return parseGithubRepoName(arg)
} else if strings.HasPrefix(arg, "alpha.dagger.io") {
return parseDaggerRepoName(arg)
} else {
return nil, fmt.Errorf("repo name does not match suported providers")
}
}
var githubRepoNameRegex = regexp.MustCompile(`(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseGithubRepoName(arg string) (*require, error) {
repoMatches := githubRepoNameRegex.FindStringSubmatch(arg)
if len(repoMatches) < 4 {
return nil, fmt.Errorf("issue when parsing github repo")
}
return &require{
prefix: "https://",
repo: repoMatches[1],
path: repoMatches[2],
version: repoMatches[3],
cloneRepo: repoMatches[1],
clonePath: repoMatches[2],
}, nil
}
var daggerRepoNameRegex = regexp.MustCompile(`alpha.dagger.io([a-zA-Z0-9/_.-]*)@?([0-9a-zA-Z.-]*)`)
func parseDaggerRepoName(arg string) (*require, error) {
repoMatches := daggerRepoNameRegex.FindStringSubmatch(arg)
if len(repoMatches) < 3 {
return nil, fmt.Errorf("issue when parsing dagger repo")
}
return &require{
prefix: "https://",
repo: "alpha.dagger.io",
path: repoMatches[1],
version: repoMatches[2],
cloneRepo: "github.com/dagger/dagger",
clonePath: path.Join("/stdlib", repoMatches[1]),
}, nil
}

100
cmd/dagger/cmd/mod/arg_test.go Normal file
View File

@ -0,0 +1,100 @@
package mod
import "testing"
func TestParseArgument(t *testing.T) {
cases := []struct {
name string
in string
want *require
hasError bool
}{
{
name: "Random",
in: "abcd/bla@:/xyz",
hasError: true,
},
{
name: "Dagger repo",
in: "github.com/dagger/dagger",
want: &require{
repo: "github.com/dagger/dagger",
path: "",
version: "",
},
},
{
name: "Dagger repo with path",
in: "github.com/dagger/dagger/stdlib",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib",
version: "",
},
},
{
name: "Dagger repo with longer path",
in: "github.com/dagger/dagger/stdlib/test/test",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "",
},
},
{
name: "Dagger repo with path and version",
in: "github.com/dagger/dagger/stdlib@v0.1",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib",
version: "v0.1",
},
},
{
name: "Dagger repo with longer path and version",
in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "v0.0.1",
},
},
{
name: "Alpha Dagger repo",
in: "alpha.dagger.io/gcp/gke@v0.1.0-alpha.20",
want: &require{
repo: "alpha.dagger.io",
path: "/gcp/gke",
version: "v0.1.0-alpha.20",
cloneRepo: "github.com/dagger/dagger",
clonePath: "/stdlib/gcp/gke",
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := parseArgument(c.in)
if err != nil && c.hasError {
return
}
if err != nil {
t.Fatal(err)
}
if got.repo != c.want.repo {
t.Errorf("repos differ: want %s, got %s", c.want.repo, got.repo)
}
if got.path != c.want.path {
t.Errorf("paths differ: want %s, got %s", c.want.path, got.path)
}
if got.version != c.want.version {
t.Errorf("versions differ: want %s, got %s", c.want.version, got.version)
}
})
}
}

219
cmd/dagger/cmd/mod/file.go
View File

@ -12,7 +12,9 @@ import (
"strings"
)
const FilePath = "./cue.mod/dagger.mod.cue"
const filePath = "./cue.mod/dagger.mod.cue"
const destBasePath = "./cue.mod/pkg"
const tmpBasePath = "./cue.mod/tmp"
// A file is the parsed, interpreted form of a cue.mod file.
type file struct {
@ -20,6 +22,84 @@ type file struct {
require []*require
}
func readModFile(workspacePath string) (*file, error) {
f, err := os.Open(path.Join(workspacePath, filePath))
if err != nil {
return nil, err
}
modFile, err := read(f)
if err != nil {
return nil, err
}
return modFile, nil
}
func read(f io.Reader) (*file, error) {
b, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
lines, err := nonEmptyLines(b)
if err != nil {
return nil, err
}
if len(lines) == 0 {
return nil, fmt.Errorf("mod file is empty, missing module name")
}
var module string
if split := strings.Split(lines[0], " "); len(split) > 1 {
module = strings.Trim(split[1], "\"")
}
var requires []*require
for i := 1; i < len(lines); i++ {
split := strings.Split(lines[i], " ")
r, err := parseArgument(split[0])
if err != nil {
return nil, err
}
r.version = split[1]
requires = append(requires, r)
}
return &file{
module: module,
require: requires,
}, nil
}
var spaceRgx = regexp.MustCompile(`\s+`)
func nonEmptyLines(b []byte) ([]string, error) {
s := strings.ReplaceAll(string(b), "\r\n", "\n")
split := strings.Split(s, "\n")
lines := make([]string, 0, len(split))
for _, l := range split {
trimmed := strings.TrimSpace(l)
if trimmed == "" {
continue
}
trimmed = spaceRgx.ReplaceAllString(trimmed, " ")
lines = append(lines, trimmed)
}
return lines, nil
}
func writeModFile(workspacePath string, f *file) error {
return ioutil.WriteFile(path.Join(workspacePath, filePath), f.contents().Bytes(), 0600)
}
func (f *file) contents() *bytes.Buffer {
var b bytes.Buffer
@ -45,148 +125,20 @@ type require struct {
repo string
path string
version string
cloneRepo string
clonePath string
}
func (r *require) cloneURL() string {
return fmt.Sprintf("%s%s", r.prefix, r.repo)
return fmt.Sprintf("%s%s", r.prefix, r.cloneRepo)
}
func (r *require) fullPath() string {
return path.Join(r.repo, r.path)
}
func read(f io.Reader) (*file, error) {
b, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
lines, err := nonEmptyLines(b)
if err != nil {
return nil, err
}
if len(lines) == 0 {
return nil, fmt.Errorf("mod file is empty")
}
var module string
if split := strings.Split(lines[0], " "); len(split) > 1 {
module = strings.Trim(split[1], "\"")
}
var requires []*require
for i := 1; i < len(lines); i++ {
split := strings.Split(lines[i], " ")
r, err := parseArgument(split[0])
if err != nil {
return nil, err
}
r.version = split[1]
requires = append(requires, r)
}
return &file{
module: module,
require: requires,
}, nil
}
func nonEmptyLines(b []byte) ([]string, error) {
s := strings.ReplaceAll(string(b), "\r\n", "\n")
split := strings.Split(s, "\n")
spaceRgx, err := regexp.Compile(`\s+`)
if err != nil {
return nil, err
}
var lines []string
for _, l := range split {
trimmed := strings.TrimSpace(l)
if trimmed == "" {
continue
}
trimmed = spaceRgx.ReplaceAllString(trimmed, " ")
lines = append(lines, trimmed)
}
return lines, nil
}
func parseArgument(arg string) (*require, error) {
if strings.HasPrefix(arg, "alpha.dagger.io") {
arg = strings.Replace(arg, "alpha.dagger.io", "github.com/dagger/dagger/stdlib", 1)
}
name, suffix, err := parseGithubRepoName(arg)
if err != nil {
return nil, err
}
repoPath, version := parseGithubRepoVersion(suffix)
return &require{
prefix: "https://",
repo: name,
path: repoPath,
version: version,
}, nil
}
func parseGithubRepoName(arg string) (string, string, error) {
repoRegex, err := regexp.Compile("(github.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+)(.*)")
if err != nil {
return "", "", err
}
repoMatches := repoRegex.FindStringSubmatch(arg)
if len(repoMatches) == 0 {
return "", "", fmt.Errorf("repo name does not match suported providers")
}
// returns 2 elements: repo name and path+version
return repoMatches[1], repoMatches[2], nil
}
func parseGithubRepoVersion(repoSuffix string) (string, string) {
if repoSuffix == "" {
return "", ""
}
i := strings.LastIndexAny(repoSuffix, "@:")
if i == -1 {
return repoSuffix, ""
}
return repoSuffix[:i], repoSuffix[i+1:]
}
func readModFile() (*file, error) {
f, err := os.Open(FilePath)
if err != nil {
return nil, err
}
modFile, err := read(f)
if err != nil {
return nil, err
}
return modFile, nil
}
func writeModFile(f *file) error {
return ioutil.WriteFile(FilePath, f.contents().Bytes(), 0600)
}
func move(r *require, sourceRepoPath, destBasePath string) error {
fmt.Println("move")
destPath := path.Join(destBasePath, r.fullPath())
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
return err
@ -199,7 +151,6 @@ func move(r *require, sourceRepoPath, destBasePath string) error {
}
func replace(r *require, sourceRepoPath, destBasePath string) error {
fmt.Println("replace")
if err := os.RemoveAll(path.Join(destBasePath, r.fullPath())); err != nil {
return err
}

85
cmd/dagger/cmd/mod/file_test.go
View File

@ -5,91 +5,6 @@ import (
"testing"
)
func TestParseArgument(t *testing.T) {
cases := []struct {
name string
in string
want *require
hasError bool
}{
{
name: "Random",
in: "abcd/bla@:/xyz",
hasError: true,
},
{
name: "Dagger repo",
in: "github.com/dagger/dagger",
want: &require{
repo: "github.com/dagger/dagger",
path: "",
version: "",
},
},
{
name: "Dagger repo with path",
in: "github.com/dagger/dagger/stdlib",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib",
version: "",
},
},
{
name: "Dagger repo with longer path",
in: "github.com/dagger/dagger/stdlib/test/test",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "",
},
},
{
name: "Dagger repo with path and version",
in: "github.com/dagger/dagger/stdlib@v0.1",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib",
version: "v0.1",
},
},
{
name: "Dagger repo with longer path and version",
in: "github.com/dagger/dagger/stdlib/test/test@v0.0.1",
want: &require{
repo: "github.com/dagger/dagger",
path: "/stdlib/test/test",
version: "v0.0.1",
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := parseArgument(c.in)
if err != nil && c.hasError {
return
}
if err != nil {
t.Fatal(err)
}
if got.repo != c.want.repo {
t.Errorf("repos differ: want %s, got %s", c.want.repo, got.repo)
}
if got.path != c.want.path {
t.Errorf("paths differ: want %s, got %s", c.want.path, got.path)
}
if got.version != c.want.version {
t.Errorf("versions differ: want %s, got %s", c.want.version, got.version)
}
})
}
}
func TestReadFile(t *testing.T) {
cases := []struct {
name string

36
cmd/dagger/cmd/mod/get.go
View File

@ -10,11 +10,12 @@ import (
"github.com/spf13/viper"
"go.dagger.io/dagger/cmd/dagger/cmd/common"
"go.dagger.io/dagger/cmd/dagger/logger"
"go.dagger.io/dagger/telemetry"
)
var getCmd = &cobra.Command{
Use: "get [packages]",
Short: "download and install packages and dependencies",
Short: "download and install dependencies",
Args: cobra.MaximumNArgs(1),
PreRun: func(cmd *cobra.Command, args []string) {
// Fix Viper bug for duplicate flags:
@ -28,12 +29,17 @@ var getCmd = &cobra.Command{
lg := logger.New()
ctx := lg.WithContext(cmd.Context())
doneCh := common.TrackCommand(ctx, cmd)
if len(args) == 0 {
lg.Fatal().Msg("need to specify package name in command argument")
}
workspace := common.CurrentWorkspace(ctx)
st := common.CurrentEnvironmentState(ctx, workspace)
doneCh := common.TrackWorkspaceCommand(ctx, cmd, workspace, st, &telemetry.Property{
Name: "packages",
Value: args,
})
// parse packages to install
var packages []*require
for _, arg := range args {
@ -47,21 +53,20 @@ var getCmd = &cobra.Command{
}
// read mod file in the current dir
modFile, err := readModFile()
modFile, err := readModFile(workspace.Path)
if err != nil {
lg.Fatal().Err(err).Msgf("error loading module file")
}
// download packages
destBasePath := "./cue.mod/pkg"
for _, p := range packages {
if err := processRequire(p, modFile, destBasePath); err != nil {
if err := processRequire(p, modFile); err != nil {
lg.Error().Err(err).Msg("error processing package")
}
}
// write to mod file in the current dir
if err = writeModFile(modFile); err != nil {
if err = writeModFile(workspace.Path, modFile); err != nil {
lg.Error().Err(err).Msg("error writing to mod file")
}
@ -69,13 +74,14 @@ var getCmd = &cobra.Command{
},
}
func processRequire(req *require, modFile *file, destBasePath string) error {
tmpRepoPath := path.Join("./cue.mod/tmp", req.repo)
if err := os.MkdirAll(tmpRepoPath, 0755); err != nil {
func processRequire(req *require, modFile *file) error {
tmpPath := path.Join(tmpBasePath, req.repo)
if err := os.MkdirAll(tmpPath, 0755); err != nil {
return fmt.Errorf("error creating tmp dir for cloning package")
}
defer os.RemoveAll(tmpPath)
r, err := clone(req, tmpRepoPath)
r, err := clone(req, tmpPath)
if err != nil {
return fmt.Errorf("error downloading package %s: %w", req, err)
}
@ -84,11 +90,11 @@ func processRequire(req *require, modFile *file, destBasePath string) error {
// requirement is new, so we should move the files and add it to the module.cue
if existing == nil {
if err := move(req, tmpRepoPath, destBasePath); err != nil {
if err := move(req, tmpPath, destBasePath); err != nil {
return err
}
modFile.require = append(modFile.require, req)
return os.RemoveAll(tmpRepoPath)
return nil
}
c, err := compareVersions(existing.version, req.version)
@ -98,7 +104,7 @@ func processRequire(req *require, modFile *file, destBasePath string) error {
// the existing requirement is newer so we skip installation
if c > 0 {
return os.RemoveAll(tmpRepoPath)
return nil
}
// the new requirement is newer so we checkout the cloned repo to that tag, change the version in the existing
@ -108,7 +114,7 @@ func processRequire(req *require, modFile *file, destBasePath string) error {
return err
}
return replace(req, tmpRepoPath, destBasePath)
return replace(req, tmpPath, destBasePath)
}
func compareVersions(reqV1, reqV2 string) (int, error) {

4
cmd/dagger/cmd/mod/repo.go
View File

@ -40,8 +40,8 @@ func clone(require *require, dir string) (*repo, error) {
return nil, err
}
if _, err := os.Stat(path.Join(dir, require.path, FilePath)); err != nil {
return nil, fmt.Errorf("repo does not contain %s", FilePath)
if _, err := os.Stat(path.Join(dir, require.clonePath, filePath)); err != nil {
return nil, fmt.Errorf("repo does not contain %s", filePath)
}
return rr, nil

16
cmd/dagger/cmd/mod/repo_test.go
View File

@ -15,8 +15,8 @@ func TestClone(t *testing.T) {
name: "resolving shorter hash version",
require: require{
prefix: "https://",
repo: "github.com/tjovicic/gcpcloudrun-cue",
path: "",
cloneRepo: "github.com/tjovicic/gcpcloudrun-cue",
clonePath: "",
version: "d530f2ea2099",
},
},
@ -24,8 +24,8 @@ func TestClone(t *testing.T) {
name: "resolving branch name",
require: require{
prefix: "https://",
repo: "github.com/tjovicic/gcpcloudrun-cue",
path: "",
cloneRepo: "github.com/tjovicic/gcpcloudrun-cue",
clonePath: "",
version: "main",
},
},
@ -33,8 +33,8 @@ func TestClone(t *testing.T) {
name: "resolving tag",
require: require{
prefix: "https://",
repo: "github.com/tjovicic/gcpcloudrun-cue",
path: "",
cloneRepo: "github.com/tjovicic/gcpcloudrun-cue",
clonePath: "",
version: "v0.3",
},
},
@ -66,8 +66,8 @@ func TestListTags(t *testing.T) {
r, err := clone(&require{
prefix: "https://",
repo: "github.com/tjovicic/gcpcloudrun-cue",
path: "",
cloneRepo: "github.com/tjovicic/gcpcloudrun-cue",
clonePath: "",
version: "",
}, tmpDir)
if err != nil {