cleanup: move packages to top level, change vanity URL

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>

state/input.go

@@ -0,0 +1,233 @@
+package state
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"cuelang.org/go/cue"
+
+	"go.dagger.io/dagger/compiler"
+)
+
+// An input is a value or artifact supplied by the user.
+//
+// - A value is any structured data which can be encoded as cue.
+//
+// - An artifact is a piece of data, like a source code checkout,
+//   binary bundle, docker image, database backup etc.
+//
+//   Artifacts can be passed as inputs, generated dynamically from
+//   other inputs, and received as outputs.
+//   Under the hood, an artifact is encoded as a LLB pipeline, and
+//   attached to the cue configuration as a
+//
+
+type Input struct {
+	Dir    *dirInput    `yaml:"dir,omitempty"`
+	Git    *gitInput    `yaml:"git,omitempty"`
+	Docker *dockerInput `yaml:"docker,omitempty"`
+	Secret *secretInput `yaml:"secret,omitempty"`
+	Text   *textInput   `yaml:"text,omitempty"`
+	JSON   *jsonInput   `yaml:"json,omitempty"`
+	YAML   *yamlInput   `yaml:"yaml,omitempty"`
+	File   *fileInput   `yaml:"file,omitempty"`
+}
+
+func (i Input) Compile(state *State) (*compiler.Value, error) {
+	switch {
+	case i.Dir != nil:
+		return i.Dir.Compile(state)
+	case i.Git != nil:
+		return i.Git.Compile(state)
+	case i.Docker != nil:
+		return i.Docker.Compile(state)
+	case i.Text != nil:
+		return i.Text.Compile(state)
+	case i.Secret != nil:
+		return i.Secret.Compile(state)
+	case i.JSON != nil:
+		return i.JSON.Compile(state)
+	case i.YAML != nil:
+		return i.YAML.Compile(state)
+	case i.File != nil:
+		return i.File.Compile(state)
+	default:
+		return nil, fmt.Errorf("input has not been set")
+	}
+}
+
+// An input artifact loaded from a local directory
+func DirInput(path string, include []string) Input {
+	return Input{
+		Dir: &dirInput{
+			Path:    path,
+			Include: include,
+		},
+	}
+}
+
+type dirInput struct {
+	Path    string   `json:"path,omitempty"`
+	Include []string `json:"include,omitempty"`
+}
+
+func (dir dirInput) Compile(state *State) (*compiler.Value, error) {
+	// FIXME: serialize an intermediate struct, instead of generating cue source
+
+	// json.Marshal([]string{}) returns []byte("null"), which wreaks havoc
+	// in Cue because `null` is not a `[...string]`
+	includeLLB := []byte("[]")
+	if len(dir.Include) > 0 {
+		var err error
+		includeLLB, err = json.Marshal(dir.Include)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	p := dir.Path
+	if !filepath.IsAbs(p) {
+		p = filepath.Clean(path.Join(state.Workspace, dir.Path))
+	}
+	if !strings.HasPrefix(p, state.Workspace) {
+		return nil, fmt.Errorf("%q is outside the workspace", dir.Path)
+	}
+
+	llb := fmt.Sprintf(
+		`#up: [{do:"local",dir:"%s", include:%s}]`,
+		p,
+		includeLLB,
+	)
+	return compiler.Compile("", llb)
+}
+
+// An input artifact loaded from a git repository
+type gitInput struct {
+	Remote string `json:"remote,omitempty"`
+	Ref    string `json:"ref,omitempty"`
+	Dir    string `json:"dir,omitempty"`
+}
+
+func GitInput(remote, ref, dir string) Input {
+	return Input{
+		Git: &gitInput{
+			Remote: remote,
+			Ref:    ref,
+			Dir:    dir,
+		},
+	}
+}
+
+func (git gitInput) Compile(_ *State) (*compiler.Value, error) {
+	ref := "HEAD"
+	if git.Ref != "" {
+		ref = git.Ref
+	}
+
+	return compiler.Compile("", fmt.Sprintf(
+		`#up: [{do:"fetch-git", remote:"%s", ref:"%s"}]`,
+		git.Remote,
+		ref,
+	))
+}
+
+// An input artifact loaded from a docker container
+func DockerInput(ref string) Input {
+	return Input{
+		Docker: &dockerInput{
+			Ref: ref,
+		},
+	}
+}
+
+type dockerInput struct {
+	Ref string `json:"ref,omitempty"`
+}
+
+func (i dockerInput) Compile(_ *State) (*compiler.Value, error) {
+	panic("NOT IMPLEMENTED")
+}
+
+// An input value encoded as text
+func TextInput(data string) Input {
+	i := textInput(data)
+	return Input{
+		Text: &i,
+	}
+}
+
+type textInput string
+
+func (i textInput) Compile(_ *State) (*compiler.Value, error) {
+	return compiler.Compile("", fmt.Sprintf("%q", i))
+}
+
+// A secret input value
+func SecretInput(data string) Input {
+	i := secretInput(data)
+	return Input{
+		Secret: &i,
+	}
+}
+
+type secretInput string
+
+func (i secretInput) Compile(_ *State) (*compiler.Value, error) {
+	return compiler.Compile("", fmt.Sprintf("%q", i))
+}
+
+// An input value encoded as JSON
+func JSONInput(data string) Input {
+	i := jsonInput(data)
+	return Input{
+		JSON: &i,
+	}
+}
+
+type jsonInput string
+
+func (i jsonInput) Compile(_ *State) (*compiler.Value, error) {
+	return compiler.DecodeJSON("", []byte(i))
+}
+
+// An input value encoded as YAML
+func YAMLInput(data string) Input {
+	i := yamlInput(data)
+	return Input{
+		YAML: &i,
+	}
+}
+
+type yamlInput string
+
+func (i yamlInput) Compile(_ *State) (*compiler.Value, error) {
+	return compiler.DecodeYAML("", []byte(i))
+}
+
+func FileInput(data string) Input {
+	return Input{
+		File: &fileInput{
+			Path: data,
+		},
+	}
+}
+
+type fileInput struct {
+	Path string `json:"data,omitempty"`
+}
+
+func (i fileInput) Compile(_ *State) (*compiler.Value, error) {
+	data, err := ioutil.ReadFile(i.Path)
+	if err != nil {
+		return nil, err
+	}
+	value := compiler.NewValue()
+	if err := value.FillPath(cue.MakePath(), data); err != nil {
+		return nil, err
+	}
+	return value, nil
+}

state/state.go

@@ -0,0 +1,46 @@
+package state
+
+// Contents of an environment serialized to a file
+type State struct {
+	// State path
+	Path string `yaml:"-"`
+
+	// Workspace path
+	Workspace string `yaml:"-"`
+
+	// Plan path
+	Plan string `yaml:"-"`
+
+	// Human-friendly environment name.
+	// A environment may have more than one name.
+	// FIXME: store multiple names?
+	Name string `yaml:"name,omitempty"`
+
+	// User Inputs
+	Inputs map[string]Input `yaml:"inputs,omitempty"`
+
+	// Computed values
+	Computed string `yaml:"-"`
+}
+
+// Cue module containing the environment plan
+// The input's top-level artifact is used as a module directory.
+func (s *State) PlanSource() Input {
+	return DirInput(s.Plan, []string{"*.cue", "cue.mod"})
+}
+
+func (s *State) SetInput(key string, value Input) error {
+	if s.Inputs == nil {
+		s.Inputs = make(map[string]Input)
+	}
+	s.Inputs[key] = value
+	return nil
+}
+
+// Remove all inputs at the given key, including sub-keys.
+// For example RemoveInputs("foo.bar") will remove all inputs
+//   at foo.bar, foo.bar.baz, etc.
+func (s *State) RemoveInputs(key string) error {
+	delete(s.Inputs, key)
+	return nil
+}

state/workspace.go

@@ -0,0 +1,270 @@
+package state
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/rs/zerolog/log"
+	"go.dagger.io/dagger/keychain"
+	"gopkg.in/yaml.v3"
+)
+
+var (
+	ErrNotInit     = errors.New("not initialized")
+	ErrAlreadyInit = errors.New("already initialized")
+	ErrNotExist    = errors.New("environment doesn't exist")
+	ErrExist       = errors.New("environment already exists")
+)
+
+const (
+	daggerDir    = ".dagger"
+	envDir       = "env"
+	stateDir     = "state"
+	planDir      = "plan"
+	manifestFile = "values.yaml"
+	computedFile = "computed.json"
+)
+
+type Workspace struct {
+	Path string
+}
+
+func Init(ctx context.Context, dir string) (*Workspace, error) {
+	root, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	daggerRoot := path.Join(root, daggerDir)
+	if err := os.Mkdir(daggerRoot, 0755); err != nil {
+		if errors.Is(err, os.ErrExist) {
+			return nil, ErrAlreadyInit
+		}
+		return nil, err
+	}
+	if err := os.Mkdir(path.Join(daggerRoot, envDir), 0755); err != nil {
+		return nil, err
+	}
+	return &Workspace{
+		Path: root,
+	}, nil
+}
+
+func Open(ctx context.Context, dir string) (*Workspace, error) {
+	_, err := os.Stat(path.Join(dir, daggerDir))
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, ErrNotInit
+		}
+		return nil, err
+	}
+
+	root, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Workspace{
+		Path: root,
+	}, nil
+}
+
+func Current(ctx context.Context) (*Workspace, error) {
+	current, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+
+	// Walk every parent directory to find .dagger
+	for {
+		_, err := os.Stat(path.Join(current, daggerDir, envDir))
+		if err == nil {
+			return Open(ctx, current)
+		}
+		parent := filepath.Dir(current)
+		if parent == current {
+			break
+		}
+		current = parent
+	}
+
+	return nil, ErrNotInit
+}
+
+func (w *Workspace) envPath(name string) string {
+	return path.Join(w.Path, daggerDir, envDir, name)
+}
+
+func (w *Workspace) List(ctx context.Context) ([]*State, error) {
+	var (
+		environments = []*State{}
+		err          error
+	)
+
+	files, err := os.ReadDir(path.Join(w.Path, daggerDir, envDir))
+	if err != nil {
+		return nil, err
+	}
+	for _, f := range files {
+		if !f.IsDir() {
+			continue
+		}
+		st, err := w.Get(ctx, f.Name())
+		if err != nil {
+			log.
+				Ctx(ctx).
+				Err(err).
+				Str("name", f.Name()).
+				Msg("failed to load environment")
+			continue
+		}
+		environments = append(environments, st)
+	}
+
+	return environments, nil
+}
+
+func (w *Workspace) Get(ctx context.Context, name string) (*State, error) {
+	envPath, err := filepath.Abs(w.envPath(name))
+	if err != nil {
+		return nil, err
+	}
+	if _, err := os.Stat(envPath); err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, ErrNotExist
+		}
+		return nil, err
+	}
+
+	manifest, err := os.ReadFile(path.Join(envPath, manifestFile))
+	if err != nil {
+		return nil, err
+	}
+	manifest, err = keychain.Decrypt(ctx, manifest)
+	if err != nil {
+		return nil, fmt.Errorf("unable to decrypt state: %w", err)
+	}
+
+	var st State
+	if err := yaml.Unmarshal(manifest, &st); err != nil {
+		return nil, err
+	}
+	st.Path = envPath
+	st.Plan = path.Join(envPath, planDir)
+	st.Workspace = w.Path
+
+	computed, err := os.ReadFile(path.Join(envPath, stateDir, computedFile))
+	if err == nil {
+		st.Computed = string(computed)
+	}
+
+	return &st, nil
+}
+
+func (w *Workspace) Save(ctx context.Context, st *State) error {
+	data, err := yaml.Marshal(st)
+	if err != nil {
+		return err
+	}
+
+	manifestPath := path.Join(st.Path, manifestFile)
+
+	currentEncrypted, err := os.ReadFile(manifestPath)
+	if err != nil {
+		return err
+	}
+	currentPlain, err := keychain.Decrypt(ctx, currentEncrypted)
+	if err != nil {
+		return fmt.Errorf("unable to decrypt state: %w", err)
+	}
+
+	// Only update the encrypted file if there were changes
+	if !bytes.Equal(data, currentPlain) {
+		encrypted, err := keychain.Reencrypt(ctx, manifestPath, data)
+		if err != nil {
+			return err
+		}
+		if err := os.WriteFile(manifestPath, encrypted, 0600); err != nil {
+			return err
+		}
+	}
+
+	if st.Computed != "" {
+		state := path.Join(st.Path, stateDir)
+		if err := os.MkdirAll(state, 0755); err != nil {
+			return err
+		}
+		err := os.WriteFile(
+			path.Join(state, "computed.json"),
+			[]byte(st.Computed),
+			0600)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (w *Workspace) Create(ctx context.Context, name string) (*State, error) {
+	envPath, err := filepath.Abs(w.envPath(name))
+	if err != nil {
+		return nil, err
+	}
+
+	// Environment directory
+	if err := os.MkdirAll(envPath, 0755); err != nil {
+		if errors.Is(err, os.ErrExist) {
+			return nil, ErrExist
+		}
+		return nil, err
+	}
+
+	// Plan directory
+	if err := os.Mkdir(path.Join(envPath, planDir), 0755); err != nil {
+		if errors.Is(err, os.ErrExist) {
+			return nil, ErrExist
+		}
+		return nil, err
+	}
+
+	manifestPath := path.Join(envPath, manifestFile)
+
+	st := &State{
+		Path:      envPath,
+		Workspace: w.Path,
+		Plan:      path.Join(envPath, planDir),
+		Name:      name,
+	}
+	data, err := yaml.Marshal(st)
+	if err != nil {
+		return nil, err
+	}
+	key, err := keychain.Default(ctx)
+	if err != nil {
+		return nil, err
+	}
+	encrypted, err := keychain.Encrypt(ctx, manifestPath, data, key)
+	if err != nil {
+		return nil, err
+	}
+	if err := os.WriteFile(manifestPath, encrypted, 0600); err != nil {
+		return nil, err
+	}
+
+	err = os.WriteFile(
+		path.Join(envPath, ".gitignore"),
+		[]byte("# dagger state\nstate/**\n"),
+		0600,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return st, nil
+}

state/workspace_test.go

@@ -0,0 +1,109 @@
+package state
+
+import (
+	"context"
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func TestWorkspace(t *testing.T) {
+	ctx := context.TODO()
+
+	root, err := os.MkdirTemp(os.TempDir(), "dagger-*")
+	require.NoError(t, err)
+
+	// Open should fail since the directory is not initialized
+	_, err = Open(ctx, root)
+	require.ErrorIs(t, ErrNotInit, err)
+
+	// Init
+	workspace, err := Init(ctx, root)
+	require.NoError(t, err)
+	require.Equal(t, root, workspace.Path)
+
+	// Create
+	st, err := workspace.Create(ctx, "test")
+	require.NoError(t, err)
+	require.Equal(t, "test", st.Name)
+
+	// Open
+	workspace, err = Open(ctx, root)
+	require.NoError(t, err)
+	require.Equal(t, root, workspace.Path)
+
+	// List
+	envs, err := workspace.List(ctx)
+	require.NoError(t, err)
+	require.Len(t, envs, 1)
+	require.Equal(t, "test", envs[0].Name)
+
+	// Get
+	env, err := workspace.Get(ctx, "test")
+	require.NoError(t, err)
+	require.Equal(t, "test", env.Name)
+
+	// Save
+	require.NoError(t, env.SetInput("foo", TextInput("bar")))
+	require.NoError(t, workspace.Save(ctx, env))
+	workspace, err = Open(ctx, root)
+	require.NoError(t, err)
+	env, err = workspace.Get(ctx, "test")
+	require.NoError(t, err)
+	require.Contains(t, env.Inputs, "foo")
+}
+
+func TestEncryption(t *testing.T) {
+	ctx := context.TODO()
+
+	readManifest := func(st *State) *State {
+		data, err := os.ReadFile(path.Join(st.Path, manifestFile))
+		require.NoError(t, err)
+		m := State{}
+		require.NoError(t, yaml.Unmarshal(data, &m))
+		return &m
+	}
+
+	root, err := os.MkdirTemp(os.TempDir(), "dagger-*")
+	require.NoError(t, err)
+	workspace, err := Init(ctx, root)
+	require.NoError(t, err)
+
+	_, err = workspace.Create(ctx, "test")
+	require.NoError(t, err)
+
+	// Set a plaintext input, make sure it is not encrypted
+	st, err := workspace.Get(ctx, "test")
+	require.NoError(t, err)
+	require.NoError(t, st.SetInput("plain", TextInput("plain")))
+	require.NoError(t, workspace.Save(ctx, st))
+	o := readManifest(st)
+	require.Contains(t, o.Inputs, "plain")
+	require.Equal(t, "plain", string(*o.Inputs["plain"].Text))
+
+	// Set a secret input, make sure it's encrypted
+	st, err = workspace.Get(ctx, "test")
+	require.NoError(t, err)
+	require.NoError(t, st.SetInput("secret", SecretInput("secret")))
+	require.NoError(t, workspace.Save(ctx, st))
+	o = readManifest(st)
+	require.Contains(t, o.Inputs, "secret")
+	secretValue := string(*o.Inputs["secret"].Secret)
+	require.NotEqual(t, "secret", secretValue)
+	require.True(t, strings.HasPrefix(secretValue, "ENC["))
+
+	// Change another input, make sure our secret didn't change
+	st, err = workspace.Get(ctx, "test")
+	require.NoError(t, err)
+	require.NoError(t, st.SetInput("plain", TextInput("different")))
+	require.NoError(t, workspace.Save(ctx, st))
+	o = readManifest(st)
+	require.Contains(t, o.Inputs, "plain")
+	require.Equal(t, "different", string(*o.Inputs["plain"].Text))
+	require.Contains(t, o.Inputs, "secret")
+	require.Equal(t, secretValue, string(*o.Inputs["secret"].Secret))
+}