Merge pull request #1312 from aluzzardi/embedding-poc

This commit is contained in:
Andrea Luzzardi 2022-01-14 01:51:20 -08:00 committed by GitHub
commit 4c673aae52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 254 additions and 0 deletions

View File

@ -1,5 +1,20 @@
package engine
// Access the source directory for the current CUE package
// This may safely be called from any package
#Source: {
$dagger: task: _name: "Source"
// Relative path to source.
path: string
// Optionally exclude certain files
include: [...string]
// Optionall include certain files
exclude: [...string]
output: #FS
}
// Create one or multiple directory in a container
#Mkdir: {
$dagger: task: _name: "Mkdir"

101
plan/task/source.go Normal file
View File

@ -0,0 +1,101 @@
package task
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"github.com/moby/buildkit/client/llb"
"github.com/rs/zerolog/log"
"go.dagger.io/dagger/compiler"
"go.dagger.io/dagger/plancontext"
"go.dagger.io/dagger/solver"
)
func init() {
Register("Source", func() Task { return &sourceTask{} })
}
type sourceTask struct {
}
func (c *sourceTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
path, err := v.Lookup("path").String()
if err != nil {
return err
}
if !fs.ValidPath(path) {
return fmt.Errorf("invalid path %q", path)
}
absPath, err := v.Lookup("path").AbsPath()
if err != nil {
return err
}
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("path %q does not exist", path)
}
pctx.LocalDirs.Add(absPath)
return nil
}
func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
lg := log.Ctx(ctx)
path, err := v.Lookup("path").AbsPath()
if err != nil {
return nil, err
}
var source struct {
Include []string
Exclude []string
}
if err := v.Decode(&source); err != nil {
return nil, err
}
lg.Debug().Str("path", path).Msg("loading local directory")
opts := []llb.LocalOption{
withCustomName(v, "Embed %s", path),
llb.IncludePatterns(source.Include),
llb.ExcludePatterns(source.Exclude),
// Without hint, multiple `llb.Local` operations on the
// same path get a different digest.
llb.SessionID(s.SessionID()),
llb.SharedKeyHint(path),
}
// FIXME: Remove the `Copy` and use `Local` directly.
//
// Copy'ing is a costly operation which should be unnecessary.
// However, using llb.Local directly breaks caching sometimes for unknown reasons.
st := llb.Scratch().File(
llb.Copy(
llb.Local(
path,
opts...,
),
"/",
"/",
),
withCustomName(v, "Embed %s [copy]", path),
)
result, err := s.Solve(ctx, st, pctx.Platform.Get())
if err != nil {
return nil, err
}
fs := pctx.FS.New(result)
return compiler.NewValue().FillFields(map[string]interface{}{
"output": fs.MarshalCUE(),
})
}

View File

@ -132,3 +132,15 @@ setup() {
"$DAGGER" --europa up ./newsecret.cue
}
@test "task: #Source" {
cd "$TESTDIR"/tasks/source
"$DAGGER" --europa up ./source.cue
"$DAGGER" --europa up ./source_include_exclude.cue
run "$DAGGER" --europa up ./source_invalid_path.cue
assert_failure
run "$DAGGER" --europa up ./source_not_exist.cue
assert_failure
}

View File

@ -0,0 +1 @@
hello

View File

@ -0,0 +1,42 @@
package main
import (
"dagger.io/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
source: engine.#Source & {
path: "."
}
exec: engine.#Exec & {
input: image.output
mounts: code: {
dest: "/src"
contents: source.output
}
args: ["/src/test.sh"]
}
verifyHello: engine.#ReadFile & {
input: source.output
path: "/hello.txt"
} & {
// assert result
contents: "hello\n"
}
verifyWorld: engine.#ReadFile & {
input: source.output
path: "/world.txt"
} & {
// assert result
contents: "world\n"
}
}
}

View File

@ -0,0 +1,45 @@
package main
import (
"dagger.io/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
sourceInclude: engine.#Source & {
path: "."
include: ["hello.txt"]
}
sourceExclude: engine.#Source & {
path: "."
exclude: ["hello.txt"]
}
test: engine.#Exec & {
input: image.output
mounts: {
include: {
dest: "/include"
contents: sourceInclude.output
}
exclude: {
dest: "/exclude"
contents: sourceExclude.output
}
}
args: ["sh", "-c",
#"""
test "$(find /include/ | wc -l)" -eq 1
test -f /include/hello.txt
test ! -f /exclude/hello.txt
"""#,
]
}
}
}

View File

@ -0,0 +1,17 @@
package main
import (
"dagger.io/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
source: engine.#Source & {
path: ".."
}
}
}

View File

@ -0,0 +1,17 @@
package main
import (
"dagger.io/dagger/engine"
)
engine.#Plan & {
actions: {
image: engine.#Pull & {
source: "alpine:3.15.0@sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3"
}
source: engine.#Source & {
path: "not/exist"
}
}
}

3
tests/tasks/source/test.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/sh
echo -n hello world > /test.txt

View File

@ -0,0 +1 @@
world