diff --git a/.github/workflows/dagger-ci.yml b/.github/workflows/dagger-ci.yml index 1492b19e..95e0c2b3 100644 --- a/.github/workflows/dagger-ci.yml +++ b/.github/workflows/dagger-ci.yml @@ -2,42 +2,84 @@ name: "Dagger CI" on: push: - branches: [ main ] + branches: [main] paths: - - '**.sh' - - '**.bash' - - '**.go' - - '**.cue' - - '**.bats' - - 'Makefile' - - 'go.mod' - - 'go.sum' - - '.github/workflows/dagger-ci.yml' + - "**.sh" + - "**.bash" + - "**.go" + - "**.cue" + - "**.bats" + - "Makefile" + - "go.mod" + - "go.sum" + - ".github/workflows/dagger-ci.yml" pull_request: - branches: [ main ] + branches: [main] paths: - - '**.sh' - - '**.bash' - - '**.go' - - '**.cue' - - '**.bats' - - 'Makefile' - - 'go.mod' - - 'go.sum' - - '.github/workflows/dagger-ci.yml' + - "**.sh" + - "**.bash" + - "**.go" + - "**.cue" + - "**.bats" + - "Makefile" + - "go.mod" + - "go.sum" + - ".github/workflows/dagger-ci.yml" env: DAGGER_LOG_FORMAT: plain jobs: build: + name: "Build" runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 - - name: Dagger CI - uses: dagger/dagger-for-github@v2 + - name: "Setup Go" + uses: actions/setup-go@v1 with: - workdir: ci - args: do build + go-version: 1.16 + + - name: "Expose GitHub Runtime" + uses: crazy-max/ghaction-github-runtime@v1 + + - name: Prepare dagger + run: | + make dagger + cp ./cmd/dagger/dagger /usr/local/bin + + - name: Build + env: + DAGGER_CACHE_TO: "type=gha,mode=max,scope=dagger-ci-build" + DAGGER_CACHE_FROM: "type=gha,scope=dagger-ci-build" + run: | + dagger do build + + lint: + name: "Lint" + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: "Setup Go" + uses: actions/setup-go@v1 + with: + go-version: 1.16 + + - name: "Expose GitHub Runtime" + uses: crazy-max/ghaction-github-runtime@v1 + + - name: Prepare dagger + run: | + make dagger + cp ./cmd/dagger/dagger /usr/local/bin + + - name: Lint + env: + DAGGER_CACHE_TO: "type=gha,mode=max,scope=dagger-ci-lint" + DAGGER_CACHE_FROM: "type=gha,scope=dagger-ci-lint" + run: | + dagger do lint diff --git a/ci.cue b/ci.cue new file mode 100644 index 00000000..310f4b18 --- /dev/null +++ b/ci.cue @@ -0,0 +1,150 @@ +package main + +import ( + "dagger.io/dagger" + + "universe.dagger.io/bash" + "universe.dagger.io/alpine" + "universe.dagger.io/docker" + "universe.dagger.io/go" + + "github.com/dagger/dagger/ci/golangci" + "github.com/dagger/dagger/ci/shellcheck" + "github.com/dagger/dagger/ci/markdownlint" +) + +dagger.#Plan & { + // FIXME: Ideally we would want to automatically set the platform's arch identical to the host + // to avoid the performance hit caused by qemu (linter goes from <3s to >3m when arch is x86) + // Uncomment if running locally on Mac M1 to bypass qemu + // platform: "linux/aarch64" + platform: "linux/amd64" + + client: filesystem: ".": read: exclude: [ + "bin", + "**/node_modules", + "cmd/dagger/dagger", + "cmd/dagger/dagger-debug", + ] + client: filesystem: "./bin": write: contents: actions.build.output + + actions: { + _source: client.filesystem["."].read.contents + + // FIXME: this can be removed once `go` supports built-in VCS info + version: { + _image: alpine.#Build & { + packages: bash: _ + packages: curl: _ + packages: git: _ + } + + _revision: bash.#Run & { + input: _image.output + workdir: "/src" + mounts: source: { + dest: "/src" + contents: _source + } + + script: contents: #""" + printf "$(git rev-parse --short HEAD)" > /revision + """# + export: files: "/revision": string + } + + output: _revision.export.files["/revision"] + } + + build: go.#Build & { + source: _source + package: "./cmd/dagger/" + os: client.platform.os + arch: client.platform.arch + + ldflags: "-s -w -X go.dagger.io/dagger/version.Revision=\(version.output)" + + env: { + CGO_ENABLED: "0" + // Makes sure the linter and unit tests complete before starting the build + // "__depends_lint": "\(goLint.exit)" + // "__depends_tests": "\(goTest.exit)" + } + } + + // Go unit tests + test: go.#Test & { + // container: image: _goImage.output + source: _source + package: "./..." + + // FIXME: doesn't work with CGO_ENABLED=0 + // command: flags: "-race": true + + env: { + // FIXME: removing this complains about lack of gcc + CGO_ENABLED: "0" + } + } + + lint: { + go: golangci.#Lint & { + source: _source + version: "1.45" + } + + shell: shellcheck.#Lint & { + source: _source + } + + markdown: markdownlint.#Lint & { + source: _source + files: ["./docs", "README.md"] + } + + cue: docker.#Build & { + // FIXME: spin off into its own package? + steps: [ + alpine.#Build & { + packages: bash: _ + packages: curl: _ + packages: git: _ + }, + + docker.#Copy & { + contents: _source + source: "go.mod" + dest: "go.mod" + }, + + // Install CUE + bash.#Run & { + script: contents: #""" + export CUE_VERSION="$(grep cue ./go.mod | cut -d' ' -f2 | head -1 | sed -E 's/\.[[:digit:]]\.[[:alnum:]]+-[[:alnum:]]+$//')" + export CUE_TARBALL="cue_${CUE_VERSION}_linux_amd64.tar.gz" + echo "Installing cue version $CUE_VERSION" + curl -L "https://github.com/cue-lang/cue/releases/download/${CUE_VERSION}/${CUE_TARBALL}" | tar zxf - -C /usr/local/bin + cue version + """# + }, + + // CACHE: copy only *.cue files + docker.#Copy & { + contents: _source + include: ["*.cue"] + dest: "/cue" + }, + + // LINT + bash.#Run & { + workdir: "/cue" + script: contents: #""" + find . -name '*.cue' -not -path '*/cue.mod/*' -print | time xargs -t -n 1 -P 8 cue fmt -s + test -z "$(git status -s . | grep -e "^ M" | grep "\.cue" | cut -d ' ' -f3 | tee /dev/stderr)" + """# + }, + ] + } + } + } +} diff --git a/ci/.gitignore b/ci/.gitignore deleted file mode 100644 index a007feab..00000000 --- a/ci/.gitignore +++ /dev/null @@ -1 +0,0 @@ -build/* diff --git a/ci/README.md b/ci/README.md deleted file mode 100644 index 0ad8a3d7..00000000 --- a/ci/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Dagger CI - -WORK IN PROGRESS - -This is the home of the new CI - -Reference: https://github.com/dagger/dagger/issues/1549 diff --git a/ci/golangci/lint.cue b/ci/golangci/lint.cue new file mode 100644 index 00000000..32008ff5 --- /dev/null +++ b/ci/golangci/lint.cue @@ -0,0 +1,37 @@ +package golangci + +import ( + "dagger.io/dagger" + + "universe.dagger.io/docker" + "universe.dagger.io/go" +) + +// Lint using golangci-lint +#Lint: { + // Source code + source: dagger.#FS + + // golangci-lint version + version: *"1.45" | string + + // timeout + timeout: *"5m" | string + + _image: docker.#Pull & { + source: "golangci/golangci-lint:v\(version)" + } + + container: go.#Container & { + "source": source + input: _image.output + command: { + name: "golangci-lint" + flags: { + run: true + "-v": true + "--timeout": timeout + } + } + } +} diff --git a/ci/images.cue b/ci/images.cue deleted file mode 100644 index 6e2296f7..00000000 --- a/ci/images.cue +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "universe.dagger.io/docker" -) - -let GoVersion = "1.17" -let GolangCILintVersion = "1.44.0" -let CUEVersion = "0.4.2" - -// Base container images used for the CI -#Images: { - - // base image to build go binaries - goBuilder: _goBuilder.output - _goBuilder: docker.#Build & { - _packages: ["bash", "git", "alpine-sdk"] - - steps: [ - docker.#Pull & { - source: "index.docker.io/golang:\(GoVersion)-alpine" - }, - for pkg in _packages { - docker.#Run & { - command: { - name: "apk" - args: ["add", pkg] - flags: { - "-U": true - "--no-cache": true - } - } - } - }, - ] - } - - // base image for the Go linter - // https://golangci-lint.run/usage/install/#docker - goLinter: _goLinter.output - _goLinter: docker.#Pull & { - source: "index.docker.io/golangci/golangci-lint:v\(GolangCILintVersion)" - } - - // base image for CUE cli + alpine distrib - cue: _cue._alpine.output - _cue: { - _cueBinary: docker.#Pull & { - source: "index.docker.io/cuelang/cue:\(CUEVersion)" - } - - _alpine: docker.#Build & { - _packages: ["bash", "git"] - - steps: [ - docker.#Pull & { - source: "index.docker.io/alpine:3" - }, - for pkg in _packages { - docker.#Run & { - command: { - name: "apk" - args: ["add", pkg] - flags: { - "-U": true - "--no-cache": true - } - } - } - }, - docker.#Copy & { - // input: _alpine.output - contents: _cueBinary.output.rootfs - source: "/usr/bin/cue" - dest: "/usr/bin/cue" - }, - ] - } - } -} diff --git a/ci/main.cue b/ci/main.cue deleted file mode 100644 index 6abdcaa0..00000000 --- a/ci/main.cue +++ /dev/null @@ -1,99 +0,0 @@ -package main - -import ( - "dagger.io/dagger" - "dagger.io/dagger/core" - "universe.dagger.io/bash" -) - -dagger.#Plan & { - - // FIXME: Ideally we would want to automatically set the platform's arch identical to the host - // to avoid the performance hit caused by qemu (linter goes from <3s to >3m when arch is x86) - // Uncomment if running locally on Mac M1 to bypass qemu - // platform: "linux/aarch64" - platform: "linux/amd64" - - client: filesystem: "./build": write: contents: actions.build.export.directories["/build"] - - actions: { - _mountGoCache: { - mounts: "go mod cache": { - dest: "/root/.gocache" - contents: core.#CacheDir & { - id: "go mod cache" - } - } - env: GOMODCACHE: mounts["go mod cache"].dest - } - - _mountSourceCode: { - mounts: "dagger source code": { - contents: _source.output - dest: "/usr/src/dagger" - } - workdir: mounts["dagger source code"].dest - } - - _baseImages: #Images - - // Go build the dagger binary - // depends on goLint and goTest to complete successfully - build: bash.#Run & { - _mountSourceCode - _mountGoCache - - input: _baseImages.goBuilder - - env: { - GOOS: client.platform.os - GOARCH: client.platform.arch - CGO_ENABLED: "0" - // Makes sure the linter and unit tests complete before starting the build - "__depends_lint": "\(goLint.exit)" - "__depends_tests": "\(goTest.exit)" - } - - script: contents: #""" - mkdir -p /build - git_revision=$(git rev-parse --short HEAD) - go build -v -o /build/dagger \ - -ldflags '-s -w -X go.dagger.io/dagger/version.Revision='${git_revision} \ - ./cmd/dagger/ - """# - - export: directories: "/build": _ - } - - // Go unit tests - goTest: bash.#Run & { - _mountSourceCode - _mountGoCache - - input: _baseImages.goBuilder - script: contents: "go test -race -v ./..." - } - - // Go lint using golangci-lint - goLint: bash.#Run & { - _mountSourceCode - _mountGoCache - - input: _baseImages.goLinter - script: contents: "golangci-lint run -v --timeout 5m" - } - - // CUE lint - cueLint: bash.#Run & { - _mountSourceCode - - input: _baseImages.cue - script: contents: #""" - # Format the cue code - find . -name '*.cue' -not -path '*/cue.mod/*' -print | time xargs -n 1 -P 8 cue fmt -s - # Check that all formatted files where committed - test -z $(git status -s . | grep -e '^ M' | grep .cue | cut -d ' ' -f3) - """# - } - } -} diff --git a/ci/markdownlint/markdownlint.cue b/ci/markdownlint/markdownlint.cue new file mode 100644 index 00000000..02043a3b --- /dev/null +++ b/ci/markdownlint/markdownlint.cue @@ -0,0 +1,36 @@ +package markdownlint + +import ( + "dagger.io/dagger" + + "universe.dagger.io/docker" +) + +#Lint: { + // Source code + source: dagger.#FS + + // shellcheck version + version: *"0.31.1" | string + + // Files to lint + files: [...string] + + _image: docker.#Pull & { + source: "tmknom/markdownlint:\(version)" + } + + container: docker.#Run & { + input: _image.output + mounts: "source": { + dest: "/src" + contents: source + } + workdir: "/src" + command: { + // FIXME: this should not be required + name: "markdownlint" + args: files + } + } +} diff --git a/ci/shellcheck/shellcheck.cue b/ci/shellcheck/shellcheck.cue new file mode 100644 index 00000000..b11bf62c --- /dev/null +++ b/ci/shellcheck/shellcheck.cue @@ -0,0 +1,34 @@ +package shellcheck + +import ( + "dagger.io/dagger" + + "universe.dagger.io/docker" +) + +#Lint: { + // Source code + source: dagger.#FS + + // shellcheck version + version: *"0.8.0" | string + + _image: docker.#Pull & { + source: "koalaman/shellcheck-alpine:v\(version)" + } + + container: docker.#Run & { + input: _image.output + mounts: "source": { + dest: "/src" + contents: source + } + workdir: "/src" + command: { + name: "sh" + args: ["-c", #""" + shellcheck $(find . -type f \( -iname \*.bats -o -iname \*.bash -o -iname \*.sh \) -not -path "*/node_modules/*" -not -path "*/bats-*/*") + """#] + } + } +} diff --git a/cue.mod/module.cue b/cue.mod/module.cue index b57d0104..dc80073f 100644 --- a/cue.mod/module.cue +++ b/cue.mod/module.cue @@ -1 +1 @@ -module: "" \ No newline at end of file +module: "github.com/dagger/dagger" diff --git a/pkg/universe.dagger.io/go/build.cue b/pkg/universe.dagger.io/go/build.cue index eb0a080e..72e1eef1 100644 --- a/pkg/universe.dagger.io/go/build.cue +++ b/pkg/universe.dagger.io/go/build.cue @@ -13,10 +13,10 @@ import ( package: *"." | string // Target architecture - arch: *"amd64" | string + arch?: string // Target OS - os: *"linux" | string + os?: string // Build tags to use for building tags: *"" | string @@ -30,10 +30,15 @@ import ( "source": source "env": { env - GOOS: os - GOARCH: arch + if os != _|_ { + GOOS: os + } + if arch != _|_ { + GOARCH: arch + } } command: { + name: "go" args: [package] flags: { build: true diff --git a/pkg/universe.dagger.io/go/container.cue b/pkg/universe.dagger.io/go/container.cue index d53009dd..df52f57b 100644 --- a/pkg/universe.dagger.io/go/container.cue +++ b/pkg/universe.dagger.io/go/container.cue @@ -18,25 +18,31 @@ import ( // Use go image _image: #Image - _sourcePath: "/src" - _cachePath: "/root/.cache/gocache" + _sourcePath: "/src" + _modCachePath: "/root/.cache/go-mod" + _buildCachePath: "/root/.cache/go-build" docker.#Run & { input: *_image.output | docker.#Image - workdir: "/src" - command: name: "go" + workdir: _sourcePath mounts: { "source": { dest: _sourcePath contents: source } - "go assets cache": { + "go mod cache": { contents: core.#CacheDir & { - id: "\(name)_assets" + id: "\(name)_mod" } - dest: _cachePath + dest: _modCachePath + } + "go build cache": { + contents: core.#CacheDir & { + id: "\(name)_build" + } + dest: _buildCachePath } } - env: GOMODCACHE: _cachePath + env: GOMODCACHE: _modCachePath } } diff --git a/pkg/universe.dagger.io/go/test.cue b/pkg/universe.dagger.io/go/test.cue index 5de2b94b..167c6a23 100644 --- a/pkg/universe.dagger.io/go/test.cue +++ b/pkg/universe.dagger.io/go/test.cue @@ -7,6 +7,7 @@ package go #Container & { command: { + name: "go" args: [package] flags: { test: true diff --git a/pkg/universe.dagger.io/go/test/container.cue b/pkg/universe.dagger.io/go/test/container.cue index 82aad0c5..4966a1a1 100644 --- a/pkg/universe.dagger.io/go/test/container.cue +++ b/pkg/universe.dagger.io/go/test/container.cue @@ -12,7 +12,10 @@ dagger.#Plan & { simple: go.#Container & { source: _source - command: args: ["version"] + command: { + name: "go" + args: ["version"] + } } override: { @@ -23,7 +26,10 @@ dagger.#Plan & { command: go.#Container & { input: base.output source: _source - command: args: ["version"] + command: { + name: "go" + args: ["version"] + } } } } diff --git a/plan/task/clientfilesystemread.go b/plan/task/clientfilesystemread.go index 3dc6a5e3..733c53ae 100644 --- a/plan/task/clientfilesystemread.go +++ b/plan/task/clientfilesystemread.go @@ -125,12 +125,9 @@ func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext. opts = append(opts, llb.IncludePatterns(dir.Include)) } - // Excludes .dagger directory by default - excludePatterns := []string{"**/.dagger/"} if len(dir.Exclude) > 0 { - excludePatterns = dir.Exclude + opts = append(opts, llb.ExcludePatterns(dir.Exclude)) } - opts = append(opts, llb.ExcludePatterns(excludePatterns)) // FIXME: Remove the `Copy` and use `Local` directly. // diff --git a/plan/task/source.go b/plan/task/source.go index 19556b39..816cd994 100644 --- a/plan/task/source.go +++ b/plan/task/source.go @@ -69,7 +69,7 @@ func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s *solv lg.Debug().Str("path", path).Msg("loading local directory") opts := []llb.LocalOption{ - withCustomName(v, "Embed %s", path), + withCustomName(v, "Source %s", path), llb.IncludePatterns(source.Include), llb.ExcludePatterns(source.Exclude), // Without hint, multiple `llb.Local` operations on the diff --git a/source.cue b/source.cue deleted file mode 100644 index c5213142..00000000 --- a/source.cue +++ /dev/null @@ -1,15 +0,0 @@ -package main - -import ( - "dagger.io/dagger/core" -) - -_source: core.#Source & { - path: "." - exclude: [ - "ci", - "node_modules", - "cmd/dagger/dagger", - "cmd/dagger/dagger-debug", - ] -}