kjuulh/dagger
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Projects Releases Wiki Activity

Merge pull request #1984 from aluzzardi/ci-improvement

Browse Source 
CI dogfood improvements
This commit is contained in:
Andrea Luzzardi 2022-04-04 18:13:15 -07:00 committed by GitHub
commit a717ec0734
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 358 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
.github/workflows/dagger-ci.yml vendored
View File

@ -2,42 +2,84 @@ name: "Dagger CI"
on:
push:
branches: [ main ]
branches: [main]
paths:
- '**.sh'
- '**.bash'
- '**.go'
- '**.cue'
- '**.bats'
- 'Makefile'
- 'go.mod'
- 'go.sum'
- '.github/workflows/dagger-ci.yml'
- "**.sh"
- "**.bash"
- "**.go"
- "**.cue"
- "**.bats"
- "Makefile"
- "go.mod"
- "go.sum"
- ".github/workflows/dagger-ci.yml"
pull_request:
branches: [ main ]
branches: [main]
paths:
- '**.sh'
- '**.bash'
- '**.go'
- '**.cue'
- '**.bats'
- 'Makefile'
- 'go.mod'
- 'go.sum'
- '.github/workflows/dagger-ci.yml'
- "**.sh"
- "**.bash"
- "**.go"
- "**.cue"
- "**.bats"
- "Makefile"
- "go.mod"
- "go.sum"
- ".github/workflows/dagger-ci.yml"
env:
DAGGER_LOG_FORMAT: plain
jobs:
build:
name: "Build"
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Dagger CI
uses: dagger/dagger-for-github@v2
- name: "Setup Go"
uses: actions/setup-go@v1
with:
workdir: ci
args: do build
go-version: 1.16
- name: "Expose GitHub Runtime"
uses: crazy-max/ghaction-github-runtime@v1
- name: Prepare dagger
run: |
make dagger
cp ./cmd/dagger/dagger /usr/local/bin
- name: Build
env:
DAGGER_CACHE_TO: "type=gha,mode=max,scope=dagger-ci-build"
DAGGER_CACHE_FROM: "type=gha,scope=dagger-ci-build"
run: |
dagger do build
lint:
name: "Lint"
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: "Setup Go"
uses: actions/setup-go@v1
with:
go-version: 1.16
- name: "Expose GitHub Runtime"
uses: crazy-max/ghaction-github-runtime@v1
- name: Prepare dagger
run: |
make dagger
cp ./cmd/dagger/dagger /usr/local/bin
- name: Lint
env:
DAGGER_CACHE_TO: "type=gha,mode=max,scope=dagger-ci-lint"
DAGGER_CACHE_FROM: "type=gha,scope=dagger-ci-lint"
run: |
dagger do lint

150
ci.cue Normal file
View File

@ -0,0 +1,150 @@
package main
import (
"dagger.io/dagger"
"universe.dagger.io/bash"
"universe.dagger.io/alpine"
"universe.dagger.io/docker"
"universe.dagger.io/go"
"github.com/dagger/dagger/ci/golangci"
"github.com/dagger/dagger/ci/shellcheck"
"github.com/dagger/dagger/ci/markdownlint"
)
dagger.#Plan & {
// FIXME: Ideally we would want to automatically set the platform's arch identical to the host
// to avoid the performance hit caused by qemu (linter goes from <3s to >3m when arch is x86)
// Uncomment if running locally on Mac M1 to bypass qemu
// platform: "linux/aarch64"
platform: "linux/amd64"
client: filesystem: ".": read: exclude: [
"bin",
"**/node_modules",
"cmd/dagger/dagger",
"cmd/dagger/dagger-debug",
]
client: filesystem: "./bin": write: contents: actions.build.output
actions: {
_source: client.filesystem["."].read.contents
// FIXME: this can be removed once `go` supports built-in VCS info
version: {
_image: alpine.#Build & {
packages: bash: _
packages: curl: _
packages: git: _
}
_revision: bash.#Run & {
input: _image.output
workdir: "/src"
mounts: source: {
dest: "/src"
contents: _source
}
script: contents: #"""
printf "$(git rev-parse --short HEAD)" > /revision
"""#
export: files: "/revision": string
}
output: _revision.export.files["/revision"]
}
build: go.#Build & {
source: _source
package: "./cmd/dagger/"
os: client.platform.os
arch: client.platform.arch
ldflags: "-s -w -X go.dagger.io/dagger/version.Revision=\(version.output)"
env: {
CGO_ENABLED: "0"
// Makes sure the linter and unit tests complete before starting the build
// "__depends_lint": "\(goLint.exit)"
// "__depends_tests": "\(goTest.exit)"
}
}
// Go unit tests
test: go.#Test & {
// container: image: _goImage.output
source: _source
package: "./..."
// FIXME: doesn't work with CGO_ENABLED=0
// command: flags: "-race": true
env: {
// FIXME: removing this complains about lack of gcc
CGO_ENABLED: "0"
}
}
lint: {
go: golangci.#Lint & {
source: _source
version: "1.45"
}
shell: shellcheck.#Lint & {
source: _source
}
markdown: markdownlint.#Lint & {
source: _source
files: ["./docs", "README.md"]
}
cue: docker.#Build & {
// FIXME: spin off into its own package?
steps: [
alpine.#Build & {
packages: bash: _
packages: curl: _
packages: git: _
},
docker.#Copy & {
contents: _source
source: "go.mod"
dest: "go.mod"
},
// Install CUE
bash.#Run & {
script: contents: #"""
export CUE_VERSION="$(grep cue ./go.mod | cut -d' ' -f2 | head -1 | sed -E 's/\.[[:digit:]]\.[[:alnum:]]+-[[:alnum:]]+$//')"
export CUE_TARBALL="cue_${CUE_VERSION}_linux_amd64.tar.gz"
echo "Installing cue version $CUE_VERSION"
curl -L "https://github.com/cue-lang/cue/releases/download/${CUE_VERSION}/${CUE_TARBALL}" | tar zxf - -C /usr/local/bin
cue version
"""#
},
// CACHE: copy only *.cue files
docker.#Copy & {
contents: _source
include: ["*.cue"]
dest: "/cue"
},
// LINT
bash.#Run & {
workdir: "/cue"
script: contents: #"""
find . -name '*.cue' -not -path '*/cue.mod/*' -print | time xargs -t -n 1 -P 8 cue fmt -s
test -z "$(git status -s . | grep -e "^ M" | grep "\.cue" | cut -d ' ' -f3 | tee /dev/stderr)"
"""#
},
]
}
}
}
}

1
ci/.gitignore vendored
View File

@ -1 +0,0 @@
build/*

7
ci/README.md
View File

@ -1,7 +0,0 @@
# Dagger CI
WORK IN PROGRESS
This is the home of the new CI
Reference: https://github.com/dagger/dagger/issues/1549

37
ci/golangci/lint.cue Normal file
View File

@ -0,0 +1,37 @@
package golangci
import (
"dagger.io/dagger"
"universe.dagger.io/docker"
"universe.dagger.io/go"
)
// Lint using golangci-lint
#Lint: {
// Source code
source: dagger.#FS
// golangci-lint version
version: *"1.45" | string
// timeout
timeout: *"5m" | string
_image: docker.#Pull & {
source: "golangci/golangci-lint:v\(version)"
}
container: go.#Container & {
"source": source
input: _image.output
command: {
name: "golangci-lint"
flags: {
run: true
"-v": true
"--timeout": timeout
}
}
}
}

80
ci/images.cue
View File

@ -1,80 +0,0 @@
package main
import (
"universe.dagger.io/docker"
)
let GoVersion = "1.17"
let GolangCILintVersion = "1.44.0"
let CUEVersion = "0.4.2"
// Base container images used for the CI
#Images: {
// base image to build go binaries
goBuilder: _goBuilder.output
_goBuilder: docker.#Build & {
_packages: ["bash", "git", "alpine-sdk"]
steps: [
docker.#Pull & {
source: "index.docker.io/golang:\(GoVersion)-alpine"
},
for pkg in _packages {
docker.#Run & {
command: {
name: "apk"
args: ["add", pkg]
flags: {
"-U": true
"--no-cache": true
}
}
}
},
]
}
// base image for the Go linter
// https://golangci-lint.run/usage/install/#docker
goLinter: _goLinter.output
_goLinter: docker.#Pull & {
source: "index.docker.io/golangci/golangci-lint:v\(GolangCILintVersion)"
}
// base image for CUE cli + alpine distrib
cue: _cue._alpine.output
_cue: {
_cueBinary: docker.#Pull & {
source: "index.docker.io/cuelang/cue:\(CUEVersion)"
}
_alpine: docker.#Build & {
_packages: ["bash", "git"]
steps: [
docker.#Pull & {
source: "index.docker.io/alpine:3"
},
for pkg in _packages {
docker.#Run & {
command: {
name: "apk"
args: ["add", pkg]
flags: {
"-U": true
"--no-cache": true
}
}
}
},
docker.#Copy & {
// input: _alpine.output
contents: _cueBinary.output.rootfs
source: "/usr/bin/cue"
dest: "/usr/bin/cue"
},
]
}
}
}

99
ci/main.cue
View File

@ -1,99 +0,0 @@
package main
import (
"dagger.io/dagger"
"dagger.io/dagger/core"
"universe.dagger.io/bash"
)
dagger.#Plan & {
// FIXME: Ideally we would want to automatically set the platform's arch identical to the host
// to avoid the performance hit caused by qemu (linter goes from <3s to >3m when arch is x86)
// Uncomment if running locally on Mac M1 to bypass qemu
// platform: "linux/aarch64"
platform: "linux/amd64"
client: filesystem: "./build": write: contents: actions.build.export.directories["/build"]
actions: {
_mountGoCache: {
mounts: "go mod cache": {
dest: "/root/.gocache"
contents: core.#CacheDir & {
id: "go mod cache"
}
}
env: GOMODCACHE: mounts["go mod cache"].dest
}
_mountSourceCode: {
mounts: "dagger source code": {
contents: _source.output
dest: "/usr/src/dagger"
}
workdir: mounts["dagger source code"].dest
}
_baseImages: #Images
// Go build the dagger binary
// depends on goLint and goTest to complete successfully
build: bash.#Run & {
_mountSourceCode
_mountGoCache
input: _baseImages.goBuilder
env: {
GOOS: client.platform.os
GOARCH: client.platform.arch
CGO_ENABLED: "0"
// Makes sure the linter and unit tests complete before starting the build
"__depends_lint": "\(goLint.exit)"
"__depends_tests": "\(goTest.exit)"
}
script: contents: #"""
mkdir -p /build
git_revision=$(git rev-parse --short HEAD)
go build -v -o /build/dagger \
-ldflags '-s -w -X go.dagger.io/dagger/version.Revision='${git_revision} \
./cmd/dagger/
"""#
export: directories: "/build": _
}
// Go unit tests
goTest: bash.#Run & {
_mountSourceCode
_mountGoCache
input: _baseImages.goBuilder
script: contents: "go test -race -v ./..."
}
// Go lint using golangci-lint
goLint: bash.#Run & {
_mountSourceCode
_mountGoCache
input: _baseImages.goLinter
script: contents: "golangci-lint run -v --timeout 5m"
}
// CUE lint
cueLint: bash.#Run & {
_mountSourceCode
input: _baseImages.cue
script: contents: #"""
# Format the cue code
find . -name '*.cue' -not -path '*/cue.mod/*' -print | time xargs -n 1 -P 8 cue fmt -s
# Check that all formatted files where committed
test -z $(git status -s . | grep -e '^ M' | grep .cue | cut -d ' ' -f3)
"""#
}
}
}

36
ci/markdownlint/markdownlint.cue Normal file
View File

@ -0,0 +1,36 @@
package markdownlint
import (
"dagger.io/dagger"
"universe.dagger.io/docker"
)
#Lint: {
// Source code
source: dagger.#FS
// shellcheck version
version: *"0.31.1" | string
// Files to lint
files: [...string]
_image: docker.#Pull & {
source: "tmknom/markdownlint:\(version)"
}
container: docker.#Run & {
input: _image.output
mounts: "source": {
dest: "/src"
contents: source
}
workdir: "/src"
command: {
// FIXME: this should not be required
name: "markdownlint"
args: files
}
}
}

34
ci/shellcheck/shellcheck.cue Normal file
View File

@ -0,0 +1,34 @@
package shellcheck
import (
"dagger.io/dagger"
"universe.dagger.io/docker"
)
#Lint: {
// Source code
source: dagger.#FS
// shellcheck version
version: *"0.8.0" | string
_image: docker.#Pull & {
source: "koalaman/shellcheck-alpine:v\(version)"
}
container: docker.#Run & {
input: _image.output
mounts: "source": {
dest: "/src"
contents: source
}
workdir: "/src"
command: {
name: "sh"
args: ["-c", #"""
shellcheck $(find . -type f \( -iname \*.bats -o -iname \*.bash -o -iname \*.sh \) -not -path "*/node_modules/*" -not -path "*/bats-*/*")
"""#]
}
}
}

2
cue.mod/module.cue
View File

@ -1 +1 @@
module: ""
module: "github.com/dagger/dagger"

13
pkg/universe.dagger.io/go/build.cue
View File

@ -13,10 +13,10 @@ import (
package: *"." | string
// Target architecture
arch: *"amd64" | string
arch?: string
// Target OS
os: *"linux" | string
os?: string
// Build tags to use for building
tags: *"" | string
@ -30,10 +30,15 @@ import (
"source": source
"env": {
env
GOOS: os
GOARCH: arch
if os != _|_ {
GOOS: os
}
if arch != _|_ {
GOARCH: arch
}
}
command: {
name: "go"
args: [package]
flags: {
build: true

22
pkg/universe.dagger.io/go/container.cue
View File

@ -18,25 +18,31 @@ import (
// Use go image
_image: #Image
_sourcePath: "/src"
_cachePath: "/root/.cache/gocache"
_sourcePath: "/src"
_modCachePath: "/root/.cache/go-mod"
_buildCachePath: "/root/.cache/go-build"
docker.#Run & {
input: *_image.output | docker.#Image
workdir: "/src"
command: name: "go"
workdir: _sourcePath
mounts: {
"source": {
dest: _sourcePath
contents: source
}
"go assets cache": {
"go mod cache": {
contents: core.#CacheDir & {
id: "\(name)_assets"
id: "\(name)_mod"
}
dest: _cachePath
dest: _modCachePath
}
"go build cache": {
contents: core.#CacheDir & {
id: "\(name)_build"
}
dest: _buildCachePath
}
}
env: GOMODCACHE: _cachePath
env: GOMODCACHE: _modCachePath
}
}

1
pkg/universe.dagger.io/go/test.cue
View File

@ -7,6 +7,7 @@ package go
#Container & {
command: {
name: "go"
args: [package]
flags: {
test: true

10
pkg/universe.dagger.io/go/test/container.cue
View File

@ -12,7 +12,10 @@ dagger.#Plan & {
simple: go.#Container & {
source: _source
command: args: ["version"]
command: {
name: "go"
args: ["version"]
}
}
override: {
@ -23,7 +26,10 @@ dagger.#Plan & {
command: go.#Container & {
input: base.output
source: _source
command: args: ["version"]
command: {
name: "go"
args: ["version"]
}
}
}
}

5
plan/task/clientfilesystemread.go
View File

@ -125,12 +125,9 @@ func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.
opts = append(opts, llb.IncludePatterns(dir.Include))
}
// Excludes .dagger directory by default
excludePatterns := []string{"**/.dagger/"}
if len(dir.Exclude) > 0 {
excludePatterns = dir.Exclude
opts = append(opts, llb.ExcludePatterns(dir.Exclude))
}
opts = append(opts, llb.ExcludePatterns(excludePatterns))
// FIXME: Remove the `Copy` and use `Local` directly.
//

2
plan/task/source.go
View File

@ -69,7 +69,7 @@ func (c *sourceTask) Run(ctx context.Context, pctx *plancontext.Context, s *solv
lg.Debug().Str("path", path).Msg("loading local directory")
opts := []llb.LocalOption{
withCustomName(v, "Embed %s", path),
withCustomName(v, "Source %s", path),
llb.IncludePatterns(source.Include),
llb.ExcludePatterns(source.Exclude),
// Without hint, multiple `llb.Local` operations on the

15
source.cue
View File

@ -1,15 +0,0 @@
package main
import (
"dagger.io/dagger/core"
)
_source: core.#Source & {
path: "."
exclude: [
"ci",
"node_modules",
"cmd/dagger/dagger",
"cmd/dagger/dagger-debug",
]
}