Skip to content

Add cache builtin #308

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions builtin/lookup.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion codegen/builtin.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ var (
"git": Git{},
"local": Local{},
"frontend": Frontend{},
"cache": Cache{},
"run": Run{},
"env": Env{},
"dir": Dir{},
Expand Down Expand Up @@ -108,7 +109,7 @@ var (
"readonly": Readonly{},
"tmpfs": Tmpfs{},
"sourcePath": SourcePath{},
"cache": Cache{},
"cache": MountCache{},
},
"option::mkdir": {
"createParents": CreateParents{},
Expand Down
68 changes: 68 additions & 0 deletions codegen/builtin_fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,74 @@ func (f Frontend) Call(ctx context.Context, cln *client.Client, val Value, opts
return NewValue(ctx, fs)
}

type Cache struct{}

func (c Cache) Call(ctx context.Context, cln *client.Client, val Value, opts Option, input Filesystem, ref string) (Value, error) {
inputDgst, err := input.Digest(ctx)
if err != nil {
return nil, Arg(ctx, 1).WithError(err)
}
named, err := reference.ParseNormalizedNamed(ref)
if err != nil {
return nil, errdefs.WithInvalidImageRef(err, Arg(ctx, 1), ref)
}
namedTagged, err := reference.WithTag(named, inputDgst.Encoded())
if err != nil {
return nil, Arg(ctx, 1).WithError(err)
}
ref = namedTagged.String()

resolver := ImageResolver(ctx)
if resolver != nil {
resolveOpt := llb.ResolveImageConfigOpt{
Platform: &input.Platform,
}

dgst, config, err := resolver.ResolveImageConfig(ctx, ref, resolveOpt)
if err == nil {
var imageOpts []llb.ImageOption
imageOpts = append(imageOpts, llb.Platform(input.Platform))
for _, opt := range SourceMap(ctx) {
imageOpts = append(imageOpts, opt)
}

canonical, err := reference.WithDigest(named, dgst)
if err != nil {
return nil, errdefs.WithInvalidImageRef(err, Arg(ctx, 1), ref)
}

cacheVal, err := NewValue(ctx, llb.Image(canonical.String()))
if err != nil {
return nil, Arg(ctx, 1).WithError(err)
}

input, err = cacheVal.Filesystem()
if err != nil {
return nil, Arg(ctx, 1).WithError(err)
}

input.State, err = input.State.WithImageConfig(config)
if err != nil {
return nil, Arg(ctx, 1).WithError(err)
}

input.Image = &solver.ImageSpec{}
err = json.Unmarshal(config, input.Image)
if err != nil {
return nil, Arg(ctx, 1).WithError(err)
}
} else { // not found
inputVal, err := NewValue(ctx, input)
if err != nil {
return nil, Arg(ctx, 0).WithError(err)
}
return (DockerPush{}).Call(ctx, cln, inputVal, opts, ref)
}
}

return NewValue(ctx, input)
}

type Env struct{}

func (e Env) Call(ctx context.Context, cln *client.Client, val Value, opts Option, key, value string) (Value, error) {
Expand Down
10 changes: 5 additions & 5 deletions codegen/builtin_option.go
Original file line number Diff line number Diff line change
Expand Up @@ -693,10 +693,10 @@ func (m Mount) Call(ctx context.Context, cln *client.Client, val Value, opts Opt
return nil, err
}

var cache *Cache
var cache *MountCache
for _, opt := range opts {
var ok bool
cache, ok = opt.(*Cache)
cache, ok = opt.(*MountCache)
if ok {
break
}
Expand Down Expand Up @@ -858,11 +858,11 @@ func (sp SourcePath) Call(ctx context.Context, cln *client.Client, val Value, op
return NewValue(ctx, append(retOpts, llbutil.WithSourcePath(path)))
}

type Cache struct {
type MountCache struct {
ast.Node
}

func (c Cache) Call(ctx context.Context, cln *client.Client, val Value, opts Option, id, mode string) (Value, error) {
func (mc MountCache) Call(ctx context.Context, cln *client.Client, val Value, opts Option, id, mode string) (Value, error) {
retOpts, err := val.Option()
if err != nil {
return nil, err
Expand All @@ -880,7 +880,7 @@ func (c Cache) Call(ctx context.Context, cln *client.Client, val Value, opts Opt
return nil, errdefs.WithInvalidSharingMode(Arg(ctx, 1), mode, []string{"shared", "private", "locked"})
}

retOpts = append(retOpts, &Cache{ProgramCounter(ctx)}, llbutil.WithPersistentCacheDir(id, sharing))
retOpts = append(retOpts, &MountCache{ProgramCounter(ctx)}, llbutil.WithPersistentCacheDir(id, sharing))
return NewValue(ctx, retOpts)
}

Expand Down
16 changes: 16 additions & 0 deletions docs/reference.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,20 @@
## <span class='hlb-type'>fs</span> functions
### <span class='hlb-type'>fs</span> <span class='hlb-name'>cache</span>(<span class='hlb-type'>fs</span> <span class='hlb-variable'>input</span>, <span class='hlb-type'>string</span> <span class='hlb-variable'>ref</span>)

!!! info "<span class='hlb-type'>fs</span> <span class='hlb-variable'>input</span>"

!!! info "<span class='hlb-type'>string</span> <span class='hlb-variable'>ref</span>"




#!hlb
fs default() {
cache scratch "ref"
}



### <span class='hlb-type'>fs</span> <span class='hlb-name'>cmd</span>(<span class='hlb-type'>string</span> <span class='hlb-variable'>args</span>)

!!! info "<span class='hlb-type'>string</span> <span class='hlb-variable'>args</span>"
Expand Down
8 changes: 8 additions & 0 deletions language/builtin.hlb
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,14 @@
# @return a scratch filesystem.
fs scratch()

# Caches an input fs by its vertex digest.
# If the image exists then use that image instead of executing input.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should probably note that this pushes the image as well as building it.

#
# @param input a filesystem to cache by vertex digest.
# @param ref a docker registry reference.
# @return a filesystem of input from either building or image.
fs cache(fs input, string ref)

# An OCI image's filesystem.
#
# @param ref a docker registry reference. if not fully qualified, it will be
Expand Down