[devbox.json] Allowing including plugins explicitly via "include" field. (#986)

## Summary

This adds the ability to include plugins explicitly via:

```
 "include": [
    "plugin:php"
  ]
```

in devbox.json. The idea is to allow including any devbox.json into an
existing one. The proposed syntax is:

* path: for local files
* https:// for remote files
* plugin: for built-in plugins

only plugin is implemented in this PR.

Improved existing plugin mechanism by using `lockfile` to decide is
`devbox.d` directory needs to be created by keeping track of the plugin
version. In the future we could allow upgrading of configs if there is
version mismatch.

A few design questions: (cc: @Lagoja @loreto )

* Should include be merged into `packges` instead? We could introspect
the include to see if it's a flake or a plugin. This would simplify the
devbox.json but it might be a weird way to include other configs.
* One kinda ugly thing of current design is that the php plugin doesn't
actually install PHP. That feels a bit wrong, but the way we've
currently designed plugins is that they trigger on packages so not sure
how to improve this. One option is to rename `builtin` to something that
makes it more explicit that it extends functionality on existing nix
packages. Maybe `extend:php`. Another option: In the future I expect
includes to contain packages which would be installed as well. So could
parametrize the php plugin so it can take the top level php package as
an input (but otherwise it could install its own)

Edit:

After chatting with @loreto and @Lagoja we agreed to keep this design
with following:
 
* replace "builtin" with "plugin". 
* Including plugins works the same way current plugins. Specifically
init hooks, and environment get added to existing environment. No new
packages are installed.

## How was it tested?

In php flake example, did `devbox services up`

Author: mikeland73

examples/flakes/php/devbox.json

@@ -8,5 +8,8 @@
   },
   "nixpkgs": {
     "commit": "f80ac848e3d6f0c12c52758c0f25c10c97ca3b62"
-  }
-}
+  },
+  "include": [
+    "plugin:php"
+  ]
+}

internal/impl/config.go

@@ -40,6 +40,13 @@ type Config struct {
 
 	// Nixpkgs specifies the repository to pull packages from
 	Nixpkgs NixpkgsConfig `json:"nixpkgs,omitempty"`
+
+	// Reserved to allow including other config files. Proposed format is:
+	// path: for local files
+	// https:// for remote files
+	// plugin: for built-in plugins
+	// This is a similar format to nix inputs
+	Include []string `json:"include,omitempty"`
 }
 
 type NixpkgsConfig struct {

internal/impl/devbox.go

@@ -125,6 +125,7 @@ func Open(path string, writer io.Writer) (*Devbox, error) {
 	if err != nil {
 		return nil, err
 	}
+	box.pluginManager.ApplyOptions(plugin.WithLockfile(lock))
 	box.lockfile = lock
 	return box, nil
 }
@@ -448,7 +449,11 @@ func (d *Devbox) saveCfg() error {
 }
 
 func (d *Devbox) Services() (services.Services, error) {
-	pluginSvcs, err := plugin.GetServices(d.packagesAsInputs(), d.projectDir)
+	pluginSvcs, err := d.pluginManager.GetServices(
+		d.packagesAsInputs(),
+		d.cfg.Include,
+		d.projectDir,
+	)
 	if err != nil {
 		return nil, err
 	}
@@ -782,7 +787,8 @@ func (d *Devbox) computeNixEnv(ctx context.Context, usePrintDevEnvCache bool) (m
 	// We still need to be able to add env variables to non-service binaries
 	// (e.g. ruby). This would involve understanding what binaries are associated
 	// to a given plugin.
-	pluginEnv, err := plugin.Env(d.packagesAsInputs(), d.projectDir, env)
+	pluginEnv, err := d.pluginManager.Env(
+		d.packagesAsInputs(), d.cfg.Include, d.projectDir, env)
 	if err != nil {
 		return nil, err
 	}

internal/impl/generate.go

@@ -61,7 +61,16 @@ func (d *Devbox) generateShellFiles() error {
 	}
 
 	for _, pkg := range d.packagesAsInputs() {
-		if err := d.pluginManager.CreateFilesAndShowReadme(d.writer, pkg, d.projectDir); err != nil {
+		if err := d.pluginManager.Create(d.writer, pkg, d.projectDir); err != nil {
+			return err
+		}
+	}
+
+	for _, included := range d.cfg.Include {
+		if err := d.lockfile.Add(included); err != nil {
+			return err
+		}
+		if err := d.pluginManager.Include(d.writer, included, d.projectDir); err != nil {
 			return err
 		}
 	}

internal/impl/packages.go

@@ -63,7 +63,6 @@ func (d *Devbox) Add(ctx context.Context, pkgsNames ...string) error {
 		}
 	}
 
-	d.pluginManager.ApplyOptions(plugin.WithAddMode())
 	if err := d.ensurePackagesAreInstalled(ctx, install); err != nil {
 		return usererr.WithUserMessage(
 			err,

internal/impl/update.go

@@ -25,7 +25,7 @@ func (d *Devbox) Update(ctx context.Context, pkgs ...string) error {
 			fmt.Fprintf(d.writer, "Skipping %s because it is not a versioned package\n", pkg)
 			continue
 		}
-		existing := d.lockfile.Entry(pkg)
+		existing := d.lockfile.Packages[pkg]
 		newEntry, err := d.lockfile.ForceResolve(pkg)
 		if err != nil {
 			return err

internal/lock/resolvers.go renamed to internal/lock/interfaces.go

@@ -26,9 +26,10 @@ type File struct {
 }
 
 type Package struct {
-	LastModified string `json:"last_modified"`
-	Resolved     string `json:"resolved"`
-	Version      string `json:"version"`
+	LastModified  string `json:"last_modified,omitempty"`
+	PluginVersion string `json:"plugin_version,omitempty"`
+	Resolved      string `json:"resolved,omitempty"`
+	Version       string `json:"version,omitempty"`
 }
 
 func GetFile(project devboxProject, resolver resolver) (*File, error) {
@@ -51,10 +52,8 @@ func GetFile(project devboxProject, resolver resolver) (*File, error) {
 
 func (l *File) Add(pkgs ...string) error {
 	for _, p := range pkgs {
-		if IsVersionedPackage(p) {
-			if _, err := l.Resolve(p); err != nil {
-				return err
-			}
+		if _, err := l.Resolve(p); err != nil {
+			return err
 		}
 	}
 	return l.Save()
@@ -71,14 +70,14 @@ func (l *File) Remove(pkgs ...string) error {
 // This avoids writing values that may need to be removed in case of error.
 func (l *File) Resolve(pkg string) (*Package, error) {
 	if entry, ok := l.Packages[pkg]; !ok || entry.Resolved == "" {
-		var locked *Package
+		locked := &Package{}
 		var err error
 		if IsVersionedPackage(pkg) {
 			locked, err = l.resolver.Resolve(pkg)
 			if err != nil {
 				return nil, err
 			}
-		} else {
+		} else if IsLegacyPackage(pkg) {
 			// These are legacy packages without a version. Resolve to nixpkgs with
 			// whatever hash is in the devbox.json
 			locked = &Package{Resolved: l.LegacyNixpkgsPath(pkg)}
@@ -94,10 +93,6 @@ func (l *File) ForceResolve(pkg string) (*Package, error) {
 	return l.Resolve(pkg)
 }
 
-func (l *File) Entry(pkg string) *Package {
-	return l.Packages[pkg]
-}
-
 func (l *File) Save() error {
 	// Never write lockfile if versioned packages is not enabled
 	if !featureflag.LockFile.Enabled() {
@@ -120,6 +115,18 @@ func IsVersionedPackage(pkg string) bool {
 	return found && name != "" && version != ""
 }
 
+// This probably belongs in input.go but can't add it there because it will
+// create a circular dependency. We could move Input into own package.
+func IsLegacyPackage(pkg string) bool {
+	return !IsVersionedPackage(pkg) &&
+		!strings.Contains(pkg, ":") &&
+		// We don't support absolute paths without "path:" prefix, but adding here
+		// just inc ase we ever do.
+		// Landau note: I don't think we should support it, it's hard to read and a
+		// bit ambiguous.
+		!strings.HasPrefix(pkg, "/")
+}
+
 func lockFilePath(project devboxProject) string {
 	return filepath.Join(project.ProjectDir(), "devbox.lock")
 }

internal/lock/lockfile.go

@@ -0,0 +1,18 @@
+package plugin
+
+import (
+	"strings"
+
+	"go.jetpack.io/devbox/internal/boxcli/usererr"
+	"go.jetpack.io/devbox/internal/nix"
+)
+
+func (m *Manager) parseInclude(include string) (*nix.Input, error) {
+	includeType, name, _ := strings.Cut(include, ":")
+	if includeType != "plugin" {
+		return nil, usererr.New("unknown include type %q", includeType)
+	} else if name == "" {
+		return nil, usererr.New("include name is required")
+	}
+	return nix.InputFromString(name, m.lockfile), nil
+}

internal/plugin/includes.go

@@ -3,8 +3,10 @@
 
 package plugin
 
+import "go.jetpack.io/devbox/internal/lock"
+
 type Manager struct {
-	addMode bool
+	lockfile *lock.File
 }
 
 type managerOption func(*Manager)
@@ -15,9 +17,9 @@ func NewManager(opts ...managerOption) *Manager {
 	return m
 }
 
-func WithAddMode() managerOption {
+func WithLockfile(lockfile *lock.File) managerOption {
 	return func(m *Manager) {
-		m.addMode = true
+		m.lockfile = lockfile
 	}
 }