[allow-insecure] Allow insecure packages with --allow-insecure flag (#…

…1265)

## Summary

This allows installing insecure packages using the `--allow-insecure`
flag:

`devbox add nodejs@16 --allow-insecure`

This saves the allow insecure state to lock file.

If user tries to do add/shell/run/install and there are insecure
pacakges that are not in marked in lock file, they will see an error
indicating they should use flag.

I used flag (instead of prompt) to limit the size of this already
massive PR. TODO (in follow up):

* When installing an insecure package, ask the user if they want to
allow it, update the devbox.json, and install it.

## How was it tested?

```bash
devbox add nodejs@16
devbox add nodejs@16 --allow-insecure
devbox run run_test
# edited lockfile to remove allow_insecure
devbox run run_test # error
devbox install # error
devbox shell # error
```

See examples/insecure

devbox.json

@@ -22,4 +22,4 @@
  "nixpkgs": {
  "commit": "3364b5b117f65fe1ce65a3cdd5612a078a3b31e3"
  }
-}
+}

examples/insecure/devbox.json

@@ -0,0 +1,15 @@
+{
+ "packages": [
+ "nodejs@16"
+ ],
+ "shell": {
+ "init_hook": [
+ "echo 'Welcome to devbox!' > /dev/null"
+ ],
+ "scripts": {
+ "run_test": [
+ "node --version"
+ ]
+ }
+ }
+}

examples/insecure/devbox.lock

@@ -0,0 +1,12 @@
+{
+ "lockfile_version": "1",
+ "packages": {
+ "nodejs@16": {
+ "allow_insecure": true,
+ "last_modified": "2023-06-29T16:20:38Z",
+ "resolved": "github:NixOS/nixpkgs/3c614fbc76fc152f3e1bc4b2263da6d90adf80fb#nodejs_16",
+ "source": "devbox-search",
+ "version": "16.20.1"
+ }
+ }
+}

internal/boxcli/add.go

@@ -18,7 +18,8 @@ import (
 const toSearchForPackages = "To search for packages, use the `devbox search` command"
 
 type addCmdFlags struct {
- config configFlags
+ config configFlags
+ allowInsecure bool
 }
 
 func addCmd() *cobra.Command {
@@ -47,13 +48,18 @@ func addCmd() *cobra.Command {
  }
 
  flags.config.register(command)
+ command.Flags().BoolVar(
+ &flags.allowInsecure, "allow-insecure", false,
+ "Allow adding packages marked as insecure.")
+
  return command
 }
 
 func addCmdFunc(cmd *cobra.Command, args []string, flags addCmdFlags) error {
  box, err := devbox.Open(&devopt.Opts{
- Dir: flags.config.path,
- Writer: cmd.ErrOrStderr(),
+ Dir: flags.config.path,
+ Writer: cmd.ErrOrStderr(),
+ AllowInsecureAdds: flags.allowInsecure,
  })
  if err != nil {
  return errors.WithStack(err)

internal/devpkg/package.go

@@ -542,3 +542,23 @@ func (p *Package) ContentAddressedPath() (string, error) {
  }
  return localPath, err
 }
+
+func (p *Package) AllowInsecure() bool {
+ return p.lockfile.Get(p.Raw).IsAllowInsecure()
+}
+
+// StoreName returns the last section of the store path. Example:
+// /nix/store/abc123-foo-1.0.0 -> foo-1.0.0
+// Warning, this is probably slowish. If you need to call this multiple times,
+// consider caching the result.
+func (p *Package) StoreName() (string, error) {
+ u, err := p.urlForInstall()
+ if err != nil {
+ return "", err
+ }
+ name, err := nix.EvalPackageName(u)
+ if err != nil {
+ return "", err
+ }
+ return name, nil
+}

internal/impl/devbox.go

@@ -52,12 +52,13 @@ const (
 )
 
 type Devbox struct {
- cfg *devconfig.Config
- lockfile *lock.File
- nix nix.Nixer
- projectDir string
- pluginManager *plugin.Manager
- pure bool
+ cfg *devconfig.Config
+ lockfile *lock.File
+ nix nix.Nixer
+ projectDir string
+ pluginManager *plugin.Manager
+ pure bool
+ allowInsecureAdds bool
 
  // Possible TODO: hardcode this to stderr. Allowing the caller to specify the
  // writer is error prone. Since it is almost always stderr, we should default
@@ -81,18 +82,24 @@ func Open(opts *devopt.Opts) (*Devbox, error) {
  }
 
  box := &Devbox{
- cfg: cfg,
- nix: &nix.Nix{},
- projectDir: projectDir,
- pluginManager: plugin.NewManager(),
- writer: opts.Writer,
- pure: opts.Pure,
+ cfg: cfg,
+ nix: &nix.Nix{},
+ projectDir: projectDir,
+ pluginManager: plugin.NewManager(),
+ writer: opts.Writer,
+ pure: opts.Pure,
+ allowInsecureAdds: opts.AllowInsecureAdds,
  }
 
  lock, err := lock.GetFile(box)
  if err != nil {
  return nil, err
  }
+ // if lockfile has any allow insecure, we need to set the env var to ensure
+ // all nix commands work.
+ if opts.AllowInsecureAdds || lock.HasAllowInsecurePackages() {
+ nix.AllowInsecurePackages()
+ }
  box.pluginManager.ApplyOptions(
  plugin.WithDevbox(box),
  plugin.WithLockfile(lock),

internal/impl/devopt/devboxopts.go

@@ -5,8 +5,9 @@ import (
 )
 
 type Opts struct {
- Dir string
- Pure bool
- IgnoreWarnings bool
- Writer io.Writer
+ AllowInsecureAdds bool
+ Dir string
+ Pure bool
+ IgnoreWarnings bool
+ Writer io.Writer
 }

internal/impl/packages.go

@@ -40,10 +40,16 @@ func (d *Devbox) Add(ctx context.Context, pkgsNames ...string) error {
 
  // Only add packages that are not already in config. If same canonical exists,
  // replace it.
- pkgs := []*devpkg.Package{}
- for _, pkg := range devpkg.PackageFromStrings(lo.Uniq(pkgsNames), d.lockfile) {
+ pkgs := devpkg.PackageFromStrings(lo.Uniq(pkgsNames), d.lockfile)
+
+ // addedPackageNames keeps track of the possibly transformed (versioned)
+ // names of added packages (even if they are already in config). We use this
+ // to know the exact name to mark as allowed insecure later on.
+ addedPackageNames := []string{}
+ for _, pkg := range pkgs {
  // If exact versioned package is already in the config, skip.
  if slices.Contains(d.cfg.Packages, pkg.Versioned()) {
+ addedPackageNames = append(addedPackageNames, pkg.Versioned())
  continue
  }
 
@@ -61,11 +67,30 @@ func (d *Devbox) Add(ctx context.Context, pkgsNames ...string) error {
  // if not, fallback to legacy vanilla nix.
  versionedPkg := devpkg.PackageFromString(pkg.Versioned(), d.lockfile)
  ok, err := versionedPkg.ValidateExists()
+ packageNameForConfig := pkg.Raw
  if err == nil && ok {
- d.cfg.Packages = append(d.cfg.Packages, pkg.Versioned())
- } else {
- // fallthrough and treat package as a legacy package.
- d.cfg.Packages = append(d.cfg.Packages, pkg.Raw)
+ // Only use versioned if it exists in search.
+ packageNameForConfig = pkg.Versioned()
+ }
+ // else {
+ // // TODO (landau): use nix.Search to check if this package exists
+ // // fallthrough and treat package as a legacy package.
+ // }
+
+ d.cfg.Packages = append(d.cfg.Packages, packageNameForConfig)
+ addedPackageNames = append(addedPackageNames, packageNameForConfig)
+ }
+
+ // Resolving here ensures we allow insecure before running ensurePackagesAreInstalled
+ // which will call print-dev-env. Resolving does not save the lockfile, we
+ // save at the end when everything has succeeded.
+ if d.allowInsecureAdds {
+ for _, name := range addedPackageNames {
+ p, err := d.lockfile.Resolve(name)
+ if err != nil {
+ return err
+ }
+ p.AllowInsecure = true
  }
  }
 
@@ -88,9 +113,7 @@ func (d *Devbox) Add(ctx context.Context, pkgsNames ...string) error {
  }
  }
 
- if err := d.lockfile.Add(
- lo.Map(pkgs, func(pkg *devpkg.Package, _ int) string { return pkg.Raw })...,
- ); err != nil {
+ if err := d.lockfile.Save(); err != nil {
  return err
  }
 

internal/lock/lockfile.go

@@ -113,6 +113,15 @@ func (l *File) Get(pkg string) *Package {
  return entry
 }
 
+func (l *File) HasAllowInsecurePackages() bool {
+ for _, pkg := range l.Packages {
+ if pkg.AllowInsecure {
+ return true
+ }
+ }
+ return false
+}
+
 // This probably belongs in input.go but can't add it there because it will
 // create a circular dependency. We could move Input into own package.
 func IsLegacyPackage(pkg string) bool {

internal/lock/package.go

@@ -9,6 +9,7 @@ const (
 )
 
 type Package struct {
+ AllowInsecure bool `json:"allow_insecure,omitempty"`
  LastModified string `json:"last_modified,omitempty"`
  PluginVersion string `json:"plugin_version,omitempty"`
  Resolved string `json:"resolved,omitempty"`
@@ -35,3 +36,10 @@ func (p *Package) GetSource() string {
  }
  return p.Source
 }
+
+func (p *Package) IsAllowInsecure() bool {
+ if p == nil {
+ return false
+ }
+ return p.AllowInsecure
+}

internal/nix/command.go

@@ -1,7 +1,6 @@
 package nix
 
 import (
- "os"
  "os/exec"
 )
 
@@ -12,6 +11,10 @@ func command(args ...string) *exec.Cmd {
  return cmd
 }
 
-func allowUnfreeEnv() []string {
- return append(os.Environ(), "NIXPKGS_ALLOW_UNFREE=1")
+func allowUnfreeEnv(curEnv []string) []string {
+ return append(curEnv, "NIXPKGS_ALLOW_UNFREE=1")
+}
+
+func allowInsecureEnv(curEnv []string) []string {
+ return append(curEnv, "NIXPKGS_ALLOW_INSECURE=1")
 }

internal/nix/eval.go

@@ -0,0 +1,56 @@
+package nix
+
+import (
+ "encoding/json"
+ "os"
+ "strconv"
+)
+
+func EvalPackageName(path string) (string, error) {
+ cmd := command("eval", "--raw", path+".name")
+ out, err := cmd.Output()
+ if err != nil {
+ return "", err
+ }
+ return string(out), nil
+}
+
+// PackageIsInsecure is a fun little nix eval that maybe works.
+func PackageIsInsecure(path string) bool {
+ cmd := command("eval", path+".meta.insecure")
+ out, err := cmd.Output()
+ if err != nil {
+ // We can't know for sure, but probably not.
+ return false
+ }
+ var insecure bool
+ if err := json.Unmarshal(out, &insecure); err != nil {
+ // We can't know for sure, but probably not.
+ return false
+ }
+ return insecure
+}
+
+func PackageKnownVulnerabilities(path string) []string {
+ cmd := command("eval", path+".meta.knownVulnerabilities")
+ out, err := cmd.Output()
+ if err != nil {
+ // We can't know for sure, but probably not.
+ return nil
+ }
+ var vulnerabilities []string
+ if err := json.Unmarshal(out, &vulnerabilities); err != nil {
+ // We can't know for sure, but probably not.
+ return nil
+ }
+ return vulnerabilities
+}
+
+func AllowInsecurePackages() {
+ os.Setenv("NIXPKGS_ALLOW_INSECURE", "1")
+}
+
+func IsInsecureAllowed() bool {
+ allowed, _ := strconv.ParseBool(os.Getenv("NIXPKGS_ALLOW_INSECURE"))
+ return allowed
+}

internal/nix/nix.go

@@ -10,11 +10,13 @@ import (
  "os"
  "os/exec"
  "path/filepath"
+ "regexp"
  "runtime/trace"
  "strings"
 
  "github.com/pkg/errors"
  "go.jetpack.io/devbox/internal/boxcli/featureflag"
+ "go.jetpack.io/devbox/internal/boxcli/usererr"
 
  "go.jetpack.io/devbox/internal/debug"
 )
@@ -69,7 +71,9 @@ func (*Nix) PrintDevEnv(ctx context.Context, args *PrintDevEnvArgs) (*PrintDevEn
  cmd.Args = append(cmd.Args, "--json")
  debug.Log("Running print-dev-env cmd: %s\n", cmd)
  data, err = cmd.Output()
- if err != nil {
+ if insecure, insecureErr := isExitErrorInsecurePackage(err); insecure {
+ return nil, insecureErr
+ } else if err != nil {
  return nil, errors.Wrapf(err, "Command: %s", cmd)
  }
 
@@ -120,6 +124,7 @@ func System() (string, error) {
  // For Savil to debug "remove nixpkgs" feature. The Search api lacks x86-darwin info.
  // So, I need to fake that I am x86-linux and inspect the output in generated devbox.lock
  // and flake.nix files.
+ // This is also used by unit tests.
  override := os.Getenv("__DEVBOX_NIX_SYSTEM")
  if override != "" {
  return override, nil
@@ -144,3 +149,19 @@ func System() (string, error) {
 func ProfileBinPath(projectDir string) string {
  return filepath.Join(projectDir, ProfilePath, "bin")
 }
+
+func isExitErrorInsecurePackage(err error) (bool, error) {
+ var exitErr *exec.ExitError
+ if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 {
+ if strings.Contains(string(exitErr.Stderr), "is marked as insecure") {
+ re := regexp.MustCompile(`Package ([^ ]+)`)
+ match := re.FindStringSubmatch(string(exitErr.Stderr))
+ return true, usererr.New(
+ "Package %s is insecure. \n\n"+
+ "To override use `devbox add <pkg> --allow-insecure`",
+ match[0],
+ )
+ }
+ }
+ return false, nil
+}