Change format of generate gemset.nix files

This commit is an attempt to make Bundix more "platform-aware."

Every rubygem has a "platform" field which names the ruby-platform that
it can run on. For the vast majority of gems, this will be `"ruby"`,
indicating that it's a gem written entirely in ruby and should be able
to run on any ruby interpreter; however, there are some gems that rely
on native extensions that might only work on certain systems.

The previous `gemset.nix` format is generated exactly from the contents
of the inputted `Gemfile.lock`. So if the `Gemfile.lock` included a
reference to `nokogiri (1.14.2)`, Bundix would render a `gemset.nix`
that included the sha256 hash for the ruby-platform version of that gem.
The trouble with this behaviour is that both RubyGems and Bundler, at
install time, will want to install the platform-specific version of the
gem most suitable for the host system. So even though the `Gemfile.lock`
references `nokogiri (1.14.2)`, if your host system's platform were
`x86_64-linux`, they will try to install `nokogiri (1.14.2-x86-linux)`
instead. This leads to issues like [1] and [2].

The new format of `gemfile.nix` is:

```nix
{
  dependencies = ["list" "of" "top-level" "dependencies"];
  platforms = {
    ruby = {
      gem-name = {
        dependencies = ["list" "of" 'transitive" "dependencies"]
        groups = [ ... ];
        version = "...";
        source = {
          # attrs describing the git, path, or RubyGems source
        };
      };
    };
    other-platform = {
      gem-name = { ... };
    };
  };
}
```

The gemset's `dependencies` entry copies the `DEPENDENCIES` section of
the `Gemfile.lock` and names all the top-level gem dependencies from the
`Gemfile`.

The gemset's `platforms` entry is an attrset where every key is a
ruby-platform and its value is an attrset of the gems particular to that
platform. This last attrset is essentially the same as the previous
`gemset.nix` format.

This commit also introduces a new nix function: `bundixEnv` (like
`bundlerEnv`, but for Bundix). This function accepts the same arguments
as `bundlerEnv`, with the addition of a `platform`. `bundixEnv` then
converts the given gemset into a format suitable for `bundlerEnv` by
selecting the gems appropriate for the given `platform`. Finally, it
delegates to `bundlerEnv`, with the platform-specific gemset.

`bundix --init` will generate an example `flake.nix` with an example
package that demonstrates how `bundixEnv` works.

See `gem help platforms` for more info about ruby platforms. The Bundler
Guide [3] provides a few examples of rubygems that support multiple
platforms.

[1] nix-community#71
[2] https://discourse.nixos.org/t/issues-with-nix-reproducibility-on-macos-trying-to-build-nokogiri-ruby-error-unknown-warning-option/22019
[3] https://guides.rubygems.org/gems-with-extensions/

flake.nix

@@ -7,63 +7,31 @@
   };
 
   outputs = { self, nixpkgs, flake-utils }:
+    {
+      overlays.default = import ./nix/overlay.nix rec {
+        pname = "bundix";
+        src = ./.;
+        extraConfigPaths = [
+          "${./.}/lib" # .gemspec file references `Bundix::Version`
+          "${./.}/${pname}.gemspec"
+        ];
+        versionRubyFile = ./lib/bundix/version.rb;
+      };
+    } //
     flake-utils.lib.eachDefaultSystem (system:
       let
-        name = "bundix";
-        version = extract-bundix-version ./lib/bundix/version.rb;
-        pkgs = import nixpkgs { inherit system; };
-
-        gems = with pkgs; bundlerEnv {
-          inherit name ruby;
-          gemdir = ./.;
-          extraConfigPaths = [
-            "${./.}/lib" # .gemspec file references `Bundix::Version`
-            "${./.}/${name}.gemspec"
-          ];
-        };
-
-        extract-bundix-version = path: with builtins;
-          let
-            pattern = ".*VERSION[[:space:]]*=[[:space:]]['\"]([^'\"]+)['\"].*";
-            captures = match pattern (readFile path);
-            version-list = if isNull captures || length captures == 0
-                           then [upstream-package.version]
-                           else captures;
-          in elemAt version-list 0;
-
-        upstream-package = import ./default.nix {
-          inherit pkgs;
-          inherit (gems) ruby;
+        pkgs = import nixpkgs {
+          inherit system;
+          overlays = [self.overlays.default];
         };
-        bundled-package = upstream-package.overrideAttrs (_old: {
-          inherit name version;
-
-          # See https://nixos.wiki/wiki/Packaging/Ruby#Build_default.nix
-          installPhase = ''
-            mkdir -p $out/{bin,share/${name}}
-            cp -r $src/{bin,lib,template} $out/share/${name}
-
-            bin=$out/bin/${name}
-            cat > $bin <<EOF
-            #!/bin/sh -e
-            exec ${gems}/bin/bundle exec \
-              ${gems.ruby}/bin/ruby \
-              $out/share/${name}/bin/${name} "\$@"
-            EOF
-            chmod +x $bin
-          '';
-        });
       in {
         packages = {
-          default = bundled-package;
-          gems = gems;
+          default = pkgs.bundix;
+          bundixEnv = pkgs.bundixEnv;
         };
 
         devShell = pkgs.mkShell {
-          buildInputs = [
-            gems
-            gems.ruby
-          ];
+          buildInputs = with pkgs.bundix; [gems gems.ruby];
         };
       }
     );

lib/bundix/bundler_proxy.rb

@@ -5,33 +5,39 @@
 
 module Bundix
   module BundlerProxy
-    # A base class for services which execute Bundler CLI classes.
+    # A base class for services which execute inside a modified Bundler
+    # environment.
     #
-    # The reason for executing bundle CLI aps via its {Bundler} ruby library,
-    # instead of the +bundle+ executable, is that +bundle+ will often quit early
-    # if sources haven't yet been fetched. +bundle+ will recommend that you
-    # execute +bundle install+ to fetch those sources; however +bundle install+
-    # may fail if it needs to build native gems (which is +bundlerEnv+'s job).
+    # The {Bundler} library does complicated things with {ENV} and frequently
+    # modifies singleton data. Bundix uses Bundler for two purposes:
+    #
+    #   1. To handle the user's +Gemfile+ and +Gemfile.lock+.
+    #   2. To load its own rubygem dependencies, like a typical ruby project.
+    #
+    # When handling user files, it's possible that they may break the instance
+    # of {Bundler} which Bundix is using for its own purposes. To avoid this, we
+    # spawn a child process to act as a sandbox. Inside the child process,
+    # {Bundler} can modify what it likes without affecting the parent process.
     class Base
-      attr_reader :cache_dir
+      attr_reader :cache_dir, :pipe_result
 
       # @param cache_dir [#to_s] The local directory where Bundix caches gems
       #   and other fetched sources.
-      def initialize(cache_dir: CACHE_DIR)
+      def initialize(cache_dir: CACHE_DIR, pipe_result: false)
         @cache_dir = cache_dir
+        @pipe_result = pipe_result
       end
 
       def call
         # We fork here because Bundler may instance_exec some .gemspec files,
         # and those file may require other ruby files. We don't want them affect
         # the ruby process running this app.
-        Process.wait(fork { with_bundler_env { cli.run } })
+        pipe_result ? do_fork_with_result : do_fork
       end
 
       protected
 
-      # @return [#run] The {Bundler::CLI} service to execute.
-      def cli
+      def bundler_process
         raise NotImplementedError
       end
 
@@ -42,11 +48,34 @@ def env
 
       private
 
+      def do_fork_with_result
+        read, write = IO.pipe
+
+        pid = fork do
+          read.close
+          Marshal.dump(with_bundler_env { bundler_process }, write)
+        end
+
+        write.close
+        result = read.read
+        Process.wait(pid)
+        raise 'child failed' if result.empty? # TODO: custom exception class
+
+        Marshal.load(result) # rubocop:disable Security/MarshalLoad
+      end
+
+      def do_fork
+        Process.wait(fork { with_bundler_env { bundler_process } })
+        nil
+      end
+
       def with_bundler_env
-        System.temp_env(**env) do
-          Bundler.reset!
-          monkey_patch_definition(Bundler.definition)
-          yield
+        Bundler.with_unbundled_env do
+          System.temp_env(env) do
+            Bundler.reset!
+            monkey_patch_definition(Bundler.definition)
+            yield
+          end
         end
       end
 

lib/bundix/bundler_proxy/cache.rb

@@ -6,6 +6,12 @@
 module Bundix
   module BundlerProxy
     # Executes +bundle cache --all+.
+    #
+    # The reason for executing this via the {Bundler} ruby library, instead of
+    # the +bundle+ executable, is that +bundle+ may quit early if any git
+    # sources haven't yet been cloned. +bundle+ will then recommend that you
+    # execute +bundle install+ to fetch them; however +bundle install+ may fail
+    # if it needs to build native gems (which is +bundlerEnv+'s job).
     class Cache < Base
       attr_reader :all_sources, :gemfile, :path
 
@@ -19,13 +25,8 @@ def initialize(path, gemfile, all_sources: true, **kwargs)
 
       protected
 
-      def cli
-        Bundler::CLI::Cache
-          .new('all-platforms' => false, # TODO: get from CLI args
-               'cache-path' => cache_dir,
-               all: all_sources,
-               path: path,
-               gemfile: gemfile)
+      def bundler_process
+        cli.run
       end
 
       def env
@@ -34,6 +35,17 @@ def env
           'BUNDLE_GEMFILE' => gemfile
         }
       end
+
+      private
+
+      def cli
+        Bundler::CLI::Cache
+          .new('all-platforms' => false, # TODO: get from CLI args
+               'cache-path' => cache_dir,
+               all: all_sources,
+               path: path,
+               gemfile: gemfile)
+      end
     end
   end
 end

lib/bundix/bundler_proxy/clone_git.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'bundler/cli'
+require 'bundler/cli/lock'
+
+module Bundix
+  module BundlerProxy
+    # This service ensures a git source is cloned within {#cache_dir}.
+    class CloneGit < Base
+      attr_reader :ignore_config, :original_source
+
+      # @param original_source [Bundler::Source::Git]
+      def initialize(original_source, ignore_config: false, **kwargs)
+        super(pipe_result: true, **kwargs)
+
+        @original_source = original_source
+        @ignore_config = ignore_config
+      end
+
+      protected
+
+      def bundler_process
+        dup_source(original_source).tap do |source|
+          git_proxy(source).checkout unless source.cache_path.exist?
+        end
+      end
+
+      def env
+        {
+          # TODO: 'BUNDLE_FROZEN' => nil,
+          'BUNDLE_PATH' => cache_dir,
+          'BUNDLE_IGNORE_CONFIG' => (ignore_config ? 'true' : nil)
+        }
+      end
+
+      private
+
+      # We must dup the source inside #{with_bundler_env}, so its paths are
+      # inside {#cache_dir}.
+      def dup_source(source)
+        Bundler::Source::Git.new(source.options).tap do |new_source|
+          new_source.cache_path # memoize path inside {#cache_dir}
+          new_source.install_path # memoize path inside {#cache_dir}
+          new_source.remote! # allow repo to be cloned
+        end
+      end
+
+      def git_proxy(source)
+        @git_proxy ||= Bundler::Source::Git::GitProxy.new(
+          source.cache_path,
+          source.uri,
+          source.options,
+          source.options['revision'],
+          source
+        )
+      end
+    end
+  end
+end