Reworked lexical packaging

This commit moves away from releases for our packaging. What replaces
them is a package directory that has a launcher, working config and
priv directories and a bunch of erlang archives for storing our beam
files.

New packaging enables the following:

  * We can compile on one version of erlang and run on another
  * We get a real priv directory experience, which isn't possible with
    escripts.
  * We get a "real" config directory
  * We get a boot script that's just a plain old elixir file
  * All of our files can be ported to windows fairly easily
  * No tricks for packaging scripts / config. They're just plain
    files, and we rely on the `:code` module to load them

I've tested this compiling in one version of elixir and erlang and
running it in another, and it seems to work great. It makes sense
though, they're just BEAM files, which should be cross platform.

Other approaches:

We all know why releases won't / don't work. I also tried building an
escript, and it kind of worked, but had the following drawbacks:

  * There is no priv directory. I was able to code around this by
    having a module that generates the port_mapper.sh file for you,
    but this was a big cumbersome.
  * An escript is a binary file, and erlang loads all of its modules
    from it. This was a problem for the second VM, as we couldn't just
    point it at the binary file and have it look for modules there. I
    also tried using the 'inet' loader, but that had a bunch of
    extremely difficult to debug issues. Worse, I would still need
    some erlang archives, so why not just make it universal?

I also tried using `mix archive.build`, but it didn't support
namespacing (obviously) and changing the task to do so would pretty
much lead us to the path we have here.

Fixes #255

apps/remote_control/lib/lexical/remote_control/project_node.ex

@@ -1,6 +1,7 @@
 defmodule Lexical.RemoteControl.ProjectNode do
   alias Lexical.Project
   alias Lexical.RemoteControl
+  alias Lexical.RemoteControl.ProjectNode
   require Logger
 
   defmodule State do
@@ -27,7 +28,7 @@ defmodule Lexical.RemoteControl.ProjectNode do
     @dialyzer {:nowarn_function, start: 3}
 
     def start(%__MODULE__{} = state, paths, from) do
-      port_wrapper = port_wrapper_executable()
+      port_wrapper = ProjectNode.Launcher.path()
 
       {:ok, elixir_executable, environment_variables} =
         RemoteControl.elixir_executable(state.project)
@@ -91,12 +92,6 @@ defmodule Lexical.RemoteControl.ProjectNode do
       end)
     end
 
-    defp port_wrapper_executable do
-      :remote_control
-      |> :code.priv_dir()
-      |> Path.join("port_wrapper.sh")
-    end
-
     defp project_rpc(%__MODULE__{} = state, module, function, args \\ []) do
       state.project
       |> Project.node_name()

apps/remote_control/lib/lexical/remote_control/project_node/launcher.ex

@@ -0,0 +1,19 @@
+defmodule Lexical.RemoteControl.ProjectNode.Launcher do
+  @moduledoc """
+  A module that provides the path of an executable to launch another
+  erlang node via ports.
+  """
+  def path do
+    path(:os.type())
+  end
+
+  def path({:unix, _}) do
+    with :non_existing <- :code.where_is_file(~c"port_wrapper.sh") do
+      :remote_control
+      |> :code.priv_dir()
+      |> Path.join("port_wrapper.sh")
+      |> Path.expand()
+    end
+    |> to_string()
+  end
+end

apps/remote_control/lib/mix/tasks/namespace/path.ex

@@ -24,7 +24,11 @@ defmodule Mix.Tasks.Namespace.Path do
         |> Namespace.Module.apply()
         |> Atom.to_string()
 
-      String.replace(path, string_name, namespaced_name)
+      if String.contains?(path, namespaced_name) do
+        path
+      else
+        String.replace(path, string_name, namespaced_name)
+      end
     end)
   end
 end

apps/remote_control/priv/port_wrapper.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 
 set_up_version_manager() {
-    if [ -e $HOME/.asdf && ! asdf which erl ]; then
+    if [ -e $HOME/.asdf ] && asdf which erl -eq 0 ; then
         VERSION_MANAGER="asdf"
-    elif [ -e $HOME/.rtx && ! rtx which erl ]; then
+    elif [ -e $HOME/.rtx ] &&  rtx which erl -eq 0; then
         VERSION_MANAGER="rtx"
     else
         VERSION_MANAGER="none"

apps/server/lib/lexical/server/boot.ex

@@ -0,0 +1,58 @@
+defmodule Lexical.Server.Boot do
+  @moduledoc """
+  This module is called when the server starts by the start script.
+
+  Packaging will ensure that config.exs and runtime.exs will be visible to the `:code` module
+  """
+  @env Mix.env()
+  @target Mix.target()
+  @dep_apps Enum.map(Mix.Dep.cached(), & &1.app)
+
+  def start do
+    {:ok, _} = Application.ensure_all_started(:mix)
+    Application.stop(:logger)
+    load_config()
+    Enum.each(@dep_apps, &load_app_modules/1)
+    Application.start(:logger)
+  end
+
+  defp load_config do
+    config = read_config("config.exs")
+    runtime = read_config("runtime.exs")
+    merged_config = Config.Reader.merge(config, runtime)
+    apply_config(merged_config)
+  end
+
+  defp apply_config(configs) do
+    for {app_name, keywords} <- configs,
+        {config_key, config_value} <- keywords do
+      Application.put_env(app_name, config_key, config_value)
+    end
+  end
+
+  defp read_config(file_name) do
+    case where_is_file(String.to_charlist(file_name)) do
+      {:ok, path} ->
+        Config.Reader.read!(path, env: @env, target: @target)
+
+      _ ->
+        []
+    end
+  end
+
+  defp where_is_file(file_name) do
+    case :code.where_is_file(file_name) do
+      :non_existing ->
+        :error
+
+      path ->
+        {:ok, List.to_string(path)}
+    end
+  end
+
+  defp load_app_modules(app_name) do
+    with {:ok, modules} <- :application.get_key(app_name, :modules) do
+      Enum.each(modules, &Code.ensure_loaded!/1)
+    end
+  end
+end

apps/server/lib/mix/tasks/package.ex

@@ -0,0 +1,201 @@
+defmodule Mix.Tasks.Package do
+  alias Mix.Tasks.Namespace
+
+  @options [strict: [path: :string]]
+
+  def run(args) do
+    Mix.Task.run(:compile)
+    {opts, _, _} = OptionParser.parse(args, @options)
+
+    package_root =
+      Keyword.get(opts, :path, Path.join([Mix.Project.build_path(), "package", "lexical"]))
+
+    Mix.Shell.IO.info("Assembling buld in #{package_root}")
+    File.mkdir_p!(package_root)
+
+    {:ok, scratch_directory} = prepare(package_root)
+    Mix.Task.run(:namespace, [scratch_directory])
+    build_archives(package_root, scratch_directory)
+    copy_consolidated_beams(package_root)
+    copy_launchers(package_root)
+    copy_priv_files(package_root)
+    copy_config(package_root)
+    File.rm_rf!(scratch_directory)
+  end
+
+  defp prepare(package_root) do
+    scratch_directory = Path.join(package_root, "scratch")
+    File.mkdir(scratch_directory)
+
+    [Mix.Project.build_path(), "lib"]
+    |> Path.join()
+    |> File.cp_r!(Path.join(scratch_directory, "lib"))
+
+    {:ok, scratch_directory}
+  end
+
+  defp build_archives(package_root, scratch_directory) do
+    scratch_directory
+    |> target_path()
+    |> File.mkdir_p!()
+
+    app_dirs = app_dirs(scratch_directory)
+
+    Enum.each(app_dirs, fn {app_name, path} ->
+      create_archive(package_root, app_name, path)
+    end)
+  end
+
+  defp app_dirs(scratch_directory) do
+    lib_directory = Path.join(scratch_directory, "lib")
+    server_deps = server_deps()
+
+    lib_directory
+    |> File.ls!()
+    |> Enum.filter(&(&1 in server_deps))
+    |> Map.new(fn dir ->
+      app_name = Path.basename(dir)
+      {app_name, Path.join([scratch_directory, "lib", dir])}
+    end)
+  end
+
+  defp create_archive(package_root, app_name, app_path) do
+    file_list = file_list(app_name, app_path)
+    zip_path = Path.join([target_path(package_root), "#{app_name}.ez"])
+
+    {:ok, _} = :zip.create(String.to_charlist(zip_path), file_list)
+    :ok
+  end
+
+  defp file_list(app_name, app_path) do
+    File.cd!(app_path, fn ->
+      beams = Path.wildcard("ebin/*.{app,beam}")
+      priv = Path.wildcard("priv/**/*", match_dot: true)
+
+      Enum.reduce(beams ++ priv, [], fn relative_path, acc ->
+        case File.read(relative_path) do
+          {:ok, contents} ->
+            zip_relative_path =
+              app_name
+              |> Path.join(relative_path)
+              |> String.to_charlist()
+
+            [{zip_relative_path, contents} | acc]
+
+          {:error, _} ->
+            acc
+        end
+      end)
+    end)
+  end
+
+  defp copy_consolidated_beams(package_root) do
+    beams_dest_dir = Path.join(package_root, "consolidated")
+
+    File.mkdir_p!(beams_dest_dir)
+
+    File.cp_r!(Mix.Project.consolidation_path(), beams_dest_dir)
+
+    # The following is required because the consolidation
+    # path is a symlink, and File.cp_r! doesn't treat symlinked
+    # directories like directories, and only copies the symlink itself.
+    beams_dest_dir
+    |> File.ls!()
+    |> Enum.each(fn relative_path ->
+      absolute_path = Path.join(beams_dest_dir, relative_path)
+      Namespace.Transform.Beams.apply(absolute_path)
+    end)
+  end
+
+  defp copy_launchers(package_root) do
+    launcher_source_dir =
+      Mix.Project.project_file()
+      |> Path.dirname()
+      |> Path.join("bin")
+
+    launcher_dest_dir = Path.join(package_root, "bin")
+
+    File.mkdir_p!(launcher_dest_dir)
+    File.cp_r!(launcher_source_dir, launcher_dest_dir)
+  end
+
+  defp target_path(scratch_directory) do
+    Path.join([scratch_directory, "lib"])
+  end
+
+  defp server_deps do
+    server_path = Mix.Project.deps_paths()[:server]
+
+    deps =
+      Mix.Project.in_project(:server, server_path, fn _ ->
+        Enum.map(Mix.Project.deps_apps(), fn app_module ->
+          app_module
+          |> Namespace.Module.apply()
+          |> to_string()
+        end)
+      end)
+
+    server_dep =
+      :server
+      |> Namespace.Module.apply()
+      |> to_string()
+
+    [server_dep | deps]
+  end
+
+  defp copy_config(package_root) do
+    config_source =
+      Mix.Project.config()[:config_path]
+      |> Path.absname()
+      |> Path.dirname()
+
+    config_dest = Path.join(package_root, "config")
+    File.mkdir_p!(config_dest)
+    File.cp_r!(config_source, config_dest)
+  end
+
+  @priv_apps [:remote_control]
+
+  defp copy_priv_files(package_root) do
+    priv_dest_dir = Path.join(package_root, "priv")
+
+    Enum.each(@priv_apps, fn app_name ->
+      case priv_dir(app_name) do
+        {:ok, priv_source_dir} ->
+          File.cp_r!(priv_source_dir, priv_dest_dir)
+
+        _ ->
+          :ok
+      end
+    end)
+  end
+
+  defp priv_dir(app) do
+    case :code.priv_dir(app) do
+      {:error, _} ->
+        :error
+
+      path ->
+        normalized =
+          path
+          |> List.to_string()
+          |> normalize_path()
+
+        {:ok, normalized}
+    end
+  end
+
+  defp normalize_path(path) do
+    case File.read_link(path) do
+      {:ok, orig} ->
+        path
+        |> Path.dirname()
+        |> Path.join(orig)
+        |> Path.expand()
+        |> Path.absname()
+
+      _ ->
+        path
+    end
+  end
+end

bin/lexical.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set_up_version_manager() {
+    if [ -e $HOME/.asdf ] &&  asdf which elixir -eq 0; then
+        VERSION_MANAGER="asdf"
+    elif [ -e $HOME/.rtx ] && rtx which elixir -eq 0; then
+        VERSION_MANAGER="rtx"
+    else
+        VERSION_MANAGER="none"
+    fi
+}
+
+
+set_up_version_manager
+
+# Start the program in the background
+case "$VERSION_MANAGER" in
+    asdf)
+        asdf env erl exec "$@" &
+        ;;
+    rtx)
+        rtx env -s bash erl exec "$@" &
+        ;;
+    *)
+        exec "$@" &
+        ;;
+esac
+
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+export ERL_LIBS="${SCRIPT_DIR}/../lib"
+elixir -pa "${SCRIPT_DIR}/../consolidated" \
+       -pa "${SCRIPT_DIR}/../config/" \
+       -pa "${SCRIPT_DIR}/../priv/" \
+       --app lx_server \
+       --eval "LXical.Server.Boot.start" \
+       --no-halt