Support for HEEx compilation (#323)

This PR adopts an approach that divides the compilation of HEEx into three steps. 
First, it focuses on the HTML portion, 
then the EEx part, where certain errors in EEx require evaluation to obtain, and finally, the compile_quoted step.

I think this layered approach is quite good, like peeling an onion.

Though there are some problems that I haven't thought of a good solution for now, such as

1. In a HEEx file, when a function component does not exist, `mix compile` will emit an error, but how can this be achieved for single file compilation? Jose suggested that we compile the relevant `ex` files directly, but it seems difficult because the latest content of those files may only exist in the `Document.Store` of the server node.
2. In a `.ex` file, like `core_components.ex`, how can the `~H` block be compiled separately?
3. The `~p` block in a `~H` block or `HEEx` file.

apps/remote_control/lib/lexical/remote_control/build/document.ex

@@ -3,7 +3,7 @@ defmodule Lexical.RemoteControl.Build.Document do
   alias Lexical.Document
   alias Lexical.RemoteControl.Build.Document.Compilers
 
-  @compilers [Compilers.Config, Compilers.Elixir, Compilers.EEx, Compilers.NoOp]
+  @compilers [Compilers.Config, Compilers.Elixir, Compilers.EEx, Compilers.HEEx, Compilers.NoOp]
 
   def compile(%Document{} = document) do
     compiler = Enum.find(@compilers, & &1.recognizes?(document))

apps/remote_control/lib/lexical/remote_control/build/document/compilers/eex.ex

@@ -4,6 +4,7 @@ defmodule Lexical.RemoteControl.Build.Document.Compilers.EEx do
   """
   alias Lexical.Document
   alias Lexical.Plugin.V1.Diagnostic.Result
+  alias Lexical.RemoteControl.Build
   alias Lexical.RemoteControl.Build.Document.Compiler
   alias Lexical.RemoteControl.Build.Document.Compilers
 
@@ -18,8 +19,15 @@ defmodule Lexical.RemoteControl.Build.Document.Compilers.EEx do
   end
 
   def compile(%Document{} = document) do
-    with {:ok, quoted} <- eex_to_quoted(document) do
-      Compilers.Quoted.compile(document, quoted, "EEx")
+    with {:ok, quoted} <- eex_to_quoted(document),
+         :ok <- eval_quoted(document, quoted) do
+      compile_quoted(document, quoted)
+    end
+  end
+
+  defp compile_quoted(%Document{} = document, quoted) do
+    with {:error, errors} <- Compilers.Quoted.compile(document, quoted, "EEx") do
+      {:error, reject_undefined_assigns(errors)}
     end
   end
 
@@ -37,6 +45,72 @@ defmodule Lexical.RemoteControl.Build.Document.Compilers.EEx do
     end
   end
 
+  defp eval_quoted(%Document{} = document, quoted_ast) do
+    result =
+      if Elixir.Features.with_diagnostics?() do
+        eval_quoted_with_diagnostics(quoted_ast, document.path)
+      else
+        do_eval_quoted(quoted_ast, document.path)
+      end
+
+    case result do
+      {:ok, _eval_result} ->
+        :ok
+
+      {{:ok, _eval_result}, _} ->
+        # Ignore warnings for now
+        # because they will be handled by `compile_quoted/2`
+        # like: `assign @thing not available in EEx template`
+        :ok
+
+      {:exception, exception, stack, _quoted_ast} ->
+        converted =
+          document
+          |> Build.Error.error_to_diagnostic(exception, stack, quoted_ast)
+          |> Map.put(:source, "EEx")
+
+        {:error, [converted]}
+
+      {{:exception, exception, stack, _quoted_ast}, all_errors_and_warnings} ->
+        converted = Build.Error.error_to_diagnostic(document, exception, stack, quoted_ast)
+        maybe_diagnostics = Build.Error.diagnostics_from_mix(document, all_errors_and_warnings)
+
+        diagnostics =
+          [converted | maybe_diagnostics]
+          |> Enum.reverse()
+          |> Build.Error.refine_diagnostics()
+          |> Enum.map(&Map.replace!(&1, :source, "EEx"))
+
+        {:error, diagnostics}
+    end
+  end
+
+  defp eval_quoted_with_diagnostics(quoted_ast, path) do
+    # Using apply to prevent a compile warning on elixir < 1.15
+    # credo:disable-for-next-line
+    apply(Code, :with_diagnostics, [fn -> do_eval_quoted(quoted_ast, path) end])
+  end
+
+  def do_eval_quoted(quoted_ast, path) do
+    try do
+      {result, _} = Code.eval_quoted(quoted_ast, [assigns: %{}], file: path)
+      {:ok, result}
+    rescue
+      exception ->
+        {filled_exception, stack} = Exception.blame(:error, exception, __STACKTRACE__)
+        {:exception, filled_exception, stack, quoted_ast}
+    end
+  end
+
+  defp reject_undefined_assigns(errors) do
+    # NOTE: Ignoring error for assigns makes sense,
+    # because we don't want such a error report,
+    # for example: `<%= @name %>`
+    Enum.reject(errors, fn %Result{message: message} ->
+      message =~ ~s[undefined variable "assigns"]
+    end)
+  end
+
   defp error_to_result(%Document{} = document, %EEx.SyntaxError{} = error) do
     position = {error.line, error.column}
 

apps/remote_control/lib/lexical/remote_control/build/document/compilers/heex.ex

@@ -0,0 +1,68 @@
+defmodule Lexical.RemoteControl.Build.Document.Compilers.HEEx do
+  @moduledoc """
+  A compiler for .heex files
+  """
+  alias Lexical.Document
+  alias Lexical.Plugin.V1.Diagnostic.Result
+  alias Lexical.RemoteControl.Build.Document.Compiler
+  alias Lexical.RemoteControl.Build.Document.Compilers
+  require Logger
+
+  @behaviour Compiler
+
+  def recognizes?(%Document{} = document) do
+    Path.extname(document.path) == ".heex"
+  end
+
+  def enabled? do
+    true
+  end
+
+  def compile(%Document{} = document) do
+    case heex_to_quoted(document) do
+      {:ok, _} ->
+        Compilers.EEx.compile(document)
+
+      other ->
+        other
+    end
+  end
+
+  defp heex_to_quoted(%Document{} = document) do
+    try do
+      source = Document.to_string(document)
+
+      opts =
+        [
+          source: source,
+          file: document.path,
+          caller: __ENV__,
+          engine: Phoenix.LiveView.TagEngine,
+          subengine: Phoenix.LiveView.Engine,
+          tag_handler: Phoenix.LiveView.HTMLEngine
+        ]
+
+      quoted = EEx.compile_string(source, opts)
+
+      {:ok, quoted}
+    rescue
+      error ->
+        {:error, [error_to_result(document, error)]}
+    end
+  end
+
+  defp error_to_result(%Document{} = document, %EEx.SyntaxError{} = error) do
+    position = {error.line, error.column}
+
+    Result.new(document.uri, position, error.message, :error, "EEx")
+  end
+
+  defp error_to_result(document, %error_struct{} = error)
+       when error_struct in [
+              TokenMissingError,
+              Phoenix.LiveView.Tokenizer.ParseError
+            ] do
+    position = {error.line, error.column}
+    Result.new(document.uri, position, error.description, :error, "HEEx")
+  end
+end

apps/remote_control/mix.exs

@@ -41,7 +41,8 @@ defmodule Lexical.RemoteControl.MixProject do
       {:lexical_test, path: "../../projects/lexical_test", only: :test},
       {:patch, "~> 0.12", only: [:dev, :test], optional: true, runtime: false},
       {:path_glob, "~> 0.2", optional: true},
-      {:sourceror, "~> 0.14.0"}
+      {:sourceror, "~> 0.14.0"},
+      {:phoenix_live_view, "~> 0.19.5", only: [:test], optional: true, runtime: false}
     ]
   end
 

apps/remote_control/test/lexical/remote_control/build/document/compilers/eex_test.exs

@@ -45,7 +45,7 @@ defmodule Lexical.RemoteControl.Build.Document.Compilers.EExTest do
     end
   end
 
-  describe "compile/1" do
+  describe "eex_to_quoted/1" do
     setup [:with_capture_server]
 
     test "handles syntax errors" do
@@ -65,6 +65,14 @@ defmodule Lexical.RemoteControl.Build.Document.Compilers.EExTest do
       assert result.source == "EEx"
       assert result.uri
     end
+  end
+
+  describe "compile_quoted/2" do
+    setup [:with_capture_server]
+
+    setup do
+      Code.compiler_options(parser_options: [columns: true, token_metadata: true])
+    end
 
     test "handles unused variables" do
       assert {:ok, [%Result{} = result]} =
@@ -75,10 +83,45 @@ defmodule Lexical.RemoteControl.Build.Document.Compilers.EExTest do
                |> compile()
 
       assert result.message =~ ~s["something" is unused]
-      assert result.position == 1
+      assert result.position in [1, {1, 5}]
       assert result.severity == :warning
       assert result.source == "EEx"
       assert result.uri =~ "file:///file.eex"
     end
   end
+
+  describe "eval_quoted/2" do
+    test "handles undefinied function" do
+      document = document_with_content(~q[
+        <%= IO.uts("thing") %>
+      ])
+
+      assert {:error, [%Result{} = result]} = compile(document)
+      assert result.message =~ "function IO.uts/1 is undefined or private"
+      assert result.position == {1, 8}
+      assert result.severity == :error
+      assert result.source == "EEx"
+      assert result.uri =~ "file:///file.eex"
+    end
+
+    @tag :with_diagnostics
+    test "handles undefinied variable" do
+      document = document_with_content(~q[
+        <%= thing %>
+      ])
+
+      assert {:error, [%Result{} = result]} = compile(document)
+
+      if Features.with_diagnostics?() do
+        assert result.message =~ "undefined variable \"thing\""
+      else
+        assert result.message =~ "undefined function thing/0"
+      end
+
+      assert result.position in [1, {1, 5}]
+      assert result.severity == :error
+      assert result.source == "EEx"
+      assert result.uri =~ "file:///file.eex"
+    end
+  end
 end

apps/remote_control/test/lexical/remote_control/build/document/compilers/heex_test.exs

@@ -0,0 +1,86 @@
+defmodule Lexical.RemoteControl.Build.Document.Compilers.HeexTest do
+  alias Lexical.Document
+  alias Lexical.Plugin.V1.Diagnostic.Result
+  alias Lexical.RemoteControl.Build.CaptureServer
+  alias Lexical.RemoteControl.Build.Document.Compilers
+  alias Lexical.RemoteControl.Dispatch
+  alias Lexical.RemoteControl.ModuleMappings
+
+  import Lexical.Test.CodeSigil
+  import Compilers.HEEx, only: [compile: 1]
+
+  use ExUnit.Case
+
+  def with_capture_server(_) do
+    start_supervised!(CaptureServer)
+    start_supervised!(Dispatch)
+    start_supervised!(ModuleMappings)
+    :ok
+  end
+
+  def document_with_content(content) do
+    Document.new("file:///file.heex", content, 0)
+  end
+
+  setup do
+    Code.compiler_options(parser_options: [columns: true, token_metadata: true])
+  end
+
+  describe "compile/1" do
+    setup [:with_capture_server]
+
+    test "handles valid HEEx content" do
+      document = document_with_content(~q[
+        <div>thing</div>
+      ])
+      assert {:ok, []} = compile(document)
+    end
+
+    test "ignore undefinied assigns" do
+      document = document_with_content(~q[
+        <div><%= @thing %></div>
+      ])
+
+      assert {:error, []} = compile(document)
+    end
+
+    test "returns error when there are unclosed tags" do
+      document = document_with_content(~q[
+        <div>thing
+      ])
+      assert {:error, [%Result{} = result]} = compile(document)
+
+      assert result.message =~
+               "end of template reached without closing tag for <div>\n  |\n1 | <div>thing\n  | ^"
+
+      assert result.position == {1, 1}
+      assert result.severity == :error
+      assert result.source == "HEEx"
+      assert result.uri =~ "file:///file.heex"
+    end
+
+    test "returns error when HEEx syntax is invalid" do
+      document = document_with_content(~q[
+        <span id=@id}></span>
+      ])
+
+      assert {:error, [%Result{} = result]} = compile(document)
+
+      assert result.message =~ "invalid attribute value after `=`. "
+      assert result.position == {1, 10}
+      assert result.severity == :error
+      assert result.source == "HEEx"
+      assert result.uri =~ "file:///file.heex"
+    end
+
+    test "handles EEx syntax error" do
+      document = document_with_content(~q[
+        <%= IO.
+      ])
+      assert {:error, [%Result{} = result]} = compile(document)
+
+      assert result.message =~ "'%>'"
+      assert result.source == "EEx"
+    end
+  end
+end

mix.lock

@@ -1,5 +1,6 @@
 %{
   "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"},
+  "castore": {:hex, :castore, "1.0.3", "7130ba6d24c8424014194676d608cb989f62ef8039efd50ff4b3f33286d06db8", [:mix], [], "hexpm", "680ab01ef5d15b161ed6a95449fac5c6b8f60055677a8e79acf01b27baa4390b"},
   "credo": {:hex, :credo, "1.7.0", "6119bee47272e85995598ee04f2ebbed3e947678dee048d10b5feca139435f75", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "6839fcf63d1f0d1c0f450abc8564a57c43d644077ab96f2934563e68b8a769d7"},
   "dialyxir": {:hex, :dialyxir, "1.3.0", "fd1672f0922b7648ff9ce7b1b26fcf0ef56dda964a459892ad15f6b4410b5284", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "00b2a4bcd6aa8db9dcb0b38c1225b7277dca9bc370b6438715667071a304696f"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.32", "fa739a0ecfa34493de19426681b23f6814573faee95dfd4b4aafe15a7b5b32c6", [:mix], [], "hexpm", "b8b0dd77d60373e77a3d7e8afa598f325e49e8663a51bcc2b88ef41838cca755"},
@@ -12,9 +13,20 @@
   "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"},
   "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"},
+  "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"},
   "patch": {:hex, :patch, "0.12.0", "2da8967d382bade20344a3e89d618bfba563b12d4ac93955468e830777f816b0", [:mix], [], "hexpm", "ffd0e9a7f2ad5054f37af84067ee88b1ad337308a1cb227e181e3967127b0235"},
   "path_glob": {:hex, :path_glob, "0.2.0", "b9e34b5045cac5ecb76ef1aa55281a52bf603bf7009002085de40958064ca312", [:mix], [{:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "be2594cb4553169a1a189f95193d910115f64f15f0d689454bb4e8cfae2e7ebc"},
+  "phoenix": {:hex, :phoenix, "1.7.7", "4cc501d4d823015007ba3cdd9c41ecaaf2ffb619d6fb283199fa8ddba89191e0", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "8966e15c395e5e37591b6ed0bd2ae7f48e961f0f60ac4c733f9566b519453085"},
+  "phoenix_html": {:hex, :phoenix_html, "3.3.2", "d6ce982c6d8247d2fc0defe625255c721fb8d5f1942c5ac051f6177bffa5973f", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "44adaf8e667c1c20fb9d284b6b0fa8dc7946ce29e81ce621860aa7e96de9a11d"},
+  "phoenix_live_view": {:hex, :phoenix_live_view, "0.19.5", "6e730595e8e9b8c5da230a814e557768828fd8dfeeb90377d2d8dbb52d4ec00a", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b2eaa0dd3cfb9bd7fb949b88217df9f25aed915e986a28ad5c8a0d054e7ca9d3"},
+  "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"},
+  "phoenix_template": {:hex, :phoenix_template, "1.0.3", "32de561eefcefa951aead30a1f94f1b5f0379bc9e340bb5c667f65f1edfa4326", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "16f4b6588a4152f3cc057b9d0c0ba7e82ee23afa65543da535313ad8d25d8e2c"},
+  "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"},
+  "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"},
   "sourceror": {:hex, :sourceror, "0.14.0", "b6b8552d0240400d66b6f107c1bab7ac1726e998efc797f178b7b517e928e314", [:mix], [], "hexpm", "809c71270ad48092d40bbe251a133e49ae229433ce103f762a2373b7a10a8d8b"},
   "stream_data": {:hex, :stream_data, "0.6.0", "e87a9a79d7ec23d10ff83eb025141ef4915eeb09d4491f79e52f2562b73e5f47", [:mix], [], "hexpm", "b92b5031b650ca480ced047578f1d57ea6dd563f5b57464ad274718c9c29501c"},
+  "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
+  "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},
+  "websock_adapter": {:hex, :websock_adapter, "0.5.4", "7af8408e7ed9d56578539594d1ee7d8461e2dd5c3f57b0f2a5352d610ddde757", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d2c238c79c52cbe223fcdae22ca0bb5007a735b9e933870e241fce66afb4f4ab"},
 }