Fix escript archive layout and support priv (#13730)

Author: jonatanklosko

lib/mix/lib/mix/tasks/escript.build.ex

@@ -36,11 +36,6 @@ defmodule Mix.Tasks.Escript.Build do
   the compiled `.beam` files to reduce the size of the escript.
   If this is not desired, check the `:strip_beams` option.
 
-  > #### `priv` directory support {: .warning}
-  >
-  > escripts do not support projects and dependencies
-  > that need to store or read artifacts from the priv directory.
-
   ## Command line options
 
   Expects the same command line options as `mix compile`.
@@ -94,6 +89,11 @@ defmodule Mix.Tasks.Escript.Build do
     * `:emu_args` - emulator arguments to embed in the escript file.
       Defaults to `""`.
 
+    * `:include_priv_for` - a list of application names (atoms) specifying
+      applications which priv directory should be included in the resulting
+      escript archive. Currently the expected way of accessing priv files
+      in an escript is via `:escript.extract/2`. Defaults to `[]`.
+
   There is one project-level option that affects how the escript is generated:
 
     * `language: :elixir | :erlang` - set it to `:erlang` for Erlang projects
@@ -185,11 +185,16 @@ defmodule Mix.Tasks.Escript.Build do
 
     escript_mod = String.to_atom(Atom.to_string(app) <> "_escript")
 
+    include_priv_for = MapSet.new(escript_opts[:include_priv_for] || [])
+
     beam_paths =
-      [project_files(), deps_files(), core_files(escript_opts, language)]
+      [
+        project_files(project, include_priv_for),
+        deps_files(include_priv_for),
+        core_files(escript_opts, language, include_priv_for)
+      ]
       |> Stream.concat()
-      |> prepare_beam_paths()
-      |> Map.merge(consolidated_paths(project))
+      |> replace_consolidated_paths(project)
 
     tuples = gen_main(project, escript_mod, main, app, language) ++ read_beams(beam_paths)
     tuples = if strip_options, do: strip_beams(tuples, strip_options), else: tuples
@@ -213,28 +218,40 @@ defmodule Mix.Tasks.Escript.Build do
     :ok
   end
 
-  defp project_files() do
-    get_files(Mix.Project.app_path())
+  defp project_files(project, include_priv_for) do
+    get_files(Mix.Project.app_path(), project[:app] in include_priv_for)
   end
 
-  defp get_files(app) do
-    Path.wildcard("#{app}/ebin/*.{app,beam}") ++
-      (Path.wildcard("#{app}/priv/**/*") |> Enum.filter(&File.regular?/1))
+  defp get_files(app_path, include_priv?) do
+    paths = Path.wildcard("#{app_path}/ebin/*.{app,beam}")
+
+    paths =
+      if include_priv? do
+        paths ++ (Path.wildcard("#{app_path}/priv/**/*") |> Enum.filter(&File.regular?/1))
+      else
+        paths
+      end
+
+    apps_dir = Path.dirname(app_path)
+
+    for path <- paths do
+      {Path.relative_to(path, apps_dir), path}
+    end
   end
 
   defp set_perms(filename) do
     stat = File.stat!(filename)
     :ok = File.chmod(filename, stat.mode ||| 0o111)
   end
 
-  defp deps_files() do
+  defp deps_files(include_priv_for) do
     deps = Mix.Dep.cached()
-    Enum.flat_map(deps, fn dep -> get_files(dep.opts[:build]) end)
+    Enum.flat_map(deps, fn dep -> get_files(dep.opts[:build], dep.app in include_priv_for) end)
   end
 
-  defp core_files(escript_opts, language) do
+  defp core_files(escript_opts, language, include_priv_for) do
     if Keyword.get(escript_opts, :embed_elixir, language == :elixir) do
-      Enum.flat_map([:elixir | extra_apps()], &app_files/1)
+      Enum.flat_map([:elixir | extra_apps()], &app_files(&1, include_priv_for))
     else
       []
     end
@@ -269,17 +286,13 @@ defmodule Mix.Tasks.Escript.Build do
     end
   end
 
-  defp app_files(app) do
+  defp app_files(app, include_priv_for) do
     case :code.where_is_file(~c"#{app}.app") do
       :non_existing -> Mix.raise("Could not find application #{app}")
-      file -> get_files(Path.dirname(Path.dirname(file)))
+      file -> get_files(Path.dirname(Path.dirname(file)), app in include_priv_for)
     end
   end
 
-  defp prepare_beam_paths(paths) do
-    for path <- paths, into: %{}, do: {Path.basename(path), path}
-  end
-
   defp read_beams(items) do
     Enum.map(items, fn {basename, beam_path} ->
       {String.to_charlist(basename), File.read!(beam_path)}
@@ -305,14 +318,31 @@ defmodule Mix.Tasks.Escript.Build do
     end
   end
 
-  defp consolidated_paths(config) do
+  defp replace_consolidated_paths(files, config) do
+    # We could write modules to a consolidated/ directory and prepend
+    # it to code path using VM args. However, when Erlang Escript
+    # boots, it prepends all second-level ebin/ directories to the
+    # path, so the unconsolidated modules would take precedence.
+    #
+    # Instead of writing consolidated/ into the archive, we replace
+    # the protocol modules with their consolidated version in their
+    # usual location. As a side benefit, this reduces the Escript
+    # file size, since we do not include the unconsolidated modules.
+
     if config[:consolidate_protocols] do
-      Mix.Project.consolidation_path(config)
-      |> Path.join("*")
-      |> Path.wildcard()
-      |> prepare_beam_paths()
+      consolidation_path = Mix.Project.consolidation_path(config)
+
+      consolidated =
+        consolidation_path
+        |> Path.join("*")
+        |> Path.wildcard()
+        |> Map.new(fn path -> {Path.basename(path), path} end)
+
+      for {zip_path, path} <- files do
+        {zip_path, consolidated[Path.basename(path)] || path}
+      end
     else
-      %{}
+      []
     end
   end
 

lib/mix/test/fixtures/escript_test/lib/escript_test.ex

@@ -11,6 +11,25 @@ defmodule EscriptTest do
     IO.inspect(Application.get_env(:foobar, :nesting, "TEST"))
   end
 
+  def main(["--app-paths"]) do
+    elixir_path = Application.app_dir(:elixir) |> String.to_charlist()
+    escript_test_path = Application.app_dir(:escript_test) |> String.to_charlist()
+
+    IO.inspect({
+      elixir_path != escript_test_path,
+      match?({:ok, _}, :erl_prim_loader.list_dir(elixir_path)),
+      match?({:ok, _}, :erl_prim_loader.list_dir(escript_test_path))
+    })
+  end
+
+  def main(["--list-priv", app_name]) do
+    # Note: :erl_prim_loader usage with Escript is currently deprecated,
+    # but we use it only in tests for convenience
+
+    app = String.to_atom(app_name)
+    :erl_prim_loader.list_dir(~c"#{:code.lib_dir(app)}/priv") |> IO.inspect()
+  end
+
   def main(_argv) do
     IO.puts(Application.get_env(:foobar, :value, "TEST"))
   end

lib/mix/test/mix/tasks/escript_test.exs

@@ -96,6 +96,17 @@ defmodule Mix.Tasks.EscriptTest do
     end
   end
 
+  defmodule EscriptWithPrivs do
+    def project do
+      [
+        app: :escript_test_with_priv,
+        version: "0.0.1",
+        escript: [main_module: EscriptTest, include_priv_for: [:escript_test_with_priv, :ok]],
+        deps: [{:ok, path: fixture_path("deps_status/deps/ok")}]
+      ]
+    end
+  end
+
   test "generate escript" do
     in_fixture("escript_test", fn ->
       Mix.Project.push(Escript)
@@ -104,6 +115,13 @@ defmodule Mix.Tasks.EscriptTest do
       assert_received {:mix_shell, :info, ["Generated escript escript_test with MIX_ENV=dev"]}
       assert System.cmd("escript", ["escript_test"]) == {"TEST\n", 0}
       assert count_abstract_code("escript_test") == 0
+
+      # Each app has a distinct, valid path
+      assert System.cmd("escript", ["escript_test", "--app-paths"]) == {"{true, true, true}\n", 0}
+
+      # Does not include priv by default
+      assert System.cmd("escript", ["escript_test", "--list-priv", "escript_test"]) ==
+               {":error\n", 0}
     end)
   end
 
@@ -229,6 +247,10 @@ defmodule Mix.Tasks.EscriptTest do
       assert_received {:mix_shell, :info, [^message]}
 
       assert System.cmd("escript", ["escript_test_with_deps"]) == {"TEST\n", 0}
+
+      # Does not include priv for deps by default
+      assert System.cmd("escript", ["escript_test_with_deps", "--list-priv", "ok"]) ==
+               {":error\n", 0}
     end)
   after
     purge([Ok.MixProject])
@@ -280,6 +302,26 @@ defmodule Mix.Tasks.EscriptTest do
     end)
   end
 
+  test "generate escript with priv" do
+    in_fixture("escript_test", fn ->
+      Mix.Project.push(EscriptWithPrivs)
+
+      Mix.Tasks.Escript.Build.run([])
+
+      message = "Generated escript escript_test_with_priv with MIX_ENV=dev"
+      assert_received {:mix_shell, :info, [^message]}
+
+      assert System.cmd("escript", [
+               "escript_test_with_priv",
+               "--list-priv",
+               "escript_test_with_priv"
+             ]) == {~s/{:ok, [~c"hello"]}\n/, 0}
+
+      assert System.cmd("escript", ["escript_test_with_priv", "--list-priv", "ok"]) ==
+               {~s/{:ok, [~c"sample"]}\n/, 0}
+    end)
+  end
+
   test "escript install and uninstall" do
     File.rm_rf!(tmp_path(".mix/escripts"))