Add make_precompiler_downloader option

bjyoungblood · bjyoungblood · commit b1ba146bd2d9 · 2024-05-07T18:24:20.000-07:00
Allows users to customize download behavior, such as adding HTTP
authentication or using an alternate protocol like SFTP.
diff --git a/lib/elixir_make/artefact.ex b/lib/elixir_make/artefact.ex
@@ -286,141 +286,8 @@ defmodule ElixirMake.Artefact do
     end
   end
 
-  ## Download
-
-  def download(url) do
-    url_charlist = String.to_charlist(url)
-
-    # TODO: Remove me when we require Elixir v1.15
-    {:ok, _} = Application.ensure_all_started(:inets)
-    {:ok, _} = Application.ensure_all_started(:ssl)
-    {:ok, _} = Application.ensure_all_started(:public_key)
-
-    if proxy = System.get_env("HTTP_PROXY") || System.get_env("http_proxy") do
-      Mix.shell().info("Using HTTP_PROXY: #{proxy}")
-      %{host: host, port: port} = URI.parse(proxy)
-
-      :httpc.set_options([{:proxy, {{String.to_charlist(host), port}, []}}])
-    end
-
-    if proxy = System.get_env("HTTPS_PROXY") || System.get_env("https_proxy") do
-      Mix.shell().info("Using HTTPS_PROXY: #{proxy}")
-      %{host: host, port: port} = URI.parse(proxy)
-      :httpc.set_options([{:https_proxy, {{String.to_charlist(host), port}, []}}])
-    end
-
-    # https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/inets
-    # TODO: This may no longer be necessary from Erlang/OTP 25.0 or later.
-    https_options = [
-      ssl:
-        [
-          verify: :verify_peer,
-          customize_hostname_check: [
-            match_fun: :public_key.pkix_verify_hostname_match_fun(:https)
-          ]
-        ] ++ cacerts_options()
-    ]
-
-    options = [body_format: :binary]
-
-    case :httpc.request(:get, {url_charlist, []}, https_options, options) do
-      {:ok, {{_, 200, _}, _headers, body}} ->
-        {:ok, body}
-
-      other ->
-        {:error, "couldn't fetch NIF from #{url}: #{inspect(other)}"}
-    end
-  end
-
-  defp cacerts_options do
-    cond do
-      path = System.get_env("ELIXIR_MAKE_CACERT") ->
-        [cacertfile: path]
-
-      certs = otp_cacerts() ->
-        [cacerts: certs]
-
-      Application.spec(:castore, :vsn) ->
-        [cacertfile: Application.app_dir(:castore, "priv/cacerts.pem")]
-
-      Application.spec(:certifi, :vsn) ->
-        [cacertfile: Application.app_dir(:certifi, "priv/cacerts.pem")]
-
-      path = cacerts_from_os() ->
-        [cacertfile: path]
-
-      true ->
-        warn_no_cacerts()
-        []
-    end
-  end
-
-  defp otp_cacerts do
-    if System.otp_release() >= "25" do
-      # cacerts_get/0 raises if no certs found
-      try do
-        :public_key.cacerts_get()
-      rescue
-        _ ->
-          nil
-      end
-    end
-  end
-
-  # https_opts and related code are taken from
-  # https://github.com/elixir-cldr/cldr_utils/blob/v2.19.1/lib/cldr/http/http.ex
-  @certificate_locations [
-    # Debian/Ubuntu/Gentoo etc.
-    "/etc/ssl/certs/ca-certificates.crt",
-
-    # Fedora/RHEL 6
-    "/etc/pki/tls/certs/ca-bundle.crt",
-
-    # OpenSUSE
-    "/etc/ssl/ca-bundle.pem",
-
-    # OpenELEC
-    "/etc/pki/tls/cacert.pem",
-
-    # CentOS/RHEL 7
-    "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
-
-    # Open SSL on MacOS
-    "/usr/local/etc/openssl/cert.pem",
-
-    # MacOS & Alpine Linux
-    "/etc/ssl/cert.pem"
-  ]
-
-  defp cacerts_from_os do
-    Enum.find(@certificate_locations, &File.exists?/1)
-  end
-
-  defp warn_no_cacerts do
-    Mix.shell().error("""
-    No certificate trust store was found.
-
-    Tried looking for: #{inspect(@certificate_locations)}
-
-    A certificate trust store is required in
-    order to download locales for your configuration.
-    Since elixir_make could not detect a system
-    installed certificate trust store one of the
-    following actions may be taken:
-
-    1. Install the hex package `castore`. It will
-       be automatically detected after recompilation.
-
-    2. Install the hex package `certifi`. It will
-       be automatically detected after recompilation.
-
-    3. Specify the location of a certificate trust store
-       by configuring it in environment variable:
-
-         export ELIXIR_MAKE_CACERT="/path/to/cacerts.pem"
-
-    4. Use OTP 25+ on an OS that has built-in certificate
-       trust store.
-    """)
+  def download(config, url) do
+    downloader = config[:make_precompile_downloader] || ElixirMake.Downloader.Httpc
+    downloader.download(url)
   end
 end
diff --git a/lib/elixir_make/downloader.ex b/lib/elixir_make/downloader.ex
@@ -0,0 +1,10 @@
+defmodule ElixirMake.Downloader do
+  @moduledoc """
+  The behaviour for downloader modules.
+  """
+
+  @doc """
+  This callback should download the artefact from the given URL.
+  """
+  @callback download(url :: String.t()) :: {:ok, iolist() | binary()} | {:error, String.t()}
+end
diff --git a/lib/elixir_make/downloader/httpc.ex b/lib/elixir_make/downloader/httpc.ex
@@ -0,0 +1,142 @@
+defmodule ElixirMake.Downloader.Httpc do
+  @moduledoc false
+
+  @behaviour ElixirMake.Downloader
+
+  @impl ElixirMake.Downloader
+  def download(url) do
+    url_charlist = String.to_charlist(url)
+
+    # TODO: Remove me when we require Elixir v1.15
+    {:ok, _} = Application.ensure_all_started(:inets)
+    {:ok, _} = Application.ensure_all_started(:ssl)
+    {:ok, _} = Application.ensure_all_started(:public_key)
+
+    if proxy = System.get_env("HTTP_PROXY") || System.get_env("http_proxy") do
+      Mix.shell().info("Using HTTP_PROXY: #{proxy}")
+      %{host: host, port: port} = URI.parse(proxy)
+
+      :httpc.set_options([{:proxy, {{String.to_charlist(host), port}, []}}])
+    end
+
+    if proxy = System.get_env("HTTPS_PROXY") || System.get_env("https_proxy") do
+      Mix.shell().info("Using HTTPS_PROXY: #{proxy}")
+      %{host: host, port: port} = URI.parse(proxy)
+      :httpc.set_options([{:https_proxy, {{String.to_charlist(host), port}, []}}])
+    end
+
+    # https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/inets
+    # TODO: This may no longer be necessary from Erlang/OTP 25.0 or later.
+    https_options = [
+      ssl:
+        [
+          verify: :verify_peer,
+          customize_hostname_check: [
+            match_fun: :public_key.pkix_verify_hostname_match_fun(:https)
+          ]
+        ] ++ cacerts_options()
+    ]
+
+    options = [body_format: :binary]
+
+    case :httpc.request(:get, {url_charlist, []}, https_options, options) do
+      {:ok, {{_, 200, _}, _headers, body}} ->
+        {:ok, body}
+
+      other ->
+        {:error, "couldn't fetch NIF from #{url}: #{inspect(other)}"}
+    end
+  end
+
+  defp cacerts_options do
+    cond do
+      path = System.get_env("ELIXIR_MAKE_CACERT") ->
+        [cacertfile: path]
+
+      certs = otp_cacerts() ->
+        [cacerts: certs]
+
+      Application.spec(:castore, :vsn) ->
+        [cacertfile: Application.app_dir(:castore, "priv/cacerts.pem")]
+
+      Application.spec(:certifi, :vsn) ->
+        [cacertfile: Application.app_dir(:certifi, "priv/cacerts.pem")]
+
+      path = cacerts_from_os() ->
+        [cacertfile: path]
+
+      true ->
+        warn_no_cacerts()
+        []
+    end
+  end
+
+  defp otp_cacerts do
+    if System.otp_release() >= "25" do
+      # cacerts_get/0 raises if no certs found
+      try do
+        :public_key.cacerts_get()
+      rescue
+        _ ->
+          nil
+      end
+    end
+  end
+
+  # https_opts and related code are taken from
+  # https://github.com/elixir-cldr/cldr_utils/blob/v2.19.1/lib/cldr/http/http.ex
+  @certificate_locations [
+    # Debian/Ubuntu/Gentoo etc.
+    "/etc/ssl/certs/ca-certificates.crt",
+
+    # Fedora/RHEL 6
+    "/etc/pki/tls/certs/ca-bundle.crt",
+
+    # OpenSUSE
+    "/etc/ssl/ca-bundle.pem",
+
+    # OpenELEC
+    "/etc/pki/tls/cacert.pem",
+
+    # CentOS/RHEL 7
+    "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
+
+    # Open SSL on MacOS
+    "/usr/local/etc/openssl/cert.pem",
+
+    # MacOS & Alpine Linux
+    "/etc/ssl/cert.pem"
+  ]
+
+  defp cacerts_from_os do
+    Enum.find(@certificate_locations, &File.exists?/1)
+  end
+
+  defp warn_no_cacerts do
+    Mix.shell().error("""
+    No certificate trust store was found.
+
+    Tried looking for: #{inspect(@certificate_locations)}
+
+    A certificate trust store is required in
+    order to download locales for your configuration.
+    Since elixir_make could not detect a system
+    installed certificate trust store one of the
+    following actions may be taken:
+
+    1. Install the hex package `castore`. It will
+       be automatically detected after recompilation.
+
+    2. Install the hex package `certifi`. It will
+       be automatically detected after recompilation.
+
+    3. Specify the location of a certificate trust store
+       by configuring it in environment variable:
+
+         export ELIXIR_MAKE_CACERT="/path/to/cacerts.pem"
+
+    4. Use OTP 25+ on an OS that has built-in certificate
+       trust store.
+    """)
+  end
+end
diff --git a/lib/mix/tasks/compile.elixir_make.ex b/lib/mix/tasks/compile.elixir_make.ex
@@ -70,6 +70,11 @@ defmodule Mix.Tasks.Compile.ElixirMake do
     * `:make_precompiler_filename` - the filename of the compiled artefact
       without its extension. Defaults to the app name.
 
+    * `:make_precompile_downloader` - a module implementing the `ElixirMake.Downloader`
+      behaviour. You can use this to customize how the precompiled artefacts
+      are downloaded, for example, to add HTTP authentication or to download
+      from an SFTP server. The default implementation uses `:httpc`.
+
     * `:make_force_build` - if build should be forced even if precompiled artefacts
       are available. Defaults to true if the app has a `-dev` version flag.
 
@@ -219,7 +224,7 @@ defmodule Mix.Tasks.Compile.ElixirMake do
         unless File.exists?(archived_fullpath) do
           Mix.shell().info("Downloading precompiled NIF to #{archived_fullpath}")
 
-          with {:ok, archived_data} <- Artefact.download(url) do
+          with {:ok, archived_data} <- Artefact.download(config, url) do
             File.mkdir_p(Path.dirname(archived_fullpath))
             File.write(archived_fullpath, archived_data)
           end
diff --git a/lib/mix/tasks/elixir_make.checksum.ex b/lib/mix/tasks/elixir_make.checksum.ex
@@ -84,7 +84,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do
           Mix.raise("you need to specify either \"--all\" or \"--only-local\" flags")
       end
 
-    artefacts = download_and_checksum_all(urls, options)
+    artefacts = download_and_checksum_all(config, urls, options)
 
     if Keyword.get(options, :print, false) do
       artefacts
@@ -97,7 +97,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do
     Artefact.write_checksums!(artefacts)
   end
 
-  defp download_and_checksum_all(urls, options) do
+  defp download_and_checksum_all(config, urls, options) do
     ignore_unavailable? = Keyword.get(options, :ignore_unavailable, false)
 
     tasks =
@@ -106,7 +106,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do
         fn {{_target, _nif_version}, url} ->
           checksum_algo = Artefact.checksum_algo()
           checksum_file_url = "#{url}.#{Atom.to_string(checksum_algo)}"
-          artifact_checksum = Artefact.download(checksum_file_url)
+          artifact_checksum = Artefact.download(config, checksum_file_url)
 
           with {:ok, body} <- artifact_checksum,
                [checksum, basename] <- String.split(body, " ", trim: true) do
@@ -117,7 +117,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do
                checksum_algo: checksum_algo
              }}
           else
-            _ -> {:download, url, Artefact.download(url)}
+            _ -> {:download, url, Artefact.download(config, url)}
           end
         end,
         timeout: :infinity,
diff --git a/test/fixtures/my_app/mix.exs b/test/fixtures/my_app/mix.exs
@@ -10,7 +10,7 @@ defmodule MyApp.Precompiler do
   @behaviour ElixirMake.Precompiler
 
   @impl true
-  def current_target, do: "target"
+  def current_target, do: {:ok, "target"}
 
   @impl true
   def all_supported_targets(_), do: ["target"]
@@ -24,3 +24,14 @@ defmodule MyApp.Precompiler do
     :ok
   end
 end
+
+defmodule MyApp.Downloader do
+  @behaviour ElixirMake.Downloader
+
+  @impl true
+  def download(url) do
+    IO.puts("Custom downloader downloading from #{url}")
+    path = String.replace(url, "https://example.com/", __DIR__ <> "/dl/")
+    File.read(path)
+  end
+end
diff --git a/test/mix/tasks/compile.make_test.exs b/test/mix/tasks/compile.make_test.exs
