diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..719619a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Handle line endings automatically for files detected as text +# and leave all files detected as binary untouched. +* text=auto + +# Elixir files +*.ex diff=elixir +*.exs diff=elixir diff --git a/.github/workflows/elixir.yaml b/.github/workflows/elixir.yaml index 4e18f44..4c3e71f 100644 --- a/.github/workflows/elixir.yaml +++ b/.github/workflows/elixir.yaml @@ -9,20 +9,21 @@ on: permissions: contents: read +env: + MIX_ENV: test + jobs: - test: - name: Test app + lint: + name: Lint runs-on: ubuntu-latest - env: - MIX_ENV: test - steps: - uses: actions/checkout@v4 - uses: erlef/setup-beam@v1 id: beam with: - version-file: .tool-versions + otp-version: "27.2" + elixir-version: "1.18.0" version-type: strict - name: Restore the deps and _build cache @@ -36,7 +37,7 @@ jobs: path: | deps _build - key: ${{ runner.os }}-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}-${{ env.MIX_ENV }}-mixlockhash-${{ env.MIX_LOCK_HASH }} + key: lint-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}-${{ env.MIX_ENV }}-mixlockhash-${{ env.MIX_LOCK_HASH }} - name: Install mix dependencies if: steps.cache.outputs.cache-hit != 'true' @@ -56,5 +57,49 @@ jobs: - name: Run dialyzer run: mix dialyzer --format github --format dialyxir + - name: Check formatted + run: mix format --check-formatted + + test: + needs: lint + strategy: + matrix: + os: [ubuntu-latest, windows-2022] + name: Test + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - uses: erlef/setup-beam@v1 + id: beam + with: + otp-version: "27.2" + elixir-version: "1.18.0" + version-type: strict + + - name: Restore the deps and _build cache + uses: actions/cache@v4 + id: cache + env: + OTP_VERSION: ${{ steps.beam.outputs.otp-version }} + ELIXIR_VERSION: ${{ steps.beam.outputs.elixir-version }} + MIX_LOCK_HASH: ${{ hashFiles('**/mix.lock') }} + with: + path: | + deps + _build + key: check_and_test-${{ runner.os }}-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}-${{ env.MIX_ENV }}-mixlockhash-${{ env.MIX_LOCK_HASH }} + + - name: Install mix dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: mix deps.get + + - name: Compile dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: mix deps.compile + + - name: Compile + run: mix compile --warnings-as-errors --force + - name: Check and Test run: mix ci diff --git a/.gitignore b/.gitignore index 41311b6..32acc7c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ numscriptex-*.tar # Temporary files, for example, from tests. /tmp/ + +# Numscript-WASM binary file +/priv/numscript.wasm diff --git a/README.md b/README.md index a83d45b..1337c15 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ -# NumscripEx +# NumscriptEx +NumscriptEx is a library that allows its users to check and run Numscripts in Elixir. And if you don't know what are +Numscripts, here a quick explanation: + [Numscript](https://docs.formance.com/numscript/) is a DSL made by [Formance](https://www.formance.com/) that simplifies complex financial transactions with scripts that are easy to read, -so yout don't need a big, complex and error-prone codebase to deal with your finances. - -You can see and execute some examples on the [Numscript Playground](https://playground.numscript.org/?template=simple-send). +so you don't need a big, complex and error-prone codebase to deal with your finances. -`NumscriptEx` is a library that allows its users to check and run said numscripts in Elixir. +You can see and execute some examples at the [Numscript Playground](https://playground.numscript.org/?template=simple-send). ## Installation You will just need to add `:numscriptex` as a dependency on your `mix.exs`, and run the `mix deps.get` command: @@ -17,6 +18,21 @@ def deps do ] end ``` +### Configuration +NumscriptEx needs some external assets ([Numscript-WASM](https://github.com/PagoPlus/numscript-wasm)), +and you can configure said assets if you want. + +Available configurations: +- `:version`: is the release version to be downloaded; +- `:retries`: number of download retries in case of failure. + +Ex: +```elixir +config :numscriptex, + version: "0.0.2", + retries: 3 +``` +These above are the default values. ## Usage This library basically has two core functions: `Numscriptex.check/1` and `Numscriptex.run/2`. @@ -30,7 +46,7 @@ But before introducing these two functions, you will need to know what is the `N A numscript needs some other data aside the script itself to run correctly, and `Numscriptex.Run` solves this problem. -If you want to know what exactly these additional data are, you can see the +If you want to know what exactly these additional data are, you can see the [numscript playground](https://playground.numscript.org/?template=simple-send) for examples. The abstraction is made by creating a struct: @@ -124,7 +140,7 @@ iex> {:ok, %{ ...> } ...> } ``` -`:script` is the only key that will always return if your script is valid, the +`:script` is the only key that will always return if your script is valid, the other three are optional. ### Run To use `run/2` your first argument must be your script (the same you used in `check/1`), and the second must be the `%Numscriptex.Run{}` struct. Ex: @@ -134,7 +150,7 @@ iex> Numscriptex.run(script, struct) {:ok, result} ``` -Where result will be something like this: +Where result will be something like this: ```elixir iex> %{ ...> postings: [ @@ -165,8 +181,8 @@ iex> %{ ...> initial_balance: 0 ...> } ...> ], -...> accountMeta: %{} -...> txMeta: %{} +...> accountMeta: %{} +...> txMeta: %{} ...> } ``` diff --git a/lib/numscriptex.ex b/lib/numscriptex.ex index 4029fae..b79f4c8 100644 --- a/lib/numscriptex.ex +++ b/lib/numscriptex.ex @@ -7,10 +7,14 @@ defmodule Numscriptex do Already checked the script and want to execute him? Use the `run/2` function. """ + alias Numscriptex.AssetsManager alias Numscriptex.Balances alias Numscriptex.CheckLog alias Numscriptex.Utilities + require AssetsManager + require Logger + @type check_log() :: CheckLog.t() @type check_result() :: %{ @@ -46,10 +50,7 @@ defmodule Numscriptex do optional(:details) => any() } - @binary :numscriptex - |> :code.priv_dir() - |> Path.join("numscript.wasm") - |> File.read!() + AssetsManager.ensure_wasm_binary_is_valid() @doc """ To use `check/1` you just need to pass your numscript as its argument. @@ -65,13 +66,13 @@ defmodule Numscriptex do """ @spec check(binary()) :: {:ok, check_result()} | {:error, errors()} def check(input) do - case process(input, :check) do - {:ok, details} -> - {:ok, %{script: input, details: normalize_check_logs(details)}} - + case execute_command(input, :check) do :ok -> {:ok, %{script: input}} + {:ok, details} -> + {:ok, %{script: input, details: normalize_check_logs(details)}} + {:error, %{reason: errors}} -> {:error, %{reason: normalize_check_logs(errors)}} end @@ -81,12 +82,12 @@ defmodule Numscriptex do To use `run/2` your first argument must be your script, and the second must be a `%Numscriptex.Run{}` (go to Numscriptex.Run module to see more) struct. Ex: - + ```elixir iex> script = "send [USD/2 100] ( source = @foo destination = @bar)" ...> balances = %{"foo" => %{"USD/2" => 500, "EUR/2" => 300}} - ...> - ...> struct = + ...> + ...> struct = ...> Numscriptex.Run.new() ...> |> Numscriptex.Run.put!(:balances, balances) ...> |> Numscriptex.Run.put!(:metadata, %{}) @@ -103,7 +104,7 @@ defmodule Numscriptex do |> Map.from_struct() |> Map.merge(%{script: numscript}) |> JSON.encode!() - |> process(:run) + |> execute_command(:run) |> maybe_put_final_balance(initial_balance) |> standardize_run_result() end @@ -135,7 +136,7 @@ defmodule Numscriptex do defp maybe_put_final_balance({:error, _reason} = error, _initial_balance), do: error - defp process(input, operation) do + defp execute_command(input, operation) do {:ok, stdout_pipe} = Wasmex.Pipe.new() {:ok, stdin_pipe} = Wasmex.Pipe.new() {:ok, stderr_pipe} = Wasmex.Pipe.new() @@ -150,33 +151,41 @@ defmodule Numscriptex do stderr: stderr_pipe } - {:ok, pid} = Wasmex.start_link(%{bytes: @binary, wasi: wasi}) + binary_path = AssetsManager.binary_path() - case Wasmex.call_function(pid, :_start, []) do - {:ok, []} -> + with {:ok, binary} <- File.read(binary_path), + {:ok, pid} <- Wasmex.start_link(%{bytes: binary, wasi: wasi}), + {{:ok, _}, _pid} <- {Wasmex.call_function(pid, :_start, []), pid} do + GenServer.stop(pid, :normal) + process(pid, stdout_pipe, stderr_pipe) + else + {{:error, _reason}, pid} -> GenServer.stop(pid) + process(pid, stdout_pipe, stderr_pipe) - Wasmex.Pipe.seek(stderr_pipe, 0) - error = Wasmex.Pipe.read(stderr_pipe) + {:error, reason} when is_atom(reason) -> + {:error, %{reason: handle_posix_errors(reason)}} - Wasmex.Pipe.seek(stdout_pipe, 0) + {:error, reason} -> + {:error, %{reason: reason}} + end + end - stdout_pipe - |> Wasmex.Pipe.read() - |> JSON.decode() + defp process(pid, stdout_pipe, stderr_pipe) when is_pid(pid) do + Wasmex.Pipe.seek(stdout_pipe, 0) + stdout = Wasmex.Pipe.read(stdout_pipe) + + Wasmex.Pipe.seek(stderr_pipe, 0) + error = Wasmex.Pipe.read(stderr_pipe) + + case JSON.decode(stdout) do + {:ok, _content} = result -> + result |> handle_process() |> maybe_put_stderr(error) |> handle_errors() {:error, _reason} -> - GenServer.stop(pid) - - Wasmex.Pipe.seek(stderr_pipe, 0) - error = Wasmex.Pipe.read(stderr_pipe) - - Wasmex.Pipe.seek(stdout_pipe, 0) - stdout = Wasmex.Pipe.read(stdout_pipe) - {:error, stdout} |> handle_process() |> maybe_put_stderr(error) @@ -184,6 +193,19 @@ defmodule Numscriptex do end end + defp handle_posix_errors(reason) do + file_read_error = + reason + |> :file.format_error() + |> to_string() + + if file_read_error =~ "unknown POSIX error" do + reason + else + "Can't read the WASM binary due to: #{file_read_error}" + end + end + defp maybe_put_stderr({:error, reason}, stderr) when is_map(reason) do is_stderr_empty? = stderr diff --git a/lib/numscriptex/assets_manager.ex b/lib/numscriptex/assets_manager.ex new file mode 100755 index 0000000..4d03457 --- /dev/null +++ b/lib/numscriptex/assets_manager.ex @@ -0,0 +1,187 @@ +defmodule Numscriptex.AssetsManager do + @moduledoc """ + `Numscriptex.AssetsManager` is responsible for ensuring that the Numscriptex + library have all it needs (i.e. a WASM binary file that is used to check and run Numscripts) + to run correctly at compile time. + """ + require Logger + + @release_version Application.compile_env(:numscriptex, :version, "0.0.2") + @retries Application.compile_env(:numscriptex, :retries, 3) + @numscript_checksums_url "https://github.com/PagoPlus/numscript-wasm/releases/download/v#{@release_version}/numscript_checksums.txt" + @numscript_wasm_url "https://github.com/PagoPlus/numscript-wasm/releases/download/v#{@release_version}/numscript.wasm" + @binary_path :numscriptex + |> :code.priv_dir() + |> Path.join("numscript.wasm") + |> to_charlist() + + defmacro ensure_wasm_binary_is_valid do + quote do + File.mkdir_p!(:code.priv_dir(:numscriptex)) + + if File.exists?(unquote(@binary_path)) do + unquote(maybe_retry_download(compare_checksums())) + else + unquote(maybe_retry_download(download_and_compare_binary())) + end + end + end + + def binary_path, do: @binary_path + + def hash_wasm_binary do + # Logic explanation: + # 1. Opens the wasm binary as a stream and split it by chunks of 1024 bytes and then iterate + # the chunks with `Enum.reduce/3`; + # 2. The reduce takes a `:crypto.hash_init/1` state as it's argument, then update the state + # with each line hash using `:crypto.hash_update/2`; + # 2.1. The `:crypto.hash_init/1` is using the sha256 hash algorithm because it's the same as the checksums, + # so if the algorithm on the checksums changed, you'll need to change here too. + # 3. Uses `:crypto.hash_final/1` to finalize the streaming hash calculation; + # 4. Encode the hash to the hexadecimal base (also the same as the checksums). + quote do + File.stream!(unquote(@binary_path), 1024) + |> Enum.reduce(:crypto.hash_init(:sha256), fn line, acc -> + :crypto.hash_update(acc, line) + end) + |> :crypto.hash_final() + |> Base.encode16(case: :lower) + end + end + + defp maybe_retry_download(result) do + quote do + case unquote(result) do + {:error, :invalid_checksums} -> + unquote(retry_numscript_wasm_download()) + + result -> + result + end + end + end + + defp download_and_compare_binary do + quote do + with :ok <- unquote(download_wasm_file()) do + unquote(compare_checksums()) + end + end + end + + defp retry_numscript_wasm_download do + quote do + retries = unquote(@retries) + + cond do + retries <= 0 -> + :ok + + retries == 1 -> + unquote(download_and_compare_binary()) + + retries > 1 -> + unquote(retry_download_reduce()) + end + end + end + + defp retry_download_reduce do + quote do + Enum.reduce_while(1..retries, :try_again, fn retry, prev_result -> + if prev_result != :ok do + {:cont, unquote(download_and_compare_binary())} + else + {:halt, prev_result} + end + end) + end + end + + defp compare_checksums do + quote do + with {:ok, checksums} <- unquote(remote_checksums()), + hash <- unquote(hash_wasm_binary()) do + if checksums == hash do + :ok + else + Logger.error("Based on the checksums, the numscript-wasm binary is not valid.") + + {:error, :invalid_checksums} + end + end + end + end + + defp download_wasm_file do + quote do + unquote(maybe_delete_wasm_binary()) + + request = + :httpc.request( + :get, + {unquote(@numscript_wasm_url), []}, + [], + stream: unquote(@binary_path) + ) + + case request do + {:ok, :saved_to_file} -> + :ok + + {:ok, {{_, status_code, status_message}, _, _}} when status_code not in 200..299 -> + Logger.error("Download request failed (#{status_code}): #{status_message}.") + + raise CompileError + + {:error, reason} -> + Logger.error( + "Failed to download Numscript-WASM binary. Reason: #{inspect(reason)}. Binary path: #{unquote(@binary_path)}" + ) + + raise CompileError + end + end + end + + defp maybe_delete_wasm_binary do + quote do + case File.rm(unquote(@binary_path)) do + :ok -> + :ok + + {:error, :enoent} -> + :ok + + {:error, reason} -> + raise File.Error, reason: reason + end + end + end + + defp remote_checksums do + quote do + case :httpc.request(:get, {unquote(@numscript_checksums_url), []}, [], []) do + {:ok, {{_, status_code, status_message}, _, _}} when status_code not in 200..299 -> + Logger.error("Download request failed (#{status_code}): #{status_message}.") + + raise CompileError + + {:ok, {{_protocol, _status_code, _status_message}, _header, body}} -> + [checksums, _file_name] = + body + |> to_string() + |> String.split() + + {:ok, String.trim(checksums)} + + {:error, reason} -> + Logger.error( + "Failed to download Numscript-WASM checksums from release assets. Reason: #{reason}." + ) + + raise CompileError + end + end + end +end diff --git a/mix.exs b/mix.exs index c1499be..8f63098 100644 --- a/mix.exs +++ b/mix.exs @@ -38,7 +38,6 @@ defmodule Numscriptex.MixProject do defp aliases do [ ci: [ - "format --check-formatted", "deps.unlock --check-unused", "credo suggest --strict --all", "test" diff --git a/priv/numscript.wasm b/priv/numscript.wasm deleted file mode 100644 index 47f6cef..0000000 Binary files a/priv/numscript.wasm and /dev/null differ diff --git a/test/numscriptex/assets_manager_test.exs b/test/numscriptex/assets_manager_test.exs new file mode 100644 index 0000000..d4f46a9 --- /dev/null +++ b/test/numscriptex/assets_manager_test.exs @@ -0,0 +1,65 @@ +defmodule Numscriptex.AssetsManagerTest do + use ExUnit.Case, async: false + + alias Numscriptex.AssetsManager + + import AssetsManager, only: [hash_wasm_binary: 0] + + require AssetsManager + require Logger + + doctest AssetsManager + + setup_all do + binary_path = + :numscriptex + |> :code.priv_dir() + |> Path.join("numscript.wasm") + + %{binary_path: binary_path} + end + + describe "ensure_wasm_binary_is_installed_and_valid/0" do + test "if the binary exists and is valid, do nothing", %{binary_path: binary_path} do + assert File.exists?(binary_path) + assert {:ok, file_stats_before} = File.stat(binary_path) + + AssetsManager.ensure_wasm_binary_is_valid() + assert {:ok, file_stats_after} = File.stat(binary_path) + + assert File.exists?(binary_path) + assert file_stats_before.ctime == file_stats_after.ctime + end + + test "if the binary does not exists, install a new one", %{binary_path: binary_path} do + assert File.rm(binary_path) == :ok + refute File.exists?(binary_path) + + AssetsManager.ensure_wasm_binary_is_valid() + + assert File.exists?(binary_path) + end + + @tag :tmp_dir + test "if the binary exists but is invalid, install a valid one", %{ + binary_path: binary_path, + tmp_dir: tmp_dir + } do + dest_path = Path.join(tmp_dir, "numscript.wasm") + expected_checksums = unquote(hash_wasm_binary()) + + File.copy!(binary_path, dest_path) + File.copy!(binary_path, binary_path, 1024) + + wrong_checksums = unquote(hash_wasm_binary()) + assert expected_checksums != wrong_checksums + + AssetsManager.ensure_wasm_binary_is_valid() + + new_checksums = unquote(hash_wasm_binary()) + assert new_checksums == expected_checksums + + File.copy!(dest_path, binary_path) + end + end +end diff --git a/test/numscriptex_test.exs b/test/numscriptex_test.exs index c0a7234..8422fa4 100644 --- a/test/numscriptex_test.exs +++ b/test/numscriptex_test.exs @@ -754,7 +754,7 @@ defmodule NumscriptexTest do assert {:error, error} = Numscriptex.run(script, struct) - assert error.reason == + assert error.details == "Not enough funds. Needed [USD/2 100] (only [USD/2 99] available)" end @@ -782,7 +782,7 @@ defmodule NumscriptexTest do struct = build_run_struct(balances, metadata, variables) assert {:error, error} = Numscriptex.run(script, struct) - assert error.reason == "Variable is missing in json: user" + assert error.details == "Variable is missing in json: user" end test "fails with invalid script" do @@ -797,7 +797,7 @@ defmodule NumscriptexTest do test "fails with invalid script but valid sctruct" do assert {:error, error} = Numscriptex.run(%{script: ""}, %Numscriptex.Run{}) - assert error.reason == + assert error.details == "json: cannot unmarshal object into Go struct field RunInputOpts.script of type string" end