Refactor Enum to Streams - closes #3 (#6)

* Refactor datasets to Streams. Closes #3

* Refactor to Streams

Author: devstopfix

README.md

@@ -1,6 +1,6 @@
 # Gnuplot Elixir
 
-A simple interface from [Elixir data][7] to the [Gnuplot graphing utility][1] that uses [Erlang Ports][5] to transmit data from your application to Gnuplot. Datasets are streamed directly to STDIN without temporary files and you can plot [1M points in 20 seconds](examples/stress.exs) on an AWS t2.medium 2 vCPU server.
+A simple interface from [Elixir data][7] to the [Gnuplot graphing utility][1] that uses [Erlang Ports][5] to transmit data from your application to Gnuplot. Datasets are streamed directly to STDIN without temporary files and you can plot [1M points in 12.7 seconds](examples/stress.exs).
 
 Please visit the [Gnuplot demos gallery](http://gnuplot.sourceforge.net/demo/) to see all the possibilities, the [manual which describes the grammar](http://www.gnuplot.info/docs_5.2/Gnuplot_5.2.pdf), and the [examples folder](examples/).
 
@@ -11,7 +11,7 @@ This is a conversion of the [Clojure Gnuplot library][4] by [aphyr][2].
 The `plot` function takes two arguments:
 
 * a list of commands (each of which is a list of terms)
-* a list of datasets (not required when plotting functions)
+* a list of Streams or Enumerable datasets (not required when plotting functions)
 
 Commands are lists of terms that normally start with an atom such as `:set`. They may be written as lists or [Word lists](https://elixir-lang.org/getting-started/sigils.html#word-lists) - the following lines are equivalent:
 
@@ -44,7 +44,7 @@ Gnuplot will by default open a window containing your plot:
 
 ### PNG of two datasets
 
-Write two datasets to a file:
+Write two datasets to a PNG file:
 
 ```elixir
 G.plot([
@@ -56,7 +56,7 @@ G.plot([
     G.list(
       ["-", :title, "uniform", :with, :points],
       ["-", :title, "normal", :with, :points])]
-  ], 
+  ],
   [
         for(n <- 0..200, do: [n, n * :rand.uniform()]),
         for(n <- 0..200, do: [n, n * :rand.normal()])
@@ -95,7 +95,7 @@ by adding `gnuplot` to your list of dependencies in `mix.exs`:
 ```elixir
 def deps do
   [
-    {:gnuplot, "~> 0.19.71"}
+    {:gnuplot, "~> 0.19.72"}
   ]
 end
 ```
@@ -110,17 +110,19 @@ Some tests create plots which require `gnuplot` to be installed. They can be be
 
 ## Performance
 
-The performance of the library on a MacBook Air is comparable to the Clojure version when `gnuplot` draws to a GUI. It is a little faster when writing directly to a PNG. It is faster still when running on a server. The times below are in milliseconds. Each plot was made in increasing order of the number of points and after a cold start of the VM.
+The performance of the library on a MacBook Air is comparable to the Clojure version when `gnuplot` draws to a GUI. It is a little faster when writing directly to a PNG when running on a server. The times below are in milliseconds. Each plot was made in increasing order of the number of points and after a cold start of the VM. The last two columns show the refactoring from Enumerable to Streams.
 
-| Points | Clojure GUI | Elixir GUI | Elixir PNG | Ubuntu 16.04 |
-| -----: | ----------: | ---------: | ---------: | -----------: |
-|      1 |       1,487 |          5 |          2 |            4 |
-|     10 |       1,397 |         10 |         10 |           <1 |
-|    1e2 |       1,400 |          4 |         49 |            1 |
-|    1e3 |       1,381 |         59 |         40 |            8 |
-|    1e4 |       1,440 |        939 |        349 |          211 |
-|    1e5 |       5,784 |      5,801 |      4,091 |        1,873 |
-|    1e6 |      49,275 |     43,464 |     41,521 |       19,916 |
+| Points | Clojure GUI | Elixir GUI | Elixir PNG | Elixir Enum   | Elixir Stream |
+| -----: | ----------: | ---------: | ---------: | ------------: | ------------: |
+|      1 |       1,487 |          5 |         18 |             4 |             5 |
+|     10 |       1,397 |         10 |          1 |            <1 |             1 |
+|    1e2 |       1,400 |          4 |         12 |             1 |             1 |
+|    1e3 |       1,381 |         59 |         52 |             8 |            10 |
+|    1e4 |       1,440 |        939 |        348 |           211 |           211 |
+|    1e5 |       5,784 |      5,801 |      3,494 |         1,873 |         1,313 |
+|    1e6 |      49,275 |     43,464 |     35,505 |        19,916 |        12,775 |
+|        |     MacBook |    MacBook |    MacBook |  Ubuntu 16.04 |  Ubuntu 16.04 |
+|        |  2.5 GHz i5 | 2.5 GHz i5 | 2.5 GHz i5 | 3.3 GHz 2vCPU | 3.3 GHz 2vCPU |
 
 ![performance](docs/perf.PNG)
 
@@ -130,7 +132,8 @@ clojure_gui = [1.487, 1.397, 1.400, 1.381, 1.440, 5.784, 49.275]
 elixir_gui  = [0.005, 0.010, 0.004, 0.059, 0.939, 5.801, 43.464]
 elixir_png  = [0.002, 0.010, 0.049, 0.040, 0.349, 4.091, 41.521]
 ubuntu_t2m = [0.004, 0.002, 0.001, 0.008, 0.211, 1.873, 19.916]
-datasets    = for ds <- [clojure_gui, elixir_gui, elixir_png, ubuntu_t2m], do: Enum.zip(points, ds)
+ubuntu_stream = [0.002, 0.001, 0.001, 0.009, 0.204, 1.279, 12.858]
+for ds <- [clojure_gui, elixir_gui, elixir_png, ubuntu_t2m, ubuntu_stream], do: Enum.zip (points, ds)
 
 G.plot([
   [:set, :title, "Render scatter plot"],
@@ -140,14 +143,16 @@ G.plot([
   ~w(set logscale xy)a,
   ~w(set grid xtics ytics)a,
   ~w(set style line 1 lw 4 lc '#63b132')a,
-  ~w(set style line 2 lw 4 lc '#421C52')a,
-  ~w(set style line 3 lw 4 lc '#732C7B')a,
+  ~w(set style line 2 lw 4 lc '#2C001E')a,
+  ~w(set style line 3 lw 4 lc '#5E2750')a,
   ~w(set style line 4 lw 4 lc '#E95420')a,
+  ~w(set style line 5 lw 4 lc '#77216F')a,
   [:plot, G.list(
-      ["-", :title, "Clojure GUI", :with, :lines, :ls, 1],
-      ["-", :title, "Elixir GUI",  :with, :lines, :ls, 2],
-      ["-", :title, "Elixir PNG",  :with, :lines, :ls, 3],
-      ["-", :title, "Elixir PNG t2.m", :with, :lines, :ls, 4]
+    ["-", :title, "Clojure GUI", :with, :lines, :ls, 1],
+    ["-", :title, "Elixir GUI", :with, :lines, :ls, 2],
+    ["-", :title, "Elixir PNG", :with, :lines, :ls, 3],
+    ["-", :title, "Elixir t2.m", :with, :lines, :ls, 4],
+    ["-", :title, "Elixir Stream", :with, :lines, :ls, 5]
   )]], datasets
 ])
 ```

docs/perf.PNG

@@ -20,27 +20,30 @@ defmodule Perf do
       ~w(set logscale xy)a,
       ~w(set grid xtics ytics)a,
       ~w(set style line 1 lw 4 lc '#63b132')a,
-      ~w(set style line 2 lw 4 lc '#421C52')a,
-      ~w(set style line 3 lw 4 lc '#732C7B')a,
+      ~w(set style line 2 lw 4 lc '#2C001E')a,
+      ~w(set style line 3 lw 4 lc '#5E2750')a,
       ~w(set style line 4 lw 4 lc '#E95420')a,
+      ~w(set style line 5 lw 4 lc '#77216F')a,
       [
         :plot,
         G.list(
-          ["-", :title, "Clojure GUI MB", :with, :lines, :ls, 1],
-          ["-", :title, "Elixir GUI MB", :with, :lines, :ls, 2],
-          ["-", :title, "Elixir PNG MB", :with, :lines, :ls, 3],
-          ["-", :title, "Elixir PNG t2.m", :with, :lines, :ls, 4]
+          ["-", :title, "Clojure GUI", :with, :lines, :ls, 1],
+          ["-", :title, "Elixir GUI", :with, :lines, :ls, 2],
+          ["-", :title, "Elixir PNG", :with, :lines, :ls, 3],
+          ["-", :title, "Elixir t2.m", :with, :lines, :ls, 4],
+          ["-", :title, "Elixir Stream", :with, :lines, :ls, 5]
         )
       ]
     ]
 
   def data do
     points = [1, 10, 100, 1000, 10_000, 100_000, 1_000_000]
-    clojure_gui = [1.487, 1.397, 1.400, 1.381, 1.440, 5.784, 49.275]
-    elixir_gui = [0.005, 0.010, 0.004, 0.059, 0.939, 5.801, 43.464]
-    elixir_png = [0.002, 0.010, 0.049, 0.040, 0.349, 4.091, 41.521]
-    ubuntu_t2m = [0.004, 0.002, 0.001, 0.008, 0.211, 1.873, 19.916]
-    for ds <- [clojure_gui, elixir_gui, elixir_png, ubuntu_t2m], do: Enum.zip(points, ds)
+    clojure_gui =   [1.487, 1.397, 1.400, 1.381, 1.440, 5.784, 49.275]
+    elixir_gui =    [0.005, 0.010, 0.004, 0.059, 0.939, 5.801, 43.464]
+    elixir_png =    [0.018, 0.001, 0.012, 0.052, 0.348, 3.494, 35.505]
+    ubuntu_t2m =    [0.004, 0.002, 0.001, 0.008, 0.211, 1.873, 19.916]
+    ubuntu_stream = [0.002, 0.001, 0.001, 0.009, 0.204, 1.279, 12.858]
+    for ds <- [clojure_gui, elixir_gui, elixir_png, ubuntu_t2m, ubuntu_stream], do: Enum.zip(points, ds)
   end
 
   def plot, do: G.plot(target() ++ commands(), data())

examples/perf.exs

@@ -19,7 +19,8 @@ defmodule Sine do
         'sin(x)',
         :title,
         "Sine Wave",
-        :ls, 1
+        :ls,
+        1
       ]
     ]
 

examples/sine.exs

@@ -19,11 +19,16 @@ defmodule Stress do
     ]
 
   def data(n),
-    do: [
-      for(i <- 0..n, do: [i / n, i * :rand.uniform()])
-    ]
-
-  def plot(n), do: G.plot(target(n) ++ commands(), data(n))
+    do:
+      Stream.unfold(0, fn i ->
+        if i <= n do
+          {[i / n, i * :rand.uniform()], i + 1}
+        else
+          nil
+        end
+      end)
+
+  def plot(n), do: G.plot(target(n) ++ commands(), [data(n)])
 end
 
 # time mix run examples/stress.exs

examples/stress.exs

@@ -28,15 +28,22 @@ defmodule Gnuplot do
   def plot(commands, datasets) do
     with {:ok, path} = gnuplot_bin(),
          cmd = Commands.format(commands),
-         data = format_datasets(datasets),
          args = ["-p", "-e", cmd],
          port = Port.open({:spawn_executable, path}, [:binary, args: args]) do
-      Enum.each(data, fn row -> send(port, {self(), {:command, row}}) end)
+      transmit(port, datasets)
       {_, :close} = send(port, {self(), :close})
       {:ok, cmd}
     end
   end
 
+  defp transmit(port, datasets) do
+    :ok =
+      datasets
+      |> format_datasets()
+      |> Stream.each(fn row -> send(port, {self(), {:command, row}}) end)
+      |> Stream.run()
+  end
+
   @doc """
   Plot a function that has no dataset.
   """
@@ -54,6 +61,8 @@ defmodule Gnuplot do
 
   def list(a, b, c, d), do: %Commands.List{xs: [a, b, c, d]}
 
+  def list(a, b, c, d, e), do: %Commands.List{xs: [a, b, c, d, e]}
+
   @doc """
   Find the gnuplot executable.
   """

lib/gnuplot.ex

@@ -12,21 +12,19 @@ defmodule Gnuplot.Dataset do
   @gnuplot_end_data "\ne\n"
 
   @doc """
-  Convert a list of datasets.
+  Format datasets into Gnuplot STDIN format. Datasets must be Enumerable and can be Streams.
 
   A dataset is a list of points, each point is a list of numbers.
   """
-  @spec format_datasets(list(t())) :: [String.t()]
   def format_datasets(datasets) do
-    Enum.flat_map(datasets, &format_dataset/1)
+    Stream.flat_map(datasets, &format_dataset/1)
   end
 
-  @spec format_dataset(t()) :: [String.t()]
   defp format_dataset(dataset) do
     dataset
-    |> Enum.map(&format_point/1)
-    |> Enum.intersperse(@gnuplot_end_row)
-    |> Enum.concat([@gnuplot_end_data])
+    |> Stream.map(&format_point/1)
+    |> Stream.intersperse(@gnuplot_end_row)
+    |> Stream.concat([@gnuplot_end_data])
   end
 
   @spec format_point(point()) :: String.t()

lib/gnuplot/dataset.ex

@@ -4,7 +4,7 @@ defmodule Gnuplot.MixProject do
   def project do
     [
       app: :gnuplot,
-      version: "0.19.71",
+      version: "0.19.73",
       elixir: "~> 1.4",
       start_permanent: Mix.env() == :prod,
       description: "Interface between Elixir and Gnuplot graphing library",

mix.exs

@@ -2,6 +2,7 @@ defmodule GnuplotTest do
   use ExUnit.Case
   alias Gnuplot, as: G
   alias Gnuplot.Commands, as: C
+  alias Gnuplot.Dataset, as: D
 
   test "List of commands" do
     assert "set xtics off;\nset ytics off" ==
@@ -33,6 +34,14 @@ defmodule GnuplotTest do
     assert "set title \"simple's\"" == C.format([[:set, :title, "simple's"]])
   end
 
+  test "Datasets" do
+    input = [[[1, 1], [1, 2]], [[2, 3], [2, 4], [2, 5]]]
+    expected = ["1 1", "\n", "1 2", "\ne\n", "2 3", "\n", "2 4", "\n", "2 5", "\ne\n"]
+
+    assert expected ==
+             input |> D.format_datasets() |> Enum.to_list()
+  end
+
   @tag gnuplot: true
   test "Gnuplot is installed" do
     assert {:ok, path} = G.gnuplot_bin()