Merge pull request #10 from tpgillam/tg/tidy

Allow specifying a mutating version of the operator

.github/workflows/ci.yml

@@ -54,4 +54,3 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
-          GKSwstype: "100" # https://discourse.julialang.org/t/generation-of-documentation-fails-qt-qpa-xcb-could-not-connect-to-display/60988

Project.toml

@@ -1,7 +1,7 @@
 name = "AssociativeWindowAggregation"
 uuid = "444271a7-5434-4a02-b82b-0e30a9223c60"
 authors = ["Thomas Gillam <[email protected]>"]
-version = "0.2.3"
+version = "0.3.0"
 
 [deps]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"

README.md

@@ -10,3 +10,6 @@ Accumulate result of appying binary associative operators on rolling windows.
 The algorithm is constant time with respect to the window length, and is numerically stable. 
 Details can be found in the [documentation](https://tpgillam.github.io/AssociativeWindowAggregation.jl/dev). 
 For demonstrations, see the [documentation examples](https://tpgillam.github.io/AssociativeWindowAggregation.jl/dev/examples) as well as the project under `examples/`.
+
+The windowed algorithm is well suited for use with [OnlineStats.jl](https://github.com/joshday/OnlineStats.jl).
+An [example](https://tpgillam.github.io/AssociativeWindowAggregation.jl/dev/examples/#OnlineStats.jl) of this combination is in the documentation.

docs/Project.toml

@@ -1,4 +1,5 @@
 [deps]
 AssociativeWindowAggregation = "444271a7-5434-4a02-b82b-0e30a9223c60"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"

docs/make.jl

@@ -1,21 +1,29 @@
 using Documenter, AssociativeWindowAggregation
 
-makedocs(
-    format=Documenter.HTML(
-        prettyurls=get(ENV, "CI", "false") == "true"
-    ),
-    modules=[AssociativeWindowAggregation],
-    sitename="AssociativeWindowAggregation.jl",
-    pages=[
-        "Home" => "index.md",
-        "examples.md",
-        "Reference" => [
-            "General window" => "reference/windowed_associative_op.md",
-            "Fixed window" => "reference/fixed_window_associative_op.md",
-            "Time window" => "reference/time_window_associative_op.md",
+# This is inspired by the following, to prevent a `gksqt` process blocking (and needing
+# manual termination) for every plot when generating documentation on MacOS.
+# https://discourse.julialang.org/t/deactivate-plot-display-to-avoid-need-for-x-server/19359/2
+# Another choice could be "100":
+# https://discourse.julialang.org/t/generation-of-documentation-fails-qt-qpa-xcb-could-not-connect-to-display/60988
+# Empirically, both choices seem to work.
+withenv("GKSwstype" => "nul") do
+    makedocs(
+        format=Documenter.HTML(
+            prettyurls=get(ENV, "CI", "false") == "true"
+        ),
+        modules=[AssociativeWindowAggregation],
+        sitename="AssociativeWindowAggregation.jl",
+        pages=[
+            "Home" => "index.md",
+            "examples.md",
+            "Reference" => [
+                "General window" => "reference/base.md",
+                "Fixed window" => "reference/fixed.md",
+                "Time window" => "reference/time.md",
+            ]
         ]
-    ]
-)
+    )
+end
 
 deploydocs(
     repo="github.com/tpgillam/AssociativeWindowAggregation.jl.git",

docs/src/examples.md

@@ -1,11 +1,11 @@
 # Examples
 
 ## Rolling mean
-Here we show a computation of the rolling mean over fixed windows. We use a 
+Here we show a computation of the rolling mean over fixed windows. We could use a 
 `FixedWindowAssociativeOp` to keep track of a rolling sum, then divide by the window
 length, only including values where the window has filled.
 
-```@example mean
+```@example mean; continued=true
 using AssociativeWindowAggregation
 using Plots
 
@@ -16,7 +16,7 @@ plot(x, y; label="raw", title="Rolling means")
 
 for window in [5, 10, 20]
     # Use this to keep track of a windowed sum.
-    state = FixedWindowAssociativeOp{Float64, +}(window)
+    state = FixedWindowAssociativeOp{Float64,+}(window)
 
     z = []
     for value in y
@@ -34,3 +34,82 @@ savefig("mean-plot.svg"); nothing # hide
 ```
 
 ![](mean-plot.svg)
+
+However, if we were to be using very large windows, we should be nervous with the implementation above. 
+This is because, by taking the ratio of two large numbers, we may suffer a loss of precision.
+
+Here is a better implementation, where we create a new object `Mean`, whcih internally stores a mean and a count.
+We then define a `merge` function, which is binary and associative.
+
+```@example mean
+struct Mean
+    n::Int64
+    mean::Float64
+end
+Mean(x::Real) = Mean(1, x)
+
+"""
+    merge(x::Mean, y::Mean) -> Mean
+
+Combine two `Mean` objects into a new `Mean`.
+"""
+function merge(x::Mean, y::Mean)
+    n = x.n + y.n
+    return Mean(n, (x.n / n) * x.mean + (y.n / n) * y.mean)
+end
+
+plot(x, y; label="raw", title="Rolling means")
+for window in [5, 10, 20]
+    # Unlike in the previous example, we now combine our custom `Mean` object.
+    state = FixedWindowAssociativeOp{Mean,merge}(window)
+
+    z = []
+    for value in y
+        # Wrap individual values in a `Mean` to allow us to add them to the window.
+        update_state!(state, Mean(value))
+        if window_full(state)
+            # `window_value` returns a `Mean` object now, so we must extract the mean.
+            push!(z, window_value(state).mean)
+        else
+            push!(z, NaN)
+        end
+    end
+
+    plot!(x, z; label="mean $window", lw=2)
+end
+savefig("mean-plot-2.svg"); nothing # hide
+```
+
+![](mean-plot-2.svg)
+
+## OnlineStats.jl
+
+Many of the estimators from [OnlineStats.jl](https://github.com/joshday/OnlineStats.jl) can be merged associatively.
+Therefore, we can compute online windowed versions of these statistics easily with this framework.
+
+Here is a simple example computing an online windowed standard deviation.
+
+```@example onlinestats
+using AssociativeWindowAggregation
+using OnlineStatsBase
+using Plots
+using Statistics
+
+# Use a window of 100 values.
+state = FixedWindowAssociativeOp{Variance,merge,merge!}(100)
+
+function _wrap(v) 
+    x = Variance()
+    fit!(x, v)
+    return x
+end
+
+values = rand(1000) .- 0.5
+output = [std(window_value(update_state!(state, _wrap(v)))) for v in values]
+
+plot(values; alpha=0.4, label="Input")
+plot!(output; label="std (window=100)")
+savefig("std-plot.svg"); nothing # hide
+```
+
+![](std-plot.svg)

docs/src/reference/windowed_associative_op.md → docs/src/reference/base.md

@@ -6,5 +6,5 @@ Every time a new value is pushed onto the end of the window, we must specify how
 
 ```@autodocs
 Modules = [AssociativeWindowAggregation]
-Pages = ["windowed_associative_op.jl"]
+Pages = ["base.jl"]
 ```

docs/src/reference/fixed_window_associative_op.md → docs/src/reference/fixed.md

@@ -4,5 +4,5 @@ A state to represent a window with a fixed capacity. It will start off empty, ca
 
 ```@autodocs
 Modules = [AssociativeWindowAggregation]
-Pages = ["fixed_window_associative_op.jl"]
+Pages = ["fixed.jl"]
 ```

docs/src/reference/time_window_associative_op.md → docs/src/reference/time.md

@@ -7,5 +7,5 @@ The window is taken to be fixed in terms of time duration rather than a fixed nu
 
 ```@autodocs
 Modules = [AssociativeWindowAggregation]
-Pages = ["time_window_associative_op.jl"]
+Pages = ["time.jl"]
 ```

examples/Project.toml

@@ -1,5 +1,8 @@
 [deps]
 AssociativeWindowAggregation = "444271a7-5434-4a02-b82b-0e30a9223c60"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e"
+OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
 RollingFunctions = "b0e4dd01-7b14-53d8-9b45-175a3e362653"

examples/benchmark.jl

@@ -15,10 +15,7 @@ function run_example(window::Integer, data::Vector{Int})
     return result
 end
 
-function run_rolling(window::Integer, data::Vector{Int})
-    return rolling(sum, data, window)
-end
-
+run_rolling(window::Integer, data::Vector{Int}) = rolling(sum, data, window)
 
 data = rand(-100:100, 100000)
 window = 200

examples/online_stats.jl

@@ -0,0 +1,37 @@
+using AssociativeWindowAggregation
+using OnlineStatsBase
+using Plots
+using Statistics
+
+struct WindowedStat{Stat} <: OnlineStat{Stat}
+    state::FixedWindowAssociativeOp{Stat,merge,merge!}
+
+    function WindowedStat{Stat}(window::Int) where {Stat<:OnlineStat}
+        return new{Stat}(FixedWindowAssociativeOp{Stat,merge,merge!}(window))
+    end
+end
+
+function OnlineStatsBase.fit!(x::WindowedStat{Stat}, v) where {Stat}
+    # Create a single-observation instance of the statistic.
+    wrapped = Stat()
+    fit!(wrapped, v)
+    update_state!(x.state, wrapped)
+    return x
+end
+
+OnlineStatsBase.value(x::WindowedStat) = window_value(x.state)
+OnlineStatsBase.nobs(x::WindowedStat) = window_size(x.state)
+
+function Base.show(io::IO, o::WindowedStat{Stat}) where {Stat}
+    nobs(o) == 0 && return print(io, "WindowStat{$Stat}()")
+    return invoke(show, Tuple{IO,OnlineStat}, io, o)
+end
+
+x = WindowedStat{Variance}(100)
+values = rand(1000) .- 0.5
+output = [std(value(fit!(x, v))) for v in values]
+
+begin
+    plot(values; alpha=0.4, label="Input")
+    plot!(output; label="std (window=100)")
+end

src/AssociativeWindowAggregation.jl

@@ -3,8 +3,8 @@ module AssociativeWindowAggregation
 export FixedWindowAssociativeOp, TimeWindowAssociativeOp, WindowedAssociativeOp
 export update_state!, window_full, window_size, window_value
 
-include("windowed_associative_op.jl")
-include("fixed_window_associative_op.jl")
-include("time_window_associative_op.jl")
+include("base.jl")
+include("fixed.jl")
+include("time.jl")
 
 end # module