Merge pull request #12 from pedromxavier/px/benchmark

Add benchmarks

.github/workflows/ci.yml

@@ -27,6 +27,8 @@ jobs:
           arch: ${{ matrix.arch }}
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
+      # - run: julia --project=benchmark --color=yes -e 'import Pkg; Pkg.add(path=@__DIR__); Pkg.instantiate()'
+      # - run: julia --project=benchmark --color=yes ./benchmark/runbenchmarks.jl
       # Future: Add Code Cov
       # - uses: julia-actions/julia-processcoverage@v1
       # - uses: codecov/codecov-action@v1

Project.toml

@@ -1,7 +1,7 @@
 name = "WiSARD"
 uuid = "2b71b7a6-5a26-4a05-9b8d-679d5aa759d4"
 authors = ["pedromxavier <[email protected]>"]
-version = "0.2.0"
+version = "0.3.0"
 
 [deps]
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

benchmark/.gitignore

@@ -0,0 +1,5 @@
+# Benchmark results
+results.json
+
+# Benchmark Tunings
+tune.json

benchmark/Project.toml

@@ -0,0 +1,5 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+WiSARD = "2b71b7a6-5a26-4a05-9b8d-679d5aa759d4"

benchmark/benchmarks.jl

@@ -0,0 +1,10 @@
+using BenchmarkTools
+using WiSARD
+using MLDatasets
+
+const SUITE = BenchmarkGroup()
+
+# -*- MNIST -*- #
+include("suites/mnist.jl")
+
+mnist_benchmark!(SUITE)

benchmark/runbenchmarks.jl

@@ -0,0 +1,19 @@
+import Pkg
+Pkg.add(path=joinpath(@__DIR__, ".."))
+Pkg.instantiate()
+
+using WiSARD
+using PkgBenchmark
+
+const RESULTS_PATH = joinpath(@__DIR__, "results")
+const RESULTS_FILE = joinpath(RESULTS_PATH, "results.json")
+
+function main()
+    results = benchmarkpkg(WiSARD)
+
+    writeresults(RESULTS_FILE, results)
+
+    return nothing
+end
+
+main() # Here we go!

benchmark/suites/mnist.jl

@@ -0,0 +1,26 @@
+function mnist_benchmark!(SUITE::BenchmarkGroup)
+    mnist_wnn = () -> WiSARD.WNN{Int,UInt}(28, 28)
+
+    SUITE["mnist"] = BenchmarkGroup(["MNIST"])
+
+    mnist_trainset = MLDatasets.MNIST(Int, :train)
+    mnist_train_x  = [mnist_trainset[i][:features] for i = 1:60_000]
+    mnist_train_y  = [mnist_trainset[i][:targets] for i = 1:60_000]
+
+    SUITE["mnist"]["train"] = @benchmarkable WiSARD.train!.(
+        $mnist_wnn(),
+        $mnist_train_x,
+        $mnist_train_y,
+    )
+
+    mnist_testset = MLDatasets.MNIST(Int, :test)
+    mnist_test_x  = [mnist_testset[i][:features] for i = 1:10_000]
+    # mnist_test_y  = [mnist_testset[i][:targets] for i = 1:10_000]
+
+    SUITE["mnist"]["test"] = @benchmarkable WiSARD.classify.(
+        $mnist_wnn(),
+        $mnist_test_x,
+    )
+
+    return nothing
+end

src/WiSARD.jl

@@ -2,10 +2,9 @@ module WiSARD
 
 using Random
 
+include("interface.jl")
 include("model/model.jl")
-
 include("encoding/encoding.jl")
-
 include("images/images.jl")
 
 end # module

src/images/images.jl

@@ -1,14 +1,14 @@
-function address(wnn::WNN{S, T}, k::T) where {S, T}
+function address(wnn::WNN{S,T}, k::T) where {S,T}
     [j for j = 1:wnn.d if (k >> (j - 1)) % 2 == 1]
 end
 
 function images(
     N::Type{<:Any},
-    wnn::WNN{S, T},
+    wnn::WNN{S,T},
     y::S;
-    w::Union{Int, Nothing} = nothing,
-    h::Union{Int, Nothing} = nothing,
-    ) where {S, T}
+    w::Union{Int,Nothing} = nothing,
+    h::Union{Int,Nothing} = nothing,
+) where {S,T<:WNNINT}
     w, h = if isnothing(w) && isnothing(h)
         1, wnn.n * wnn.d
     elseif isnothing(w) || isnothing(h)
@@ -23,42 +23,39 @@ function images(
     M = 0
 
     for i = 1:wnn.n
+        δ = wnn.n * (i - 1)
         for (k, v) in cls[i]
             for j in address(wnn, k)
-                M = max(M, img[wnn.map[(i - 1) * wnn.n + j]] += v)
+                M = max(M, img[wnn.map[δ+j]] += v)
             end
         end
     end
 
-    return if M == 0
-        N[img[(i - 1) * w + j] for i=1:h, j=1:w]
+    if iszero(M)
+        return reshape(img, h, w)
     else
-        N[img[(i - 1) * w + j] / M for i=1:h, j=1:w]
+        return reshape(img, h, w) / M
     end
 end
 
 function images(
-    wnn::WNN{S, T},
+    wnn::WNN{S,T},
     y::S;
-    w::Union{Int, Nothing} = nothing,
-    h::Union{Int, Nothing} = nothing,
-    ) where{S, T}
-    images(Float64, wnn, y; w=w, h=h)
+    w::Union{Int,Nothing} = nothing,
+    h::Union{Int,Nothing} = nothing,
+) where {S,T}
+    images(Float64, wnn, y; w = w, h = h)
 end
 
 function images(
     N::Type{<:Any},
-    wnn::WNN{S, T};
-    w::Union{Int, Nothing} = nothing,
-    h::Union{Int, Nothing} = nothing,
-    ) where {S, T}
-    Dict{S, Array{N}}(y => images(N, wnn, y; w=w, h=h) for y in keys(wnn.cls))
+    wnn::WNN{S,T};
+    w::Union{Int,Nothing} = nothing,
+    h::Union{Int,Nothing} = nothing,
+) where {S,T}
+    Dict{S,Array{N}}(y => images(N, wnn, y; w = w, h = h) for y in keys(wnn.cls))
 end
 
-function images(
-    wnn::WNN;
-    w::Union{Int, Nothing} = nothing,
-    h::Union{Int, Nothing} = nothing,
-    )
-    images(Float64, wnn; w=w, h=h)
+function images(wnn::WNN; w::Union{Int,Nothing} = nothing, h::Union{Int,Nothing} = nothing)
+    images(Float64, wnn; w = w, h = h)
 end

src/interface.jl

@@ -0,0 +1,17 @@
+@doc raw"""
+""" function train! end
+
+@doc raw"""
+""" function classhint! end
+
+@doc raw"""
+""" function classify end
+
+@doc raw"""
+""" function address end
+
+@doc raw"""
+""" function images end
+
+@doc raw"""
+""" function deaddress end