Update Tests

pedromxavier · Oct 27, 2022 · 2715ac7 · 2715ac7 · pedromxavier · Oct 27, 2022
1 parent 564f4a1
commit 2715ac7
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 37 deletions.
diff --git a/test/Project.toml b/test/Project.toml
@@ -1,7 +1,9 @@
 [deps]
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]

diff --git a/test/examples/fmnist.jl b/test/examples/fmnist.jl
@@ -1,29 +1,34 @@
 using MLDatasets: FashionMNIST
 
-function test_fmnist(α::Float64 = 0.75)
+function test_fmnist(α::Float64 = 0.75, δ::Float64 = 1E-2, k::Integer = 5)
     trainset = FashionMNIST(Int, :train)
     testset  = FashionMNIST(Int, :test)
 
     n = length(trainset)
     x = [trainset[i][:features] for i = 1:n]
-    y = [trainset[i][:targets]  for i = 1:n]
+    y = [trainset[i][:targets] for i = 1:n]
 
     n̂ = length(testset)
     x̂ = [testset[i][:features] for i = 1:n̂]
-    ŷ = [testset[i][:targets]  for i = 1:n̂]
+    ŷ = [testset[i][:targets] for i = 1:n̂]
 
     @testset "FashionMNIST ≥$α" begin
-        wnn = WNN{Int,UInt64}(28, 28)
+        α⃗ = Vector{Float64}(undef, k)
 
-        WiSARD.classhint!(wnn, collect(0:9))
+        for j = 1:k
+            wnn = WNN{Int,UInt64}(28, 28)
 
-        train!.(wnn, x, y)
+            train!.(wnn, x, y)
 
-        ȳ = classify.(wnn, x̂)
-        ᾱ = WiSARD.accuracy(ŷ, ȳ)
-
-        @info "ᾱ = $ᾱ @ FashionMNIST"
+            ȳ    = classify.(wnn, x̂)
+            α⃗[j] = WiSARD.accuracy(ŷ, ȳ)
+        end
+
+        ᾱ, δ̄ = mean(α⃗), std(α⃗)
+
+        @info @sprintf("ᾱ = %.2f ± %.2f @ FashionMNIST", ᾱ, δ̄)
         @test ᾱ >= α
+        @test δ̄ <= δ
     end
 
     return nothing

diff --git a/test/examples/iris.jl b/test/examples/iris.jl
@@ -1,17 +1,16 @@
 using DataFrames
 using MLDatasets: Iris
 
-function test_iris(α::Float64 = 0.85, β::Float64 = 6/7)
-    dataset = Iris(; as_df=true)
+function test_iris(α::Float64 = 0.85, β::Float64 = 6 / 7, δ::Float64 = 1E-1, k::Integer = 10)
+    dataset = Iris(; as_df = true)
     N, M    = size(dataset.features)
     a       = [minimum(dataset.features[!, i]) for i = 1:M]
     b       = [maximum(dataset.features[!, i]) for i = 1:M]
     n       = trunc(Int, N * β)
     n̂       = N - n
-    i       = shuffle(1:N)
 
-    trainset = dataset[i[begin:n]]
-    testset  = dataset[i[n+1:end]]
+    trainset = dataset[n̂+1:N]
+    testset  = dataset[1:n̂]
 
     x = [collect(Float64, trainset[:features][i, :]) for i = 1:n]
     y = collect(String, trainset[:targets][!, 1])
@@ -27,17 +26,22 @@ function test_iris(α::Float64 = 0.85, β::Float64 = 6/7)
     ẑ = τ.(x̂)
 
     @testset "Iris ≥$α" begin
-        wnn = WNN{String,UInt64}(8, 4)
+        α⃗ = Vector{Float64}(undef, k)
 
-        # WiSARD.classhint!(wnn, collect(0:9))
+        for j = 1:k
+            wnn = WNN{String,UInt64}(8, 4; seed=j)
 
-        train!.(wnn, z, y)
+            train!.(wnn, z, y)
 
-        ȳ = classify.(wnn, ẑ)
-        ᾱ = WiSARD.accuracy(ŷ, ȳ)
-
-        @info "ᾱ = $ᾱ @ Iris"
+            ȳ    = classify.(wnn, ẑ)
+            α⃗[j] = WiSARD.accuracy(ŷ, ȳ)
+        end
+
+        ᾱ, δ̄ = mean(α⃗), std(α⃗)
+
+        @info @sprintf("ᾱ = %.2f ± %.2f @ Iris", ᾱ, δ̄)
         @test ᾱ >= α
+        @test δ̄ <= δ
     end
 
     return nothing

diff --git a/test/examples/mnist.jl b/test/examples/mnist.jl
@@ -1,6 +1,6 @@
 using MLDatasets: MNIST
 
-function test_mnist(α::Float64 = 0.9)
+function test_mnist(α::Float64 = 0.9, δ::Float64 = 1E-2, k::Integer = 5)
     trainset = MNIST(Int, :train)
     testset  = MNIST(Int, :test)
 
@@ -13,17 +13,24 @@ function test_mnist(α::Float64 = 0.9)
     ŷ = [testset[i][:targets] for i = 1:n̂]
 
     @testset "MNIST ≥$α" begin
-        wnn = WNN{Int,UInt64}(28, 28)
+        α⃗ = Vector{Float64}(undef, k)
 
-        WiSARD.classhint!(wnn, collect(0:9))
+        for j = 1:k
+            wnn = WNN{Int,UInt64}(28, 28)
 
-        train!.(wnn, x, y)
+            WiSARD.classhint!(wnn, collect(0:9))
 
-        ȳ = classify.(wnn, x̂)
-        ᾱ = WiSARD.accuracy(ŷ, ȳ)
-
-        @info "ᾱ = $ᾱ @ MNIST"
+            train!.(wnn, x, y)
+
+            ȳ    = classify.(wnn, x̂)
+            α⃗[j] = WiSARD.accuracy(ŷ, ȳ)
+        end 
+
+        ᾱ, δ̄ = mean(α⃗), std(α⃗)
+
+        @info @sprintf("ᾱ = %.2f ± %.2f @ MNIST", ᾱ, δ̄)
         @test ᾱ >= α
+        @test δ̄ <= δ
     end
 
     return nothing

diff --git a/test/examples/xyz.jl b/test/examples/xyz.jl
@@ -1,4 +1,4 @@
-function test_xyz(α::Float64=1.0)
+function test_xyz(α::Float64=1.0, δ::Float64=0.0, k::Integer = 10)
     x = x̂ = [
         [
             1 0 1
@@ -23,15 +23,22 @@ function test_xyz(α::Float64=1.0)
     y = ŷ = [:x, :y, :z]
 
     @testset "XYZ ≥$α" begin
-        wnn = WNN{Symbol,UInt}(3, 4)
+        α⃗ = Vector{Float64}(undef, k)
 
-        train!.(wnn, x, y)
+        for j = 1:k
+            wnn = WNN{Symbol,UInt64}(3, 4; seed=j)
 
-        ȳ = classify.(wnn, x̂)
-        ᾱ = WiSARD.accuracy(ŷ, ȳ)
+            train!.(wnn, x, y)
 
-        @info "ᾱ = $ᾱ @ XYZ"
-        @test ᾱ >= α 
+            ȳ    = classify.(wnn, x̂)
+            α⃗[j] = WiSARD.accuracy(ŷ, ȳ)
+        end
+
+        ᾱ, δ̄ = mean(α⃗), std(α⃗)
+
+        @info @sprintf("ᾱ = %.2f ± %.2f @ XYZ", ᾱ, δ̄)
+        @test ᾱ >= α
+        @test δ̄ <= δ
     end
 
     return nothing

diff --git a/test/runtests.jl b/test/runtests.jl
@@ -1,6 +1,8 @@
 using Test
+using Printf
 using Random
 using WiSARD
+using Statistics
 
 # -*- Accept MLDatasets Downloads -*- #
 ENV["DATADEPS_ALWAYS_ACCEPT"] = true