diff --git a/test/Project.toml b/test/Project.toml index 5305826..4ac3033 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,7 +1,9 @@ [deps] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] diff --git a/test/examples/fmnist.jl b/test/examples/fmnist.jl index fbe122c..a86ddaa 100644 --- a/test/examples/fmnist.jl +++ b/test/examples/fmnist.jl @@ -1,29 +1,34 @@ using MLDatasets: FashionMNIST -function test_fmnist(α::Float64 = 0.75) +function test_fmnist(α::Float64 = 0.75, δ::Float64 = 1E-2, k::Integer = 5) trainset = FashionMNIST(Int, :train) testset = FashionMNIST(Int, :test) n = length(trainset) x = [trainset[i][:features] for i = 1:n] - y = [trainset[i][:targets] for i = 1:n] + y = [trainset[i][:targets] for i = 1:n] n̂ = length(testset) x̂ = [testset[i][:features] for i = 1:n̂] - ŷ = [testset[i][:targets] for i = 1:n̂] + ŷ = [testset[i][:targets] for i = 1:n̂] @testset "FashionMNIST ≥$α" begin - wnn = WNN{Int,UInt64}(28, 28) + α⃗ = Vector{Float64}(undef, k) - WiSARD.classhint!(wnn, collect(0:9)) + for j = 1:k + wnn = WNN{Int,UInt64}(28, 28) - train!.(wnn, x, y) + train!.(wnn, x, y) - ȳ = classify.(wnn, x̂) - ᾱ = WiSARD.accuracy(ŷ, ȳ) - - @info "ᾱ = $ᾱ @ FashionMNIST" + ȳ = classify.(wnn, x̂) + α⃗[j] = WiSARD.accuracy(ŷ, ȳ) + end + + ᾱ, δ̄ = mean(α⃗), std(α⃗) + + @info @sprintf("ᾱ = %.2f ± %.2f @ FashionMNIST", ᾱ, δ̄) @test ᾱ >= α + @test δ̄ <= δ end return nothing diff --git a/test/examples/iris.jl b/test/examples/iris.jl index 651e75d..5e31473 100644 --- a/test/examples/iris.jl +++ b/test/examples/iris.jl @@ -1,17 +1,16 @@ using DataFrames using MLDatasets: Iris -function test_iris(α::Float64 = 0.85, β::Float64 = 6/7) - dataset = Iris(; as_df=true) +function test_iris(α::Float64 = 0.85, β::Float64 = 6 / 7, δ::Float64 = 1E-1, k::Integer = 10) + dataset = Iris(; as_df = true) N, M = size(dataset.features) a = [minimum(dataset.features[!, i]) for i = 1:M] b = [maximum(dataset.features[!, i]) for i = 1:M] n = trunc(Int, N * β) n̂ = N - n - i = shuffle(1:N) - trainset = dataset[i[begin:n]] - testset = dataset[i[n+1:end]] + trainset = dataset[n̂+1:N] + testset = dataset[1:n̂] x = [collect(Float64, trainset[:features][i, :]) for i = 1:n] y = collect(String, trainset[:targets][!, 1]) @@ -27,17 +26,22 @@ function test_iris(α::Float64 = 0.85, β::Float64 = 6/7) ẑ = τ.(x̂) @testset "Iris ≥$α" begin - wnn = WNN{String,UInt64}(8, 4) + α⃗ = Vector{Float64}(undef, k) - # WiSARD.classhint!(wnn, collect(0:9)) + for j = 1:k + wnn = WNN{String,UInt64}(8, 4; seed=j) - train!.(wnn, z, y) + train!.(wnn, z, y) - ȳ = classify.(wnn, ẑ) - ᾱ = WiSARD.accuracy(ŷ, ȳ) - - @info "ᾱ = $ᾱ @ Iris" + ȳ = classify.(wnn, ẑ) + α⃗[j] = WiSARD.accuracy(ŷ, ȳ) + end + + ᾱ, δ̄ = mean(α⃗), std(α⃗) + + @info @sprintf("ᾱ = %.2f ± %.2f @ Iris", ᾱ, δ̄) @test ᾱ >= α + @test δ̄ <= δ end return nothing diff --git a/test/examples/mnist.jl b/test/examples/mnist.jl index 7f80656..a589a15 100644 --- a/test/examples/mnist.jl +++ b/test/examples/mnist.jl @@ -1,6 +1,6 @@ using MLDatasets: MNIST -function test_mnist(α::Float64 = 0.9) +function test_mnist(α::Float64 = 0.9, δ::Float64 = 1E-2, k::Integer = 5) trainset = MNIST(Int, :train) testset = MNIST(Int, :test) @@ -13,17 +13,24 @@ function test_mnist(α::Float64 = 0.9) ŷ = [testset[i][:targets] for i = 1:n̂] @testset "MNIST ≥$α" begin - wnn = WNN{Int,UInt64}(28, 28) + α⃗ = Vector{Float64}(undef, k) - WiSARD.classhint!(wnn, collect(0:9)) + for j = 1:k + wnn = WNN{Int,UInt64}(28, 28) - train!.(wnn, x, y) + WiSARD.classhint!(wnn, collect(0:9)) - ȳ = classify.(wnn, x̂) - ᾱ = WiSARD.accuracy(ŷ, ȳ) - - @info "ᾱ = $ᾱ @ MNIST" + train!.(wnn, x, y) + + ȳ = classify.(wnn, x̂) + α⃗[j] = WiSARD.accuracy(ŷ, ȳ) + end + + ᾱ, δ̄ = mean(α⃗), std(α⃗) + + @info @sprintf("ᾱ = %.2f ± %.2f @ MNIST", ᾱ, δ̄) @test ᾱ >= α + @test δ̄ <= δ end return nothing diff --git a/test/examples/xyz.jl b/test/examples/xyz.jl index 8e3590b..6512f85 100644 --- a/test/examples/xyz.jl +++ b/test/examples/xyz.jl @@ -1,4 +1,4 @@ -function test_xyz(α::Float64=1.0) +function test_xyz(α::Float64=1.0, δ::Float64=0.0, k::Integer = 10) x = x̂ = [ [ 1 0 1 @@ -23,15 +23,22 @@ function test_xyz(α::Float64=1.0) y = ŷ = [:x, :y, :z] @testset "XYZ ≥$α" begin - wnn = WNN{Symbol,UInt}(3, 4) + α⃗ = Vector{Float64}(undef, k) - train!.(wnn, x, y) + for j = 1:k + wnn = WNN{Symbol,UInt64}(3, 4; seed=j) - ȳ = classify.(wnn, x̂) - ᾱ = WiSARD.accuracy(ŷ, ȳ) + train!.(wnn, x, y) - @info "ᾱ = $ᾱ @ XYZ" - @test ᾱ >= α + ȳ = classify.(wnn, x̂) + α⃗[j] = WiSARD.accuracy(ŷ, ȳ) + end + + ᾱ, δ̄ = mean(α⃗), std(α⃗) + + @info @sprintf("ᾱ = %.2f ± %.2f @ XYZ", ᾱ, δ̄) + @test ᾱ >= α + @test δ̄ <= δ end return nothing diff --git a/test/runtests.jl b/test/runtests.jl index 567c86e..c011da9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,8 @@ using Test +using Printf using Random using WiSARD +using Statistics # -*- Accept MLDatasets Downloads -*- # ENV["DATADEPS_ALWAYS_ACCEPT"] = true