From 3933b3235e5ece42f8ad27288396eb1a717f7f49 Mon Sep 17 00:00:00 2001 From: JonasIsensee Date: Tue, 2 Jan 2024 16:36:03 +0100 Subject: [PATCH] fix vararg tuples (#519) * fix vararg tuples * make tests pass * fix opaquedata * version & changelog --- CHANGELOG.md | 4 + Project.toml | 2 +- src/data/specialcased_types.jl | 8 +- src/data/writing_datatypes.jl | 21 ++- test/loadsave.jl | 14 +- test/test_files.jl | 226 ++++++++++++++++----------------- 6 files changed, 149 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d3df66b..36bfc224 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.4.41 + - fix ntuple type with typevar length + - fix OpaqueData test (HDF5 compat) + ## 0.4.40 - fix unitialized custom-serialized objects - allow serializing pkg-modules by name diff --git a/Project.toml b/Project.toml index 01cd6d94..c6472873 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "JLD2" uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -version = "0.4.40" +version = "0.4.41" [deps] FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" diff --git a/src/data/specialcased_types.jl b/src/data/specialcased_types.jl index f9590272..5760cf8b 100644 --- a/src/data/specialcased_types.jl +++ b/src/data/specialcased_types.jl @@ -3,8 +3,8 @@ struct OpaqueData{N} data::Vector{UInt8} OpaqueData(data) = new{length(data)}(data) end -odr_sizeof(x::Type{OpaqueData{N}}) where {N} = UInt32(N) -function jlconvert(rr::ReadRepresentation{OpaqueData{N}, Vector{UInt8}}, ::JLDFile, ptr::Ptr, ::RelOffset) where N + +function jlconvert(rr::ReadRepresentation{OpaqueData{N}, NTuple{N,UInt8}}, ::JLDFile, ptr::Ptr, ::RelOffset) where N data = Vector{UInt8}(undef, N) unsafe_copyto!(pointer(data), convert(Ptr{UInt8}, ptr), N) OpaqueData(data) @@ -62,7 +62,7 @@ function jltype(f::JLDFile, dt::BasicDatatype) throw(UnsupportedFeatureException("Encountered an unsupported string type. $dt")) end elseif dt.class << 4 == DT_OPAQUE << 4 - return ReadRepresentation{OpaqueData{Int(dt.size)},Vector{UInt8}}() + return ReadRepresentation{OpaqueData{Int(dt.size)},NTuple{Int(dt.size),UInt8}}() else throw(UnsupportedFeatureException("Encountered an unsupported type.")) @@ -77,7 +77,7 @@ function jltype(f::JLDFile, dt::BasicDatatype) throw(UnsupportedFeatureException("Encountered an unsupported string type.")) end elseif dt.class << 4 == DT_OPAQUE << 4 - return ReadRepresentation{OpaqueData{Int(dt.size)},Vector{UInt8}}() + return ReadRepresentation{OpaqueData{Int(dt.size)},NTuple{Int(dt.size),UInt8}}() elseif dt.class << 4 == DT_REFERENCE << 4 return ReadRepresentation{Any,RelOffset}() else diff --git a/src/data/writing_datatypes.jl b/src/data/writing_datatypes.jl index 6b6e1f35..e0b06276 100644 --- a/src/data/writing_datatypes.jl +++ b/src/data/writing_datatypes.jl @@ -428,21 +428,28 @@ end # This is a trick to compactly write long NTuple -# This uses that NTuple{N,T} == Tuple{T,T,T,T,...,T} -function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::Type{NTuple{N,ET}}, wsession::JLDWriteSession) where {N, ET} - if isempty(T.parameters) +# This uses that NTuple{N,T} === Tuple{T,T,T,T,...,T} +function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::Type{<: NTuple}, wsession::JLDWriteSession) + params = T.parameters + N = length(params) + if N ≤ 1 store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, "Tuple"), f.datatype_wsession) - h5convert_uninitialized!(out+odr_sizeof(Vlen{UInt8}), Vlen{UInt8}) - else + if N == 0 + h5convert_uninitialized!(out+odr_sizeof(Vlen{UInt8}), Vlen{UInt8}) + else # N==1 + # this also catches NTuples with indeterminate length + refs = refs_from_types(f, params, wsession) + store_vlen!(out+odr_sizeof(Vlen{UInt8}), RelOffset, f, refs, f.datatype_wsession) + end + else # actual NTuple with more than one entry store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, "NTuple"), f.datatype_wsession) + ET = params[1] # T === Tuple{ET,ET,ET,...} refs = refs_from_types(f, Any[N,ET], wsession) store_vlen!(out+odr_sizeof(Vlen{UInt8}), RelOffset, f, refs, f.datatype_wsession) end nothing end - - ## Union Types const H5TYPE_UNION = CompoundDatatype( diff --git a/test/loadsave.jl b/test/loadsave.jl index 725c43c3..2a76b5b7 100644 --- a/test/loadsave.jl +++ b/test/loadsave.jl @@ -701,5 +701,17 @@ end o = load_object("test.jld2") @test !any(isassigned.(Ref(o), eachindex(o))) end - end + +@testset "Issue 486 store NTuple type with indeterminate length" begin + cd(mktempdir()) do + s_type = Tuple{Int, Tuple{Vararg{Int, T}} where T} + a = Dict{s_type, Int}() + a[(0, (1, 2, 3))] = 4 + if VERSION < v"1.7.0-A" + @test_broken a == (save_object("test.jld2", a); load_object("test.jld2")) + else + @test a == (save_object("test.jld2", a); load_object("test.jld2")) + end + end +end diff --git a/test/test_files.jl b/test/test_files.jl index 423bf4b2..7cacac3c 100644 --- a/test/test_files.jl +++ b/test/test_files.jl @@ -6,120 +6,120 @@ testfiles = artifact"testfiles/JLD2TestFiles-0.1.0/artifacts" @testset "HDF5 compat test files" begin # These are test files copied from the HDF5.jl test suite - cd(testfiles) do - - fn = "compound.h5" - jldopen(fn) do f - data = f["data"] - @test data[1] == data[2] - nt = data[1] - @test nt.wgt == 1.0 - @test nt.xyz == [-2.4559041161056125, 0.43236207188504794, -0.5088338908493437] - @test nt.uvw == [-0.44966656055677057, 0.6453930541533174, 0.6174688574881305] - @test nt.E == 1.1915731810042547 - end - - # Should return some enum type and load names correctly - fn = "h5ex_t_enum.h5" - jldopen(fn) do f - @test size(f["DS1"]) == (7,4) - end - - fn = "h5ex_t_array.h5" - jldopen(fn) do f - @test f["DS1"][1] == (0:-1:-4) .* [0,1,2]' - @test f["DS1"][2] == hcat(collect(0:4), ones(Int,5), collect(2:-1:-2)) - end - - fn = "h5ex_t_float.h5" - jldopen(fn) do f - @test size(f["DS1"]) == (7,4) - @test f["DS1"][9] ≈ 5/3 - end - - # Big Endian Integers are not implemented - fn = "h5ex_t_int.h5" - jldopen(fn) do f - @test f["DS1"] == [0:-1:-6 zeros(Int,7) 0:6 0:2:12] - end - - fn = "h5ex_t_objref.h5" - jldopen(fn) do f - @test f["DS1"][1] === f["G1"] - @test f["DS1"][2] === f["DS2"] - end - - fn = "h5ex_t_opaque.h5" - jldopen(fn) do f - @test f["DS1"][4].data == [0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x30] - end - - fn = "h5ex_t_string.h5" - jldopen(fn) do f - @test f["DS1"] == ["Parting", "is such", "sweet", "sorrow."] - end - - fn = "h5ex_t_vlen.h5" - jldopen(fn) do f - @test f["DS1"][1] == [3, 2, 1] - @test f["DS1"][2] == [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144] - end - - fn = "h5ex_t_vlstring.h5" - jldopen(fn) do f - @test f["DS1"] == ["Parting", "is such", "sweet", "sorrow."] - end - - fn = "nullterm_ascii.h5" - jldopen(fn) do f - @test f["test"] == "Hello World" - end - - fn = "large_fractal_heap.h5" - jldopen(fn) do f - @test length(keys(f)) == 200000 - end - - fn = "netcdf.nc" - jldopen(fn) do f - @test f["hello"] == ones(5) - #@test_broken f["x"] - #@test_broken f["z"] - @test f["grouped/data"] == 0:9 - #@test_broken f["grouped/y"] - end - - fn = "simple.nc" - jldopen(fn) do f - @test f["dim1"] == [2, 4, 6] - @test f["dim2"] == ["a", "b", "c", "d"] - @test f["mydata"] == Matrix(reshape(1:12, 4, 3)) - JLD2.load_attributes(f, "dim1") # not sure what to test for. just not erroring so far - JLD2.load_attributes(f, "dim2") - JLD2.load_attributes(f, "mydata") - end - - # julia> using JLD - # julia> struct A; x::Int; y::Float64; z::String; end - # julia> save("jldstruct.jld", "a", A(1,2.0,"3")) - fn = "jldstruct.jld" - jldopen(fn) do f - a = f["a"] - @test a.x == 1 - @test a.y == 2.0 - @test a.z == "3" - end - - fn = "chunking1.h5" - jldopen(fn) do f - @test f["uncompressed_chunks"] == reshape(1:1000., 25, 40) - @test f["compressed_chunks"] == reshape(1:1000., 25, 40) - @test f["shuffle_compressed_chunks"] == reshape(1:1000, 25, 40) - @test size(f["incomplete_allocation"]) == (50,50,10) - @test f["incomplete_allocation"][1:50,1:50, 2] == reshape(1:2500, 50,50) - #f["incomplete_allocation"][1,1,1] == 0 - end + + fn = joinpath(testfiles,"compound.h5") + jldopen(fn) do f + data = f["data"] + @test data[1] == data[2] + nt = data[1] + @test nt.wgt == 1.0 + @test nt.xyz == [-2.4559041161056125, 0.43236207188504794, -0.5088338908493437] + @test nt.uvw == [-0.44966656055677057, 0.6453930541533174, 0.6174688574881305] + @test nt.E == 1.1915731810042547 + end + + # Should return some enum type and load names correctly + fn = joinpath(testfiles,"h5ex_t_enum.h5") + jldopen(fn) do f + @test size(f["DS1"]) == (7,4) + end + + fn = joinpath(testfiles,"h5ex_t_array.h5") + jldopen(fn) do f + @test f["DS1"][1] == (0:-1:-4) .* [0,1,2]' + @test f["DS1"][2] == hcat(collect(0:4), ones(Int,5), collect(2:-1:-2)) + end + + fn = joinpath(testfiles,"h5ex_t_float.h5") + jldopen(fn) do f + @test size(f["DS1"]) == (7,4) + @test f["DS1"][9] ≈ 5/3 + end + + # Big Endian Integers are not implemented + fn = joinpath(testfiles,"h5ex_t_int.h5") + jldopen(fn) do f + @test f["DS1"] == [0:-1:-6 zeros(Int,7) 0:6 0:2:12] + end + + fn = joinpath(testfiles,"h5ex_t_objref.h5") + jldopen(fn) do f + @test f["DS1"][1] === f["G1"] + @test f["DS1"][2] === f["DS2"] + end + + fn = joinpath(testfiles,"h5ex_t_opaque.h5") + jldopen(fn) do f + @test f["DS1"][1].data == [0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x30] + @test f["DS1"][2].data == [0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x31] + @test f["DS1"][3].data == [0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x32] + @test f["DS1"][4].data == [0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x33] + end + + fn = joinpath(testfiles,"h5ex_t_string.h5") + jldopen(fn) do f + @test f["DS1"] == ["Parting", "is such", "sweet", "sorrow."] + end + + fn = joinpath(testfiles,"h5ex_t_vlen.h5") + jldopen(fn) do f + @test f["DS1"][1] == [3, 2, 1] + @test f["DS1"][2] == [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144] + end + + fn = joinpath(testfiles,"h5ex_t_vlstring.h5") + jldopen(fn) do f + @test f["DS1"] == ["Parting", "is such", "sweet", "sorrow."] + end + + fn = joinpath(testfiles,"nullterm_ascii.h5") + jldopen(fn) do f + @test f["test"] == "Hello World" + end + + fn = joinpath(testfiles,"large_fractal_heap.h5") + jldopen(fn) do f + @test length(keys(f)) == 200000 + end + + fn = joinpath(testfiles,"netcdf.nc") + jldopen(fn) do f + @test f["hello"] == ones(5) + #@test_broken f["x"] + #@test_broken f["z"] + @test f["grouped/data"] == 0:9 + #@test_broken f["grouped/y"] + end + + fn = joinpath(testfiles,"simple.nc") + jldopen(fn) do f + @test f["dim1"] == [2, 4, 6] + @test f["dim2"] == ["a", "b", "c", "d"] + @test f["mydata"] == Matrix(reshape(1:12, 4, 3)) + JLD2.load_attributes(f, "dim1") # not sure what to test for. just not erroring so far + JLD2.load_attributes(f, "dim2") + JLD2.load_attributes(f, "mydata") + end + + # julia> using JLD + # julia> struct A; x::Int; y::Float64; z::String; end + # julia> save("jldstruct.jld", "a", A(1,2.0,"3")) + fn = joinpath(testfiles,"jldstruct.jld") + jldopen(fn) do f + a = f["a"] + @test a.x == 1 + @test a.y == 2.0 + @test a.z == "3" + end + fn = joinpath(testfiles,"chunking1.h5") + jldopen(fn) do f + @test f["uncompressed_chunks"] == reshape(1:1000., 25, 40) + @test f["compressed_chunks"] == reshape(1:1000., 25, 40) + @test f["shuffle_compressed_chunks"] == reshape(1:1000, 25, 40) + @test size(f["incomplete_allocation"]) == (50,50,10) + @test f["incomplete_allocation"][1:50,1:50, 2] == reshape(1:2500, 50,50) + #f["incomplete_allocation"][1,1,1] == 0 end end