struct upgrade pathway (#439)

* struct upgrade pathway * fix upgrade of array elements * remove Upgrade in type params * fix unnecessary conversion * tests, docs, and changelog Co-authored-by: Jonas Isensee <[email protected]>
JuliaIO · Nov 26, 2022 · 7ed395e · 7ed395e · JonasIsensee · Nov 26, 2022
1 parent 3a6a763
commit 7ed395e
Show file tree

Hide file tree

Showing 8 changed files with 114 additions and 10 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,6 @@
+## 0.4.29
+ - added `Upgrade` feature
+
 ## 0.4.28
  - compatibility to julia v1.9-dev (@eschnett)
 

diff --git a/Project.toml b/Project.toml
@@ -1,6 +1,6 @@
 name = "JLD2"
 uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
-version = "0.4.28"
+version = "0.4.29"
 
 [deps]
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"

diff --git a/docs/src/advanced.md b/docs/src/advanced.md
@@ -43,6 +43,37 @@ julia> f["a"]
 A_old(42)
 ```
 
+## Upgrading old structures on load
+The section above explains how you can make JLD2 load old structs with a different DataType name as target.
+A different method for loading old data is described here:
+
+```julia
+# This is the old version of the struct stored in the file
+struct OldStructVersion
+    x::Int
+    y::Float64
+end
+orig = OldStructVersion(1,2.0)
+jldsave("test.jld2"; data=orig)
+
+### new session
+
+# This is the new version of your struct
+struct UpdatedStruct
+    x::Float64 # no longer int
+    y::Float64
+    z::Float64 # = x*y
+end
+
+# When upgrading a struct, JLD2 will load the fields of the old struct into a `NamedTuple`
+# and call `rconvert` on it. Here we implement a conversion method that returns an `UpdatedStruct`
+JLD2.rconvert(::Type{UpdatedStruct}, nt::NamedTuple) = UpdatedStruct(Float64(nt.x), nt.y, nt.x*nt.y)
+
+# Here we provide the `typemap` keyword argument. It is a dictionary mapping the stored struct name
+# to an `Upgrade` instance with the new struct.
+load("test.jld2", "data"; typemap=Dict("Main.OldStructVersion" => JLD2.Upgrade(UpdatedStruct)))
+```
+
 ## Groups - Appending to files
 
 

diff --git a/docs/src/customserialization.md b/docs/src/customserialization.md
@@ -1,8 +1,4 @@
-# Custom Serialization (Experimental)
-
-Version `v0.3.0` of introduces support for custom serialization.
-For now this feature is considered experimental as it passes tests but 
-has little testing in the wild. → Please test and report if you encounter problems.
+# Custom Serialization
 
 The API is simple enough, to enable custom serialization for your type `A` you define
 a new type e.g. `ASerialization` that contains the fields you want to store and define

diff --git a/src/JLD2.jl b/src/JLD2.jl
@@ -145,6 +145,15 @@ read as type `S`.
 """
 struct CustomSerialization{T,S} end
 
+"""
+    Upgrade(T)
+
+Specify an upgrade path for serialized structs using the `typemap`` keyword argument
+and `rconvert`.
+"""
+struct Upgrade
+    target
+end
 
 struct Filter
     id::UInt16

diff --git a/src/data/reconstructing_datatypes.jl b/src/data/reconstructing_datatypes.jl
@@ -243,6 +243,20 @@ function constructrr(f::JLDFile, T::DataType, dt::CompoundDatatype,
     end
 end
 
+function constructrr(f::JLDFile, u::Upgrade, dt::CompoundDatatype,
+                     attrs::Vector{ReadAttribute},
+                     hard_failure::Bool=false)
+    field_datatypes = read_field_datatypes(f, attrs)
+
+
+    rodr = reconstruct_odr(f, dt, field_datatypes)
+    types = typeof(rodr).parameters[2].parameters
+
+    T2 = NamedTuple{tuple(dt.names...), typeof(rodr).parameters[2]}
+
+    return (ReadRepresentation{u.target, CustomSerialization{T2, rodr}}(), false)    
+end
+
 function constructrr(f::JLDFile, T::UnionAll, dt::CompoundDatatype,
                      attrs::Vector{ReadAttribute},
                      hard_failure::Bool=false)
@@ -328,7 +342,15 @@ function jlconvert(rr::ReadRepresentation{T,DataTypeODR()},
 
     params, unknown_params = types_from_refs(f, ptr+odr_sizeof(Vlen{UInt8}))
     # For cross-platform compatibility convert integer type parameters to system precision
-    params = [p isa Union{Int64,Int32} ? Int(p) : p for p in params]
+    params = map(params) do p
+        if p isa Union{Int64,Int32}
+            Int(p)
+        elseif p isa Upgrade
+            p.target
+        else
+            p
+        end
+    end
     hasparams = !isempty(params)
     mypath = String(jlconvert(ReadRepresentation{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE))
 

diff --git a/src/datasets.jl b/src/datasets.jl
@@ -266,10 +266,14 @@ function read_data(f::JLDFile,
                 if isa(T, UnknownType)
                     str = typestring(T)
                     @warn("type $(str) does not exist in workspace; interpreting Array{$str} as Array{Any}")
-                    T = Any
+                    rr = ReadRepresentation{Any,RelOffset}()
+                elseif T isa Upgrade
+                    rr = ReadRepresentation{T.target, RelOffset}()
+                else
+                    rr = ReadRepresentation{T, RelOffset}()
                 end
                 seek(io, startpos)
-                return read_array(f, dataspace, ReadRepresentation{T,RelOffset}(),
+                return read_array(f, dataspace, rr,
                                   layout, FilterPipeline(), header_offset, attributes)
             end
         end

diff --git a/test/loadsave.jl b/test/loadsave.jl
@@ -598,4 +598,43 @@ JLD2.rconvert(::Type{CR}, dsa::CRSerialized) = CR(dsa.r)
         t1 = load_object("kk.jld2")
         @test t1 === t1.r
     end
-end
+end
+
+###################################################################################################
+##             `Upgrade` Tests
+###################################################################################################
+
+struct OldStructVersion
+    x::Int
+    y::Float64
+end
+
+struct UpdatedStruct
+    x::Float64 # no longer int
+    y::Float64
+    z::Float64 # x*y
+end
+JLD2.rconvert(::Type{UpdatedStruct}, nt::NamedTuple) = UpdatedStruct(Float64(nt.x), nt.y, nt.x*nt.y)
+# Dummy Upgrade, keeps struct but changes values
+JLD2.rconvert(::Type{OldStructVersion}, nt::NamedTuple) = OldStructVersion(nt.x, 2*nt.y)
+
+
+@testset "Explicit `Upgrade`ing" begin
+    cd(mktempdir()) do
+        orig = OldStructVersion(1,2.0)
+        newver = UpdatedStruct(1.0, 2.0, 2.0)
+        save_object("test.jld2", orig)
+        @test orig == load_object("test.jld2")
+        @test newver == JLD2.load("test.jld2", "single_stored_object"; typemap=Dict("Main.OldStructVersion" => JLD2.Upgrade(UpdatedStruct)))
+
+        # Test for container with vector elements being upgraded
+        wrapped_orig = ([orig,],)
+        wrapped_newver = ([OldStructVersion(1,4.0),],)
+        save_object("test.jld2", wrapped_orig)
+        @test wrapped_orig == load_object("test.jld2")
+        @test wrapped_newver == JLD2.load("test.jld2", "single_stored_object"; typemap=Dict("Main.OldStructVersion" => JLD2.Upgrade(OldStructVersion)))
+    end
+end
+
+
+