From f1315af849b1093a0d27ef3816f6de029f3f6fa0 Mon Sep 17 00:00:00 2001 From: Andy Ferris Date: Mon, 24 Oct 2016 15:01:41 +1000 Subject: [PATCH 01/13] Added support for `Float16` * Saving and loading Float16 values is now supported * Float16 is no longer an "opaque" type. * Added some more wrapper functions relating to setting and getting floating-point datatype fields * `H5T_FLOAT16` needs to be created dynamically. Attempts at making the list of natively supported types dynamically modifiable by the user is blocked by the fact that `HDF5BitsKind` is used for dispatch everywhere... * Unit test --- src/HDF5.jl | 70 +++++++++++++++++++++++++++++++++++++++++++-------- test/plain.jl | 6 +++++ 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/src/HDF5.jl b/src/HDF5.jl index 935111143..e0b1a9c7d 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -224,6 +224,16 @@ const H5T_NATIVE_DOUBLE = read_const(:H5T_NATIVE_DOUBLE_g) # Library versions const H5F_LIBVER_EARLIEST = 0 const H5F_LIBVER_LATEST = 1 +# Constructed types (occurs at runtime) +function make_float16() + FLOAT16 = h5t_copy(H5T_NATIVE_FLOAT) + h5t_set_fields(FLOAT16, 15, 10, 5, 0, 10) + h5t_set_size(FLOAT16, 2) + h5t_set_ebias(FLOAT16, 15) + h5t_lock(FLOAT16) + return FLOAT16 +end +# const H5T_FLOAT16 = make_float16() (in `__init__()`) ## Conversion between Julia types and HDF5 atomic types hdf5_type_id(::Type{Int8}) = H5T_NATIVE_INT8 @@ -234,10 +244,11 @@ hdf5_type_id(::Type{Int32}) = H5T_NATIVE_INT32 hdf5_type_id(::Type{UInt32}) = H5T_NATIVE_UINT32 hdf5_type_id(::Type{Int64}) = H5T_NATIVE_INT64 hdf5_type_id(::Type{UInt64}) = H5T_NATIVE_UINT64 +#hdf5_type_id(::Type{Float16}) = H5T_FLOAT16 (in `__init__()`) hdf5_type_id(::Type{Float32}) = H5T_NATIVE_FLOAT hdf5_type_id(::Type{Float64}) = H5T_NATIVE_DOUBLE -const HDF5BitsKind = Union{Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Float32, Float64} +const HDF5BitsKind = Union{Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Float16, Float32, Float64} const BitsKindOrString = Union{HDF5BitsKind, String} # It's not safe to use particular id codes because these can change, so we use characteristics of the type. @@ -250,6 +261,7 @@ const hdf5_type_map = Dict( (H5T_INTEGER, H5T_SGN_NONE, convert(Csize_t, 2)) => UInt16, (H5T_INTEGER, H5T_SGN_NONE, convert(Csize_t, 4)) => UInt32, (H5T_INTEGER, H5T_SGN_NONE, convert(Csize_t, 8)) => UInt64, + (H5T_FLOAT, nothing, convert(Csize_t, 2)) => Float16, (H5T_FLOAT, nothing, convert(Csize_t, 4)) => Float32, (H5T_FLOAT, nothing, convert(Csize_t, 8)) => Float64, ) @@ -1830,17 +1842,30 @@ function hdf5_to_julia_eltype(objtype) error("character set ", cset, " not recognized") end elseif class_id == H5T_INTEGER || class_id == H5T_FLOAT - native_type = h5t_get_native_type(objtype.id) - try - native_size = h5t_get_size(native_type) - if class_id == H5T_INTEGER - is_signed = h5t_get_sign(native_type) - else - is_signed = nothing + # First look in the type last for a match + # otherwise fall back to a native datatype + # Allows for users to dynamically add types to the typemap + t_size = h5t_get_size(objtype) + if class_id == H5T_INTEGER + is_signed = h5t_get_sign(objtype) + else + is_signed = nothing # probably should include the mantissa size, etc... + end + if haskey(hdf5_type_map, (class_id, is_signed, t_size)) + T = hdf5_type_map[(class_id, is_signed, t_size)] + else + native_type = h5t_get_native_type(objtype.id) + try + native_size = h5t_get_size(native_type) + if class_id == H5T_INTEGER + is_signed = h5t_get_sign(native_type) + else + is_signed = nothing + end + T = hdf5_type_map[(class_id, is_signed, native_size)] + finally + h5t_close(native_type) end - T = hdf5_type_map[(class_id, is_signed, native_size)] - finally - h5t_close(native_type) end elseif class_id == H5T_ENUM super_type = h5t_get_super(objtype.id) @@ -2024,7 +2049,10 @@ for (jlname, h5name, outtype, argtypes, argsyms, msg) in (:h5s_select_hyperslab, :H5Sselect_hyperslab, Herr, (Hid, Cint, Ptr{Hsize}, Ptr{Hsize}, Ptr{Hsize}, Ptr{Hsize}), (:dspace_id, :seloper, :start, :stride, :count, :block), "Error selecting hyperslab"), (:h5t_commit, :H5Tcommit2, Herr, (Hid, Ptr{UInt8}, Hid, Hid, Hid, Hid), (:loc_id, :name, :dtype_id, :lcpl_id, :tcpl_id, :tapl_id), "Error committing type"), (:h5t_close, :H5Tclose, Herr, (Hid,), (:dtype_id,), "Error closing datatype"), + (:h5t_lock, :H5Tlock, Herr, (Hid,), (:dtype_id,), "Error locking datatype"), (:h5t_set_cset, :H5Tset_cset, Herr, (Hid, Cint), (:dtype_id, :cset), "Error setting character set in datatype"), + (:h5t_set_ebias, :H5Tset_ebias, Herr, (Hid, Csize_t), (:dtype_id, :ebias), "Error setting exponential bias of floating-point type"), + (:h5t_set_fields, :H5Tset_fields, Herr, (Hid, Csize_t, Csize_t, Csize_t, Csize_t, Csize_t), (:dtype_id, :spos, :epos, :esize, :mpos, :msize), "Error setting floating-point type fields"), (:h5t_set_size, :H5Tset_size, Herr, (Hid, Csize_t), (:dtype_id, :sz), "Error setting size of datatype"), ) @@ -2110,6 +2138,7 @@ for (jlname, h5name, outtype, argtypes, argsyms, ex_error) in (:h5t_get_array_ndims, :H5Tget_array_ndims, Cint, (Hid,), (:dtype_id,), :(error("Error getting ndims of array"))), (:h5t_get_class, :H5Tget_class, Cint, (Hid,), (:dtype_id,), :(error("Error getting class"))), (:h5t_get_cset, :H5Tget_cset, Cint, (Hid,), (:dtype_id,), :(error("Error getting character set encoding"))), + (:h5t_get_ebias, :H5Tget_ebias, Csize_t, (Hid,), (:dtype_id,), :(error("Error getting exponential bias"))), (:h5t_get_member_class, :H5Tget_member_class, Cint, (Hid, Cuint), (:dtype_id, :index), :(error("Error getting class of compound datatype member #", index))), (:h5t_get_member_index, :H5Tget_member_index, Cint, (Hid, Ptr{UInt8}), (:dtype_id, :membername), :(error("Error getting index of compound datatype member \"", membername, "\""))), (:h5t_get_member_offset, :H5Tget_member_offset, Csize_t, (Hid, Cuint), (:dtype_id, :index), :(error("Error getting offset of compound datatype member #", index))), @@ -2200,6 +2229,21 @@ function h5s_get_simple_extent_dims(space_id::Hid) h5s_get_simple_extent_dims(space_id, dims, maxdims) return tuple(reverse!(dims)...), tuple(reverse!(maxdims)...) end +function h5t_get_fields(type_id::Hid) + spos = Ref{Csize_t}() + epos = Ref{Csize_t}() + esize = Ref{Csize_t}() + mpos = Ref{Csize_t}() + msize = Ref{Csize_t}() + herr = ccall((:H5Tget_fields, libhdf5), + Herr, + (Hid, Ptr{Csize_t}, Ptr{Csize_t}, Ptr{Csize_t}, Ptr{Csize_t}, Ptr{Csize_t}), + type_id, spos, epos, esize, mpos, msize) + if herr < 0 + error("Error getting fields of floating-point datatype") + end + return (spos[], epos[], esize[], mpos[], msize[]) +end function h5t_get_member_name(type_id::Hid, index::Integer) pn = ccall((:H5Tget_member_name, libhdf5), Ptr{UInt8}, @@ -2434,6 +2478,10 @@ function __init__() UTF8_ATTRIBUTE_PROPERTIES[] = p_create(H5P_ATTRIBUTE_CREATE) h5p_set_char_encoding(UTF8_ATTRIBUTE_PROPERTIES[].id, cset(Compat.UTF8String)) + # Set up Float16 (must occur at runtime) + eval(:(const H5T_FLOAT16 = make_float16())) + eval(:(hdf5_type_id(::Type{Float16}) = H5T_FLOAT16)) + rehash!(hdf5_type_map, length(hdf5_type_map.keys)) rehash!(hdf5_prop_get_set, length(hdf5_prop_get_set.keys)) diff --git a/test/plain.jl b/test/plain.jl index e1e9e0432..c9c05c28b 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -325,3 +325,9 @@ using Compat.String end end + +arr_float16 = Float16[1.0, 0.25, 0.5, 8.0] +h5write("test_float16.h5", "x", arr_float16) +arr_float16_2 = h5read("test_float16.h5", "x") +@test isa(arr_float16_2, Vector{Float16}) +@test arr_float16_2 == arr_float16 From b6eb8d156cc1b3723303f412468f77cf62160d1e Mon Sep 17 00:00:00 2001 From: Andy Ferris Date: Wed, 26 Oct 2016 16:18:39 +1000 Subject: [PATCH 02/13] Aesthetic fixes --- src/HDF5.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/HDF5.jl b/src/HDF5.jl index e0b1a9c7d..b0ddbba01 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -1842,7 +1842,7 @@ function hdf5_to_julia_eltype(objtype) error("character set ", cset, " not recognized") end elseif class_id == H5T_INTEGER || class_id == H5T_FLOAT - # First look in the type last for a match + # First look in the type list for a match # otherwise fall back to a native datatype # Allows for users to dynamically add types to the typemap t_size = h5t_get_size(objtype) @@ -2479,8 +2479,8 @@ function __init__() h5p_set_char_encoding(UTF8_ATTRIBUTE_PROPERTIES[].id, cset(Compat.UTF8String)) # Set up Float16 (must occur at runtime) - eval(:(const H5T_FLOAT16 = make_float16())) - eval(:(hdf5_type_id(::Type{Float16}) = H5T_FLOAT16)) + @eval(const H5T_FLOAT16 = make_float16()) + @eval(hdf5_type_id(::Type{Float16}) = H5T_FLOAT16) rehash!(hdf5_type_map, length(hdf5_type_map.keys)) rehash!(hdf5_prop_get_set, length(hdf5_prop_get_set.keys)) From 429bc75e7583884c1c1574a7b701644191ed44b8 Mon Sep 17 00:00:00 2001 From: Mus M Date: Wed, 22 Mar 2017 15:00:42 -0400 Subject: [PATCH 03/13] work around macro with parens --- src/HDF5.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HDF5.jl b/src/HDF5.jl index b0ddbba01..11781a9dc 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -2480,7 +2480,7 @@ function __init__() # Set up Float16 (must occur at runtime) @eval(const H5T_FLOAT16 = make_float16()) - @eval(hdf5_type_id(::Type{Float16}) = H5T_FLOAT16) + @eval hdf5_type_id(::Type{Float16}) = H5T_FLOAT16 rehash!(hdf5_type_map, length(hdf5_type_map.keys)) rehash!(hdf5_prop_get_set, length(hdf5_prop_get_set.keys)) From e51c84e55fbe8569ecbe3ece3769629438cc7324 Mon Sep 17 00:00:00 2001 From: Mus M Date: Tue, 28 Mar 2017 11:01:54 -0400 Subject: [PATCH 04/13] Update plain.jl --- test/plain.jl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/plain.jl b/test/plain.jl index feee5ed36..d6d476812 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -332,10 +332,12 @@ using Compat.String str = read(fid["test"]) @test str == "Hello World" end -end + + # Test Float16 support + arr_float16 = Float16[1.0, 0.25, 0.5, 8.0] + h5write("test_float16.h5", "x", arr_float16) + arr_float16_2 = h5read("test_float16.h5", "x") + @test isa(arr_float16_2, Vector{Float16}) + @test arr_float16_2 == arr_float16 -arr_float16 = Float16[1.0, 0.25, 0.5, 8.0] -h5write("test_float16.h5", "x", arr_float16) -arr_float16_2 = h5read("test_float16.h5", "x") -@test isa(arr_float16_2, Vector{Float16}) -@test arr_float16_2 == arr_float16 +end From 5c7b240a6cf89dd22dcf848c0a1332f903902dfc Mon Sep 17 00:00:00 2001 From: Andy Ferris Date: Wed, 29 Mar 2017 10:20:27 +1000 Subject: [PATCH 05/13] Put Float16 tests in temp directory and cleanup --- test/plain.jl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/plain.jl b/test/plain.jl index d6d476812..fd7adb247 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -303,8 +303,6 @@ using Compat.String @test names(fid["mygroup"]) == Compat.ASCIIString["y"] end - rm(tmpdir, recursive=true) - d = h5read(joinpath(test_path, "compound.h5"), "/data") @test typeof(d[1]) === HDF5.HDF5Compound{4} @test length(d) == 2 @@ -332,12 +330,13 @@ using Compat.String str = read(fid["test"]) @test str == "Hello World" end - + # Test Float16 support arr_float16 = Float16[1.0, 0.25, 0.5, 8.0] - h5write("test_float16.h5", "x", arr_float16) - arr_float16_2 = h5read("test_float16.h5", "x") + h5write(joinpath(tmpdir, "test_float16.h5"), "x", arr_float16) + arr_float16_2 = h5read(joinpath(tmpdir, "test_float16.h5"), "x") @test isa(arr_float16_2, Vector{Float16}) @test arr_float16_2 == arr_float16 + rm(tmpdir, recursive=true) end From 33dfb18a48c7361763087fc0cec0f30cc9a725ec Mon Sep 17 00:00:00 2001 From: Mus M Date: Wed, 29 Mar 2017 09:01:43 -0400 Subject: [PATCH 06/13] rm both temporary files created during testing --- test/plain.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/plain.jl b/test/plain.jl index fd7adb247..1800d9299 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -6,9 +6,10 @@ using Compat.String @testset "plain" begin const test_path = splitdir(@__FILE__)[1] + const tmpdir = mktempdir() # Create a new file - fn = joinpath(tempdir(),"test.h5") + fn = joinpath(tempdir(), "test1.h5") f = h5open(fn, "w") # Write scalars f["Float64"] = 3.2 @@ -263,8 +264,7 @@ using Compat.String close(fid) # more do syntax: atomic rename version - tmpdir = mktempdir() - outfile = joinpath(tmpdir, "test.h5") + outfile = joinpath(tmpdir, "test2.h5") # create a new file h5rewrite(outfile) do fid From be6645b0c8fe913a0d3f9f4af872a7958d31b0e1 Mon Sep 17 00:00:00 2001 From: Mus M Date: Wed, 29 Mar 2017 09:06:11 -0400 Subject: [PATCH 07/13] Fix typo tempdir() to tmpdir --- test/plain.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plain.jl b/test/plain.jl index 1800d9299..268f40580 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -9,7 +9,7 @@ using Compat.String const tmpdir = mktempdir() # Create a new file - fn = joinpath(tempdir(), "test1.h5") + fn = joinpath(tmpdir, "test1.h5") f = h5open(fn, "w") # Write scalars f["Float64"] = 3.2 From bab0763adcdcbc07d9344e581506567202ed24f8 Mon Sep 17 00:00:00 2001 From: Mus M Date: Wed, 29 Mar 2017 09:10:45 -0400 Subject: [PATCH 08/13] Update plain.jl --- test/plain.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/plain.jl b/test/plain.jl index 268f40580..c82103dab 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -6,7 +6,8 @@ using Compat.String @testset "plain" begin const test_path = splitdir(@__FILE__)[1] - const tmpdir = mktempdir() + + tmpdir = mktempdir() # Create a new file fn = joinpath(tmpdir, "test1.h5") @@ -264,6 +265,8 @@ using Compat.String close(fid) # more do syntax: atomic rename version + rm(tmpdir, recursive=true) + tmpdir = mktempdir() outfile = joinpath(tmpdir, "test2.h5") # create a new file From ef2d1a6c93629fcf923c6c02f2d992320d7455f0 Mon Sep 17 00:00:00 2001 From: Mus M Date: Wed, 29 Mar 2017 09:11:29 -0400 Subject: [PATCH 09/13] Update plain.jl --- test/plain.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/plain.jl b/test/plain.jl index c82103dab..03e775335 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -10,7 +10,7 @@ using Compat.String tmpdir = mktempdir() # Create a new file - fn = joinpath(tmpdir, "test1.h5") + fn = joinpath(tmpdir, "test.h5") f = h5open(fn, "w") # Write scalars f["Float64"] = 3.2 @@ -267,7 +267,7 @@ using Compat.String # more do syntax: atomic rename version rm(tmpdir, recursive=true) tmpdir = mktempdir() - outfile = joinpath(tmpdir, "test2.h5") + outfile = joinpath(tmpdir, "test.h5") # create a new file h5rewrite(outfile) do fid From 78a260aa0aee7558e4dd241c76494f88d04599c2 Mon Sep 17 00:00:00 2001 From: Mus M Date: Wed, 29 Mar 2017 09:33:14 -0400 Subject: [PATCH 10/13] update open of nullterm_ascii file --- test/plain.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plain.jl b/test/plain.jl index 03e775335..a22c7f7b9 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -329,7 +329,7 @@ using Compat.String end # Test null terminated ASCII string (e.g. exported by h5py) #332 - h5open("test_nullterm_ascii.h5","r") do fid + h5open(joinpath(test_path, "test_nullterm_ascii.h5"), "r") do fid str = read(fid["test"]) @test str == "Hello World" end From 861cdf4af486d28c9256cf498a9605a0cb9190a5 Mon Sep 17 00:00:00 2001 From: Mus M Date: Wed, 29 Mar 2017 09:53:02 -0400 Subject: [PATCH 11/13] Update plain.jl --- test/plain.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/plain.jl b/test/plain.jl index a22c7f7b9..8d5244033 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -7,10 +7,8 @@ using Compat.String const test_path = splitdir(@__FILE__)[1] - tmpdir = mktempdir() - # Create a new file - fn = joinpath(tmpdir, "test.h5") + fn = joinpath(tempdir(), "test.h5") f = h5open(fn, "w") # Write scalars f["Float64"] = 3.2 @@ -265,7 +263,6 @@ using Compat.String close(fid) # more do syntax: atomic rename version - rm(tmpdir, recursive=true) tmpdir = mktempdir() outfile = joinpath(tmpdir, "test.h5") @@ -341,5 +338,7 @@ using Compat.String @test isa(arr_float16_2, Vector{Float16}) @test arr_float16_2 == arr_float16 + # Remove created files + rm(fn) rm(tmpdir, recursive=true) end From 9a7788e0beedc92933cd0f15742ee0eb667045d1 Mon Sep 17 00:00:00 2001 From: Mus M Date: Wed, 29 Mar 2017 09:53:20 -0400 Subject: [PATCH 12/13] Update plain.jl --- test/plain.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plain.jl b/test/plain.jl index 8d5244033..51b6d9f73 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -6,7 +6,7 @@ using Compat.String @testset "plain" begin const test_path = splitdir(@__FILE__)[1] - + # Create a new file fn = joinpath(tempdir(), "test.h5") f = h5open(fn, "w") From ca674a8342da877a28f9767ace0de7d88b75b00f Mon Sep 17 00:00:00 2001 From: Mus M Date: Wed, 28 Jun 2017 12:56:05 -0400 Subject: [PATCH 13/13] Update plain.jl --- test/plain.jl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/plain.jl b/test/plain.jl index c48bab972..c4eecdbda 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -303,6 +303,14 @@ h5open(outfile, "r") do fid @test names(fid) == ["mygroup"] @test names(fid["mygroup"]) == ["y"] end + +# Test Float16 support +arr_float16 = Float16[1.0, 0.25, 0.5, 8.0] +h5write(joinpath(tmpdir, "test_float16.h5"), "x", arr_float16) +arr_float16_2 = h5read(joinpath(tmpdir, "test_float16.h5"), "x") +@test isa(arr_float16_2, Vector{Float16}) +@test arr_float16_2 == arr_float16 + rm(tmpdir, recursive=true) test_files = joinpath(@__DIR__, "test_files") @@ -329,13 +337,6 @@ h5open(fn, "r", "libver_bounds", @test intarray == [1, 2, 3] end -# Test Float16 support -arr_float16 = Float16[1.0, 0.25, 0.5, 8.0] -h5write(joinpath(tmpdir, "test_float16.h5"), "x", arr_float16) -arr_float16_2 = h5read(joinpath(tmpdir, "test_float16.h5"), "x") -@test isa(arr_float16_2, Vector{Float16}) -@test arr_float16_2 == arr_float16 - # Test null terminated ASCII string (e.g. exported by h5py) #332 h5open(joinpath(test_files, "nullterm_ascii.h5"), "r") do fid str = read(fid["test"])