diff --git a/BITGROOM/config/cmake/binex/example/CMakeLists.txt b/BITGROOM/config/cmake/binex/example/CMakeLists.txt index 94967e3b5..5f03586f8 100644 --- a/BITGROOM/config/cmake/binex/example/CMakeLists.txt +++ b/BITGROOM/config/cmake/binex/example/CMakeLists.txt @@ -34,6 +34,17 @@ foreach (example ${dyn_examples}) endforeach () if (H5PL_BUILD_TESTING) + # h5repack test input generator. Target name is plugin-suffixed because the + # BITROUND and BITGROOM trees share the same CMake project and source file + # basename (issue #240). + add_executable (h5repack_floats_bitgroom ${PROJECT_SOURCE_DIR}/h5repack_floats.c) + target_include_directories(h5repack_floats_bitgroom PRIVATE ${H5PL_HDF5_INCLUDE_DIRS}) + TARGET_C_PROPERTIES (h5repack_floats_bitgroom ${LIB_TYPE}) + target_link_libraries (h5repack_floats_bitgroom PRIVATE ${H5PL_HDF5_LINK_LIBS}) + if (NOT WIN32) + target_link_libraries (h5repack_floats_bitgroom PRIVATE ${CMAKE_DL_LIBS}) + endif () + macro (ADD_H5_TEST testname) add_test ( NAME ${testname}-clearall @@ -121,21 +132,47 @@ if (H5PL_BUILD_TESTING) NAME H5BITGROOM_UD-${testname}-clearall-objects COMMAND ${CMAKE_COMMAND} -E remove + ${resultfile} out-${testname}.${resultfile} ${testname}.${resultfile}.out ${testname}.${resultfile}.out.err ${resultfile}-${testname}.out ${resultfile}-${testname}.out.err + ${resultfile}-${testname}-data.out + ${resultfile}-${testname}-data.out.err ) if (NOT "${last_test}" STREQUAL "") set_tests_properties (H5BITGROOM_UD-${testname}-clearall-objects PROPERTIES DEPENDS ${last_test}) endif () set (last_test "H5BITGROOM_UD-${testname}-clearall-objects") if (WIN32) - set (${TESTLIBDIR} "${HDF5_TOOLS_DIR}") + set (TESTLIBDIR "${HDF5_TOOLS_DIR}") else () - set (${TESTLIBDIR} "${HDF5_LIBRARY_PATH}") + set (TESTLIBDIR "${HDF5_LIBRARY_PATH}") endif () + # Generate the float-data input file for h5repack. The BitGroom filter's + # set_local() callback removes the filter from the pipeline on non-float + # datasets, so the input must contain float data for the test to actually + # exercise the filter (issue #240). Routed through runTest.cmake so + # PATH/DYLD_LIBRARY_PATH/LD_LIBRARY_PATH are set up for the helper to find + # the HDF5 runtime library on every platform (Windows in particular has + # no rpath equivalent and reports STATUS_DLL_NOT_FOUND if the HDF5 DLL is + # not on PATH). + add_test ( + NAME H5BITGROOM_UD-${testname}-gen-input + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_EXPECT=0" + -D "TEST_OUTPUT=h5repack_floats_bitgroom.out" + -D "TEST_SKIP_COMPARE=1" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -P "${H5PL_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5BITGROOM_UD-${testname}-gen-input PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS H5BITGROOM_UD-${testname}-clearall-objects) + set (last_test "H5BITGROOM_UD-${testname}-gen-input") if (HDF5_BUILD_TOOLS OR HDF5_PROVIDES_TOOLS) add_test ( NAME H5BITGROOM_UD-${testname} @@ -150,7 +187,7 @@ if (H5PL_BUILD_TESTING) -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" -P "${H5PL_RESOURCES_DIR}/runTest.cmake" ) - set_tests_properties (H5BITGROOM_UD-${testname} PROPERTIES DEPENDS H5BITGROOM_UD-${testname}-clearall-objects) + set_tests_properties (H5BITGROOM_UD-${testname} PROPERTIES DEPENDS H5BITGROOM_UD-${testname}-gen-input) add_test ( NAME H5BITGROOM_UD-h5dump-${testname} COMMAND "${CMAKE_COMMAND}" @@ -167,23 +204,43 @@ if (H5PL_BUILD_TESTING) WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" DEPENDS H5BITGROOM_UD-${testname}) set (last_test "H5BITGROOM_UD-h5dump-${testname}") + # Verify that the filter actually modifies the dataset values: dump the + # data (no -pH) and compare against a captured reference of the + # quantized output. BitGroom's encoding is bit-deterministic across + # platforms (its inner loop is pure integer bit ops, with the bit-zero + # count derived once per call from NSD via ceil() of a constant + # expression), so the captured reference is portable. + add_test ( + NAME H5BITGROOM_UD-h5dump-data-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5PL_HDF5_DUMP_EXECUTABLE}" + -D "TEST_ARGS:STRING=out-${testname}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_OUTPUT=${resultfile}-${testname}-data.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}-${testname}-data.ddl" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -P "${H5PL_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5BITGROOM_UD-h5dump-data-${testname} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS H5BITGROOM_UD-h5dump-${testname}) + set (last_test "H5BITGROOM_UD-h5dump-data-${testname}") endif () endmacro () # -------------------------------------------------------------------- # Copy all the HDF5 files from the source directory into the test directory # -------------------------------------------------------------------- - set (LIST_HDF5_TEST_FILES - h5repack_layout.h5 - ) set (LIST_OTHER_TEST_FILES h5ex_d_bitgroom.ddl h5ex_d_bitgroom.tst - h5repack_layout.h5-ud_convert.ddl - ud_convert.h5repack_layout.h5.tst + h5repack_floats.h5-ud_convert.ddl + h5repack_floats.h5-ud_convert-data.ddl + ud_convert.h5repack_floats.h5.tst ) - foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) + foreach (h5_file ${LIST_OTHER_TEST_FILES}) HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/${h5_file}" "example_files") endforeach () add_custom_target(example_files ALL COMMENT "Copying files needed by example tests" DEPENDS ${example_files_list}) @@ -192,7 +249,7 @@ if (H5PL_BUILD_TESTING) if (NOT DISABLE_H5PL_ENCODER) #UD BITGROOM - ADD_H5_UD_TEST (ud_convert 0 h5repack_layout.h5 --enable-error-stack -v -f UD=32022,0,5,3,4,0,0,0 -l CHUNK=4x8) + ADD_H5_UD_TEST (ud_convert 0 h5repack_floats.h5 --enable-error-stack -v -f UD=32022,0,5,3,4,0,0,0 -l CHUNK=4x8) endif () endif () diff --git a/BITGROOM/config/cmake/binex/example/h5repack_floats.c b/BITGROOM/config/cmake/binex/example/h5repack_floats.c new file mode 100644 index 000000000..e4127da49 --- /dev/null +++ b/BITGROOM/config/cmake/binex/example/h5repack_floats.c @@ -0,0 +1,87 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of the HDF5 BitGroom filter plugin source. The full * + * copyright notice, including terms governing use, modification, and * + * redistribution, is contained in the file COPYING, which can be found at * + * the root of the source code distribution tree. If you do not have access * + * to this file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Generates an HDF5 file with a chunked dataset of IEEE 32-bit floats and no + * filters applied. Used as input for the h5repack BitGroom test so the + * filter's set_local() callback retains the filter in the pipeline (the + * callback removes itself on non-float data). See issue #240. + */ + +#include "hdf5.h" +#include + +#define FILENAME "h5repack_floats.h5" +#define DATASET "data" +#define DIM0 32 +#define DIM1 64 +#define CHUNK0 4 +#define CHUNK1 8 + +int +main(void) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t space_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + hsize_t dims[2] = {DIM0, DIM1}; + hsize_t chunk[2] = {CHUNK0, CHUNK1}; + float wdata[DIM0][DIM1]; + int i, j; + int ret_value = 1; + + /* Non-integer floats: the 0.137f multiplier ensures every value carries + * non-trivial mantissa bits, so quantization by BitGroom is visible + * across the entire dataset rather than only on values that exceed the + * retained mantissa width. */ + for (i = 0; i < DIM0; i++) + for (j = 0; j < DIM1; j++) + wdata[i][j] = ((float)(i * j) - (float)j) * 0.137f; + + file_id = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if (file_id < 0) + goto done; + + space_id = H5Screate_simple(2, dims, NULL); + if (space_id < 0) + goto done; + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + if (dcpl_id < 0) + goto done; + if (H5Pset_chunk(dcpl_id, 2, chunk) < 0) + goto done; + + dset_id = H5Dcreate(file_id, DATASET, H5T_IEEE_F32LE, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + if (dset_id < 0) + goto done; + + if (H5Dwrite(dset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void *)wdata) < 0) + goto done; + + ret_value = 0; + +done: + if (dset_id >= 0) + H5Dclose(dset_id); + if (dcpl_id >= 0) + H5Pclose(dcpl_id); + if (space_id >= 0) + H5Sclose(space_id); + if (file_id >= 0) + H5Fclose(file_id); + + if (ret_value != 0) + fprintf(stderr, "h5repack_floats: failed to create %s\n", FILENAME); + + return ret_value; +} diff --git a/BITGROOM/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl b/BITGROOM/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl new file mode 100644 index 000000000..bfb9d303e --- /dev/null +++ b/BITGROOM/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl @@ -0,0 +1,318 @@ +HDF5 "out-ud_convert.h5repack_floats.h5" { +GROUP "/" { + DATASET "data" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + DATA { + (0,0): 0, -0.137024, -0.273926, -0.411011, -0.547852, -0.685059, + (0,6): -0.821777, -0.959228, -1.0957, -1.2334, -1.36963, -1.50732, + (0,12): -1.64355, -1.78125, -1.91797, -2.05566, -2.19141, -2.3291, + (0,18): -2.46582, -2.60352, -2.73926, -2.87793, -3.01367, -3.15137, + (0,24): -3.28711, -3.42578, -3.56152, -3.69922, -3.83594, -3.97363, + (0,30): -4.10938, -4.24805, -4.38281, -4.52148, -4.65625, -4.79687, + (0,36): -4.93164, -5.07031, -5.20508, -5.34375, -5.47852, -5.61719, + (0,42): -5.75391, -5.89258, -6.02734, -6.16602, -6.30078, -6.43945, + (0,48): -6.57422, -6.71484, -6.84961, -6.98828, -7.12305, -7.26172, + (0,54): -7.39648, -7.53516, -7.67188, -7.81055, -7.94531, -8.08594, + (0,60): -8.21875, -8.35937, -8.49219, -8.63281, + (1,0): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,21): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,42): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,63): 0, + (2,0): 0, 0.137024, 0.273926, 0.411011, 0.547852, 0.685059, 0.821777, + (2,7): 0.959228, 1.0957, 1.2334, 1.36963, 1.50732, 1.64355, 1.78125, + (2,14): 1.91797, 2.05566, 2.19141, 2.3291, 2.46582, 2.60352, 2.73926, + (2,21): 2.87793, 3.01367, 3.15137, 3.28711, 3.42578, 3.56152, 3.69922, + (2,28): 3.83594, 3.97363, 4.10938, 4.24805, 4.38281, 4.52148, 4.65625, + (2,35): 4.79687, 4.93164, 5.07031, 5.20508, 5.34375, 5.47852, 5.61719, + (2,42): 5.75391, 5.89258, 6.02734, 6.16602, 6.30078, 6.43945, 6.57422, + (2,49): 6.71484, 6.84961, 6.98828, 7.12305, 7.26172, 7.39648, 7.53516, + (2,56): 7.67188, 7.81055, 7.94531, 8.08594, 8.21875, 8.35937, 8.49219, + (2,63): 8.63281, + (3,0): 0, 0.274048, 0.547852, 0.822021, 1.0957, 1.37012, 1.64355, + (3,7): 1.91846, 2.19141, 2.4668, 2.73926, 3.01465, 3.28711, 3.5625, + (3,14): 3.83594, 4.11133, 4.38281, 4.6582, 4.93164, 5.20703, 5.47852, + (3,21): 5.75586, 6.02734, 6.30273, 6.57422, 6.85156, 7.12305, 7.39844, + (3,28): 7.67188, 7.94727, 8.21875, 8.49609, 8.76562, 9.04297, 9.3125, + (3,35): 9.59375, 9.86328, 10.1406, 10.4102, 10.6875, 10.957, 11.2344, + (3,42): 11.5078, 11.7852, 12.0547, 12.332, 12.6016, 12.8789, 13.1484, + (3,49): 13.4297, 13.6992, 13.9766, 14.2461, 14.5234, 14.793, 15.0703, + (3,56): 15.3438, 15.6211, 15.8906, 16.1719, 16.4375, 16.7187, 16.9844, + (3,63): 17.2656, + (4,0): 0, 0.411011, 0.821777, 1.2334, 1.64355, 2.05566, 2.46582, + (4,7): 2.87793, 3.28711, 3.69922, 4.10938, 4.52148, 4.93164, 5.34375, + (4,14): 5.75391, 6.16602, 6.57422, 6.98828, 7.39648, 7.81055, 8.21875, + (4,21): 8.63281, 9.03906, 9.45312, 9.86328, 10.2773, 10.6836, 11.0977, + (4,28): 11.5078, 11.9219, 12.3281, 12.7422, 13.1484, 13.5664, 13.9727, + (4,35): 14.3867, 14.793, 15.207, 15.6172, 16.0312, 16.4375, 16.8516, + (4,42): 17.2578, 17.6797, 18.0781, 18.5, 18.8984, 19.3203, 19.7266, + (4,49): 20.1406, 20.5469, 20.9687, 21.3672, 21.7891, 22.1875, 22.6094, + (4,56): 23.0156, 23.4297, 23.8359, 24.25, 24.6562, 25.0781, 25.4766, + (4,63): 25.8984, + (5,0): 0, 0.548096, 1.0957, 1.64404, 2.19141, 2.74023, 3.28711, + (5,7): 3.83691, 4.38281, 4.93359, 5.47852, 6.0293, 6.57422, 7.125, + (5,14): 7.67188, 8.22266, 8.76562, 9.31641, 9.86328, 10.4141, 10.957, + (5,21): 11.5117, 12.0547, 12.6055, 13.1484, 13.7031, 14.2461, 14.7969, + (5,28): 15.3438, 15.8945, 16.4375, 16.9922, 17.5312, 18.0859, 18.625, + (5,35): 19.1875, 19.7266, 20.2812, 20.8203, 21.375, 21.9141, 22.4687, + (5,42): 23.0156, 23.5703, 24.1094, 24.6641, 25.2031, 25.7578, 26.2969, + (5,49): 26.8594, 27.3984, 27.9531, 28.4922, 29.0469, 29.5859, 30.1406, + (5,56): 30.6875, 31.2422, 31.7812, 32.3437, 32.875, 33.4375, 33.9688, + (5,63): 34.5312, + (6,0): 0, 0.685059, 1.36963, 2.05566, 2.73926, 3.42578, 4.10938, + (6,7): 4.79687, 5.47852, 6.16602, 6.84961, 7.53516, 8.21875, 8.90625, + (6,14): 9.58984, 10.2773, 10.957, 11.6484, 12.3281, 13.0156, 13.6992, + (6,21): 14.3867, 15.0664, 15.7578, 16.4375, 17.1328, 17.8047, 18.5, + (6,28): 19.1797, 19.8672, 20.5469, 21.2422, 21.9141, 22.6094, 23.2891, + (6,35): 23.9766, 24.6562, 25.3516, 26.0234, 26.7187, 27.3984, 28.0859, + (6,42): 28.7656, 29.4609, 30.1328, 30.8281, 31.5078, 32.2031, 32.875, + (6,49): 33.5781, 34.25, 34.9375, 35.6094, 36.3125, 36.9844, 37.6875, + (6,56): 38.3594, 39.0469, 39.7188, 40.4219, 41.0938, 41.7969, 42.4688, + (6,63): 43.1562, + (7,0): 0, 0.822021, 1.64355, 2.4668, 3.28711, 4.11133, 4.93164, + (7,7): 5.75586, 6.57422, 7.39844, 8.21875, 9.04297, 9.86328, 10.6875, + (7,14): 11.5078, 12.332, 13.1484, 13.9766, 14.793, 15.6211, 16.4375, + (7,21): 17.2656, 18.0781, 18.9062, 19.7266, 20.5547, 21.3672, 22.1953, + (7,28): 23.0156, 23.8437, 24.6562, 25.4844, 26.2969, 27.1328, 27.9453, + (7,35): 28.7734, 29.5859, 30.4141, 31.2344, 32.0625, 32.875, 33.7031, + (7,42): 34.5156, 35.3594, 36.1562, 37, 37.7969, 38.6406, 39.4531, + (7,49): 40.2812, 41.0938, 41.9375, 42.7344, 43.5781, 44.375, 45.2187, + (7,56): 46.0312, 46.8594, 47.6719, 48.5, 49.3125, 50.1562, 50.9531, + (7,63): 51.7969, + (8,0): 0, 0.959228, 1.91797, 2.87793, 3.83594, 4.79687, 5.75391, + (8,7): 6.71484, 7.67188, 8.63281, 9.58984, 10.5508, 11.5078, 12.4687, + (8,14): 13.4258, 14.3867, 15.3438, 16.3047, 17.2578, 18.2266, 19.1797, + (8,21): 20.1406, 21.0938, 22.0625, 23.0156, 23.9766, 24.9297, 25.8984, + (8,28): 26.8516, 27.8125, 28.7656, 29.7344, 30.6875, 31.6484, 32.5938, + (8,35): 33.5781, 34.5156, 35.4844, 36.4375, 37.4062, 38.3594, 39.3281, + (8,42): 40.2656, 41.25, 42.1875, 43.1562, 44.1094, 45.0781, 46.0312, + (8,49): 47, 47.9375, 48.9219, 49.8594, 50.8281, 51.7812, 52.75, + (8,56): 53.7031, 54.6719, 55.6094, 56.5937, 57.5312, 58.5, 59.4531, + (8,63): 60.4219, + (9,0): 0, 1.09619, 2.19141, 3.28809, 4.38281, 5.48047, 6.57422, + (9,7): 7.67383, 8.76562, 9.86719, 10.957, 12.0586, 13.1484, 14.25, + (9,14): 15.3438, 16.4453, 17.5312, 18.6328, 19.7266, 20.8281, 21.9141, + (9,21): 23.0234, 24.1094, 25.2109, 26.2969, 27.4062, 28.4922, 29.5937, + (9,28): 30.6875, 31.7891, 32.875, 33.9844, 35.0625, 36.1719, 37.25, + (9,35): 38.375, 39.4531, 40.5625, 41.6406, 42.75, 43.8281, 44.9375, + (9,42): 46.0312, 47.1406, 48.2188, 49.3281, 50.4062, 51.5156, 52.5938, + (9,49): 53.7187, 54.7969, 55.9062, 56.9844, 58.0937, 59.1719, 60.2812, + (9,56): 61.375, 62.4844, 63.5625, 64.6875, 65.75, 66.875, 67.9375, + (9,63): 69.0625, + (10,0): 0, 1.2334, 2.46582, 3.69922, 4.93164, 6.16602, 7.39648, + (10,7): 8.63281, 9.86328, 11.0977, 12.3281, 13.5664, 14.793, 16.0312, + (10,14): 17.2578, 18.5, 19.7266, 20.9687, 22.1875, 23.4297, 24.6562, + (10,21): 25.8984, 27.125, 28.3594, 29.5859, 30.8281, 32.0469, 33.2969, + (10,28): 34.5156, 35.7656, 36.9844, 38.2344, 39.4531, 40.7031, 41.9219, + (10,35): 43.1562, 44.375, 45.625, 46.8438, 48.0937, 49.3125, 50.5625, + (10,42): 51.7812, 53.0312, 54.25, 55.5, 56.7031, 57.9531, 59.1719, + (10,49): 60.4219, 61.6406, 62.8906, 64.0938, 65.375, 66.5625, 67.8437, + (10,56): 69.0312, 70.2812, 71.5, 72.75, 73.9688, 75.2187, 76.4375, + (10,63): 77.6875, + (11,0): 0, 1.37012, 2.73926, 4.11133, 5.47852, 6.85156, 8.21875, + (11,7): 9.59375, 10.957, 12.332, 13.6992, 15.0703, 16.4375, 17.8125, + (11,14): 19.1797, 20.5547, 21.9141, 23.2969, 24.6562, 26.0312, 27.3984, + (11,21): 28.7734, 30.1328, 31.5156, 32.875, 34.2656, 35.6094, 37, + (11,28): 38.3594, 39.7344, 41.0938, 42.4844, 43.8281, 45.2187, 46.5781, + (11,35): 47.9531, 49.3125, 50.7031, 52.0469, 53.4375, 54.7969, 56.1719, + (11,42): 57.5312, 58.9219, 60.2656, 61.6562, 63.0156, 64.4062, 65.75, + (11,49): 67.1562, 68.5, 69.875, 71.2188, 72.625, 73.9688, 75.375, + (11,56): 76.7188, 78.0937, 79.4375, 80.8437, 82.1875, 83.5937, 84.9375, + (11,63): 86.3125, + (12,0): 0, 1.50732, 3.01367, 4.52148, 6.02734, 7.53516, 9.03906, + (12,7): 10.5508, 12.0547, 13.5664, 15.0664, 16.5781, 18.0781, 19.5937, + (12,14): 21.0938, 22.6094, 24.1094, 25.625, 27.125, 28.6406, 30.1328, + (12,21): 31.6484, 33.1406, 34.6719, 36.1562, 37.6875, 39.1719, 40.7031, + (12,28): 42.1875, 43.7031, 45.2031, 46.7187, 48.2188, 49.7344, 51.2344, + (12,35): 52.75, 54.25, 55.7656, 57.2656, 58.7812, 60.2656, 61.7969, + (12,42): 63.2812, 64.8125, 66.2812, 67.8437, 69.3125, 70.8437, 72.3125, + (12,49): 73.8437, 75.3438, 76.875, 78.3438, 79.875, 81.375, 82.9062, + (12,56): 84.375, 85.9062, 87.375, 88.9375, 90.4062, 91.9375, 93.4062, + (12,63): 94.9687, + (13,0): 0, 1.64404, 3.28711, 4.93359, 6.57422, 8.22266, 9.86328, + (13,7): 11.5117, 13.1484, 14.7969, 16.4375, 18.0859, 19.7266, 21.375, + (13,14): 23.0156, 24.6641, 26.2969, 27.9531, 29.5859, 31.2422, 32.875, + (13,21): 34.5312, 36.1562, 37.8125, 39.4531, 41.1094, 42.7344, 44.3906, + (13,28): 46.0312, 47.6875, 49.3125, 50.9687, 52.5938, 54.2656, 55.8906, + (13,35): 57.5469, 59.1719, 60.8281, 62.4688, 64.125, 65.75, 67.4062, + (13,42): 69.0312, 70.7187, 72.3125, 74, 75.5938, 77.2812, 78.9062, + (13,49): 80.5625, 82.1875, 83.875, 85.4688, 87.1562, 88.75, 90.4375, + (13,56): 92.0625, 93.7187, 95.3438, 97, 98.625, 100.312, 101.906, + (13,63): 103.594, + (14,0): 0, 1.78125, 3.56152, 5.34375, 7.12305, 8.90625, 10.6836, + (14,7): 12.4687, 14.2461, 16.0312, 17.8047, 19.5937, 21.3672, 23.1562, + (14,14): 24.9297, 26.7187, 28.4922, 30.2812, 32.0469, 33.8437, 35.6094, + (14,21): 37.4062, 39.1719, 40.9687, 42.7344, 44.5312, 46.2969, 48.0937, + (14,28): 49.8594, 51.6562, 53.4219, 55.2187, 56.9844, 58.7812, 60.5469, + (14,35): 62.3437, 64.0938, 65.9062, 67.6562, 69.4687, 71.2188, 73.0312, + (14,42): 74.7812, 76.5937, 78.3438, 80.1562, 81.9062, 83.7187, 85.4688, + (14,49): 87.2812, 89.0312, 90.8437, 92.5938, 94.4062, 96.1562, 97.9687, + (14,56): 99.7188, 101.531, 103.281, 105.094, 106.844, 108.656, 110.406, + (14,63): 112.219, + (15,0): 0, 1.91846, 3.83594, 5.75586, 7.67188, 9.59375, 11.5078, + (15,7): 13.4297, 15.3438, 17.2656, 19.1797, 21.1016, 23.0156, 24.9375, + (15,14): 26.8516, 28.7734, 30.6875, 32.6094, 34.5156, 36.4531, 38.3594, + (15,21): 40.2812, 42.1875, 44.125, 46.0312, 47.9531, 49.8594, 51.7969, + (15,28): 53.7031, 55.625, 57.5312, 59.4687, 61.375, 63.2969, 65.1875, + (15,35): 67.1562, 69.0312, 70.9687, 72.875, 74.8125, 76.7188, 78.6562, + (15,42): 80.5312, 82.5, 84.375, 86.3125, 88.2188, 90.1562, 92.0625, 94, + (15,50): 95.875, 97.8437, 99.7188, 101.656, 103.562, 105.5, 107.406, + (15,57): 109.344, 111.219, 113.187, 115.062, 117, 118.906, 120.844, + (16,0): 0, 2.05566, 4.10938, 6.16602, 8.21875, 10.2773, 12.3281, + (16,7): 14.3867, 16.4375, 18.5, 20.5469, 22.6094, 24.6562, 26.7187, + (16,14): 28.7656, 30.8281, 32.875, 34.9375, 36.9844, 39.0469, 41.0938, + (16,21): 43.1562, 45.2031, 47.2656, 49.3125, 51.375, 53.4219, 55.5, + (16,28): 57.5312, 59.6094, 61.6406, 63.7187, 65.75, 67.8437, 69.8438, + (16,35): 71.9375, 73.9688, 76.0625, 78.0625, 80.1562, 82.1875, 84.2812, + (16,42): 86.2812, 88.375, 90.4062, 92.5, 94.5, 96.5937, 98.625, + (16,49): 100.719, 102.719, 104.812, 106.844, 108.937, 110.969, 113.031, + (16,56): 115.062, 117.156, 119.188, 121.25, 123.281, 125.375, 127.406, + (16,63): 129.5, + (17,0): 0, 2.19238, 4.38281, 6.57617, 8.76562, 10.9609, 13.1484, + (17,7): 15.3477, 17.5312, 19.7344, 21.9141, 24.1172, 26.2969, 28.5, + (17,14): 30.6875, 32.8906, 35.0625, 37.2656, 39.4531, 41.6562, 43.8281, + (17,21): 46.0469, 48.2188, 50.4219, 52.5938, 54.8125, 56.9844, 59.1875, + (17,28): 61.375, 63.5781, 65.75, 67.9687, 70.125, 72.3437, 74.5, 76.75, + (17,36): 78.9062, 81.125, 83.2812, 85.5, 87.6562, 89.875, 92.0625, + (17,43): 94.2812, 96.4375, 98.6562, 100.812, 103.031, 105.188, 107.437, + (17,50): 109.594, 111.812, 113.969, 116.187, 118.344, 120.562, 122.75, + (17,57): 124.969, 127.125, 129.375, 131.5, 133.75, 135.875, 138.125, + (18,0): 0, 2.3291, 4.65625, 6.98828, 9.3125, 11.6484, 13.9727, 16.3047, + (18,8): 18.625, 20.9687, 23.2891, 25.625, 27.9453, 30.2812, 32.5938, + (18,15): 34.9375, 37.25, 39.5937, 41.9219, 44.2656, 46.5781, 48.9219, + (18,22): 51.2344, 53.5781, 55.8906, 58.2344, 60.5469, 62.8906, 65.1875, + (18,29): 67.5625, 69.8438, 72.2187, 74.5, 76.875, 79.1562, 81.5312, + (18,36): 83.8438, 86.1875, 88.5, 90.8437, 93.1562, 95.5, 97.8125, + (18,43): 100.156, 102.469, 104.812, 107.125, 109.469, 111.781, 114.125, + (18,50): 116.438, 118.781, 121.094, 123.437, 125.75, 128.125, 130.375, + (18,57): 132.812, 135.062, 137.437, 139.688, 142.125, 144.375, 146.75, + (19,0): 0, 2.4668, 4.93164, 7.39844, 9.86328, 12.332, 14.793, 17.2656, + (19,8): 19.7266, 22.1953, 24.6562, 27.1328, 29.5859, 32.0625, 34.5156, + (19,15): 37, 39.4531, 41.9375, 44.375, 46.8594, 49.3125, 51.7969, + (19,22): 54.25, 56.7187, 59.1719, 61.6562, 64.0938, 66.5937, 69.0312, + (19,29): 71.5312, 73.9688, 76.4687, 78.9062, 81.4062, 83.8438, 86.3125, + (19,36): 88.75, 91.25, 93.6875, 96.1875, 98.625, 101.125, 103.562, + (19,43): 106.062, 108.5, 111, 113.406, 115.906, 118.344, 120.844, + (19,50): 123.281, 125.781, 128.188, 130.75, 133.125, 135.687, 138.062, + (19,57): 140.562, 143, 145.5, 147.938, 150.437, 152.875, 155.375, + (20,0): 0, 2.60352, 5.20508, 7.81055, 10.4102, 13.0156, 15.6172, + (20,7): 18.2266, 20.8203, 23.4297, 26.0234, 28.6406, 31.2344, 33.8437, + (20,14): 36.4375, 39.0469, 41.6406, 44.2656, 46.8438, 49.4687, 52.0469, + (20,21): 54.6719, 57.2656, 59.875, 62.4688, 65.0937, 67.6562, 70.2812, + (20,28): 72.875, 75.5, 78.0625, 80.7187, 83.2812, 85.9062, 88.5, + (20,35): 91.125, 93.6875, 96.3125, 98.9062, 101.531, 104.094, 106.75, + (20,42): 109.312, 111.937, 114.531, 117.156, 119.719, 122.344, 124.938, + (20,49): 127.562, 130.125, 132.812, 135.312, 138, 140.5, 143.187, + (20,56): 145.75, 148.375, 150.938, 153.625, 156.125, 158.812, 161.375, + (20,63): 164, + (21,0): 0, 2.74023, 5.47852, 8.22266, 10.957, 13.7031, 16.4375, + (21,7): 19.1875, 21.9141, 24.6641, 27.3984, 30.1406, 32.875, 35.625, + (21,14): 38.3594, 41.1094, 43.8281, 46.5937, 49.3125, 52.0625, 54.7969, + (21,21): 57.5469, 60.2656, 63.0312, 65.75, 68.5312, 71.2188, 74, + (21,28): 76.7188, 79.4687, 82.1875, 84.9687, 87.6562, 90.4375, 93.1562, + (21,35): 95.9062, 98.625, 101.406, 104.094, 106.875, 109.594, 112.344, + (21,42): 115.062, 117.844, 120.531, 123.312, 126.031, 128.812, 131.5, + (21,49): 134.312, 137, 139.75, 142.438, 145.25, 147.938, 150.75, + (21,56): 153.438, 156.187, 158.875, 161.687, 164.375, 167.187, 169.875, + (21,63): 172.625, + (22,0): 0, 2.87793, 5.75391, 8.63281, 11.5078, 14.3867, 17.2578, + (22,7): 20.1406, 23.0156, 25.8984, 28.7656, 31.6484, 34.5156, 37.4062, + (22,14): 40.2656, 43.1562, 46.0312, 48.9219, 51.7812, 54.6719, 57.5312, + (22,21): 60.4219, 63.2812, 66.1875, 69.0312, 71.9375, 74.7812, 77.6875, + (22,28): 80.5312, 83.4375, 86.2812, 89.1875, 92.0625, 94.9687, 97.8125, + (22,35): 100.719, 103.562, 106.469, 109.312, 112.219, 115.062, 117.969, + (22,42): 120.812, 123.719, 126.562, 129.5, 132.312, 135.25, 138.062, + (22,49): 141, 143.812, 146.75, 149.562, 152.5, 155.312, 158.25, + (22,56): 161.062, 164, 166.812, 169.75, 172.562, 175.5, 178.312, + (22,63): 181.312, + (23,0): 0, 3.01465, 6.02734, 9.04297, 12.0547, 15.0703, 18.0781, + (23,7): 21.1016, 24.1094, 27.1328, 30.1328, 33.1562, 36.1562, 39.1875, + (23,14): 42.1875, 45.2187, 48.2188, 51.25, 54.25, 57.2812, 60.2656, + (23,21): 63.2969, 66.2812, 69.3437, 72.3125, 75.375, 78.3438, 81.4062, + (23,28): 84.375, 87.4062, 90.4062, 93.4375, 96.4375, 99.4687, 102.469, + (23,35): 105.5, 108.5, 111.531, 114.531, 117.562, 120.531, 123.594, + (23,42): 126.562, 129.625, 132.562, 135.687, 138.625, 141.687, 144.625, + (23,49): 147.687, 150.688, 153.75, 156.688, 159.75, 162.75, 165.812, + (23,56): 168.75, 171.812, 174.75, 177.875, 180.812, 183.875, 186.812, + (23,63): 189.937, + (24,0): 0, 3.15137, 6.30078, 9.45312, 12.6016, 15.7578, 18.8984, + (24,7): 22.0625, 25.2031, 28.3594, 31.5078, 34.6719, 37.7969, 40.9687, + (24,14): 44.1094, 47.2656, 50.4062, 53.5781, 56.7031, 59.875, 63.0156, + (24,21): 66.1875, 69.3125, 72.5, 75.5938, 78.7812, 81.9062, 85.0937, + (24,28): 88.2188, 91.4062, 94.5, 97.6875, 100.812, 104, 107.125, + (24,35): 110.312, 113.406, 116.594, 119.719, 122.906, 126.031, 129.25, + (24,42): 132.312, 135.5, 138.625, 141.812, 144.938, 148.125, 151.188, + (24,49): 154.437, 157.5, 160.75, 163.812, 167.062, 170.125, 173.312, + (24,56): 176.438, 179.625, 182.75, 185.937, 189, 192.25, 195.312, + (24,63): 198.562, + (25,0): 0, 3.28809, 6.57422, 9.86719, 13.1484, 16.4453, 19.7266, + (25,7): 23.0234, 26.2969, 29.5937, 32.875, 36.1719, 39.4531, 42.75, + (25,14): 46.0312, 49.3281, 52.5938, 55.9062, 59.1719, 62.4844, 65.75, + (25,21): 69.0625, 72.3125, 75.625, 78.9062, 82.2187, 85.4688, 88.7812, + (25,28): 92.0625, 95.375, 98.625, 101.937, 105.188, 108.531, 111.781, + (25,35): 115.094, 118.344, 121.656, 124.938, 128.25, 131.5, 134.812, + (25,42): 138.062, 141.437, 144.625, 148, 151.188, 154.562, 157.812, + (25,49): 161.125, 164.375, 167.75, 170.938, 174.312, 177.5, 180.875, + (25,56): 184.125, 187.437, 190.688, 194, 197.25, 200.625, 203.812, + (25,63): 207.187, + (26,0): 0, 3.42578, 6.84961, 10.2773, 13.6992, 17.1328, 20.5469, + (26,7): 23.9766, 27.3984, 30.8281, 34.25, 37.6875, 41.0938, 44.5312, + (26,14): 47.9375, 51.375, 54.7969, 58.2344, 61.6406, 65.0937, 68.5, + (26,21): 71.9375, 75.3438, 78.7812, 82.1875, 85.6562, 89.0312, 92.5, + (26,28): 95.875, 99.3437, 102.719, 106.187, 109.594, 113.031, 116.438, + (26,35): 119.875, 123.281, 126.75, 130.125, 133.625, 137, 140.437, + (26,42): 143.812, 147.312, 150.688, 154.187, 157.5, 161, 164.375, + (26,49): 167.875, 171.25, 174.687, 178.062, 181.562, 184.938, 188.437, + (26,56): 191.75, 195.25, 198.625, 202.125, 205.438, 208.937, 212.312, + (26,63): 215.812, + (27,0): 0, 3.5625, 7.12305, 10.6875, 14.2461, 17.8125, 21.3672, + (27,7): 24.9375, 28.4922, 32.0625, 35.6094, 39.1875, 42.7344, 46.3125, + (27,14): 49.8594, 53.4375, 56.9844, 60.5625, 64.0938, 67.6875, 71.2188, + (27,21): 74.8125, 78.3438, 81.9375, 85.4688, 89.0625, 92.5938, 96.1875, + (27,28): 99.7188, 103.312, 106.844, 110.437, 113.969, 117.562, 121.094, + (27,35): 124.687, 128.188, 131.812, 135.312, 138.937, 142.438, 146.062, + (27,42): 149.562, 153.187, 156.688, 160.312, 163.812, 167.437, 170.938, + (27,49): 174.562, 178.062, 181.687, 185.188, 188.812, 192.312, 195.937, + (27,56): 199.438, 203.062, 206.562, 210.187, 213.688, 217.312, 220.812, + (27,63): 224.437, + (28,0): 0, 3.69922, 7.39648, 11.0977, 14.793, 18.5, 22.1875, 25.8984, + (28,8): 29.5859, 33.2969, 36.9844, 40.7031, 44.375, 48.0937, 51.7812, + (28,15): 55.5, 59.1719, 62.8906, 66.5625, 70.2812, 73.9688, 77.6875, + (28,22): 81.375, 85.0937, 88.75, 92.5, 96.1562, 99.875, 103.562, + (28,29): 107.281, 110.969, 114.687, 118.344, 122.094, 125.75, 129.5, + (28,36): 133.125, 136.875, 140.5, 144.312, 147.938, 151.687, 155.312, + (28,43): 159.062, 162.75, 166.5, 170.125, 173.875, 177.5, 181.312, + (28,50): 184.938, 188.687, 192.312, 196.062, 199.688, 203.5, 207.125, + (28,57): 210.875, 214.5, 218.25, 221.938, 225.687, 229.312, 233.062, + (29,0): 0, 3.83691, 7.67188, 11.5117, 15.3438, 19.1875, 23.0156, + (29,7): 26.8594, 30.6875, 34.5312, 38.3594, 42.2031, 46.0312, 49.875, + (29,14): 53.7031, 57.5469, 61.375, 65.2187, 69.0312, 72.9062, 76.7188, + (29,21): 80.5625, 84.375, 88.25, 92.0625, 95.9062, 99.7188, 103.594, + (29,28): 107.406, 111.25, 115.062, 118.937, 122.75, 126.594, 130.375, + (29,35): 134.312, 138.062, 141.937, 145.75, 149.625, 153.438, 157.312, + (29,42): 161.062, 165, 168.75, 172.625, 176.438, 180.312, 184.125, 188, + (29,50): 191.75, 195.687, 199.438, 203.312, 207.125, 211, 214.812, + (29,57): 218.687, 222.438, 226.375, 230.125, 234, 237.812, 241.687, + (30,0): 0, 3.97363, 7.94531, 11.9219, 15.8906, 19.8672, 23.8359, + (30,7): 27.8125, 31.7812, 35.7656, 39.7188, 43.7031, 47.6719, 51.6562, + (30,14): 55.6094, 59.6094, 63.5625, 67.5625, 71.5, 75.5, 79.4375, + (30,21): 83.4375, 87.375, 91.4062, 95.3438, 99.3437, 103.281, 107.281, + (30,28): 111.219, 115.219, 119.188, 123.187, 127.125, 131.125, 135.062, + (30,35): 139.062, 143, 147.062, 150.938, 155, 158.875, 162.937, + (30,42): 166.812, 170.875, 174.75, 178.812, 182.75, 186.75, 190.688, + (30,49): 194.687, 198.625, 202.625, 206.562, 210.625, 214.5, 218.562, + (30,56): 222.438, 226.5, 230.375, 234.437, 238.375, 242.375, 246.312, + (30,63): 250.312, + (31,0): 0, 4.11133, 8.21875, 12.332, 16.4375, 20.5547, 24.6562, + (31,7): 28.7734, 32.875, 37, 41.0938, 45.2187, 49.3125, 53.4375, + (31,14): 57.5312, 61.6562, 65.75, 69.875, 73.9688, 78.0937, 82.1875, + (31,21): 86.3125, 90.4062, 94.5312, 98.625, 102.75, 106.844, 111, + (31,28): 115.062, 119.219, 123.281, 127.437, 131.5, 135.687, 139.688, + (31,35): 143.875, 147.938, 152.125, 156.125, 160.312, 164.375, 168.562, + (31,42): 172.562, 176.75, 180.812, 185, 189, 193.187, 197.25, 201.437, + (31,50): 205.438, 209.625, 213.688, 217.875, 221.938, 226.062, 230.125, + (31,57): 234.312, 238.375, 242.5, 246.562, 250.75, 254.812, 259 + } + } +} +} diff --git a/BITGROOM/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert.ddl b/BITGROOM/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert.ddl new file mode 100644 index 000000000..f4d34f36c --- /dev/null +++ b/BITGROOM/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert.ddl @@ -0,0 +1,26 @@ +HDF5 "out-ud_convert.h5repack_floats.h5" { +GROUP "/" { + DATASET "data" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + STORAGE_LAYOUT { + CHUNKED ( 4, 8 ) + SIZE 8192 (1.000:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32022 + COMMENT BitGroom filter (Zender, 2016 GMD: http://www.geosci-model-dev.net/9/3199/2016) + PARAMS { 3 4 0 0 0 } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } +} +} diff --git a/BITGROOM/config/cmake/binex/example/testfiles/h5repack_layout.h5 b/BITGROOM/config/cmake/binex/example/testfiles/h5repack_layout.h5 deleted file mode 100644 index 2cc44b140..000000000 Binary files a/BITGROOM/config/cmake/binex/example/testfiles/h5repack_layout.h5 and /dev/null differ diff --git a/BITGROOM/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl b/BITGROOM/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl deleted file mode 100644 index dd4ae1308..000000000 --- a/BITGROOM/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl +++ /dev/null @@ -1,130 +0,0 @@ -HDF5 "out-ud_convert.h5repack_layout.h5" { -GROUP "/" { - DATASET "dset1" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset2" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset3" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset4" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_chunk" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, H5S_UNLIMITED ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_compact" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_contiguous" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } -} -} diff --git a/BITGROOM/example/testfiles/ud_convert.h5repack_layout.h5.tst b/BITGROOM/config/cmake/binex/example/testfiles/ud_convert.h5repack_floats.h5.tst similarity index 53% rename from BITGROOM/example/testfiles/ud_convert.h5repack_layout.h5.tst rename to BITGROOM/config/cmake/binex/example/testfiles/ud_convert.h5repack_floats.h5.tst index afcd00550..e658a2828 100644 --- a/BITGROOM/example/testfiles/ud_convert.h5repack_layout.h5.tst +++ b/BITGROOM/config/cmake/binex/example/testfiles/ud_convert.h5repack_floats.h5.tst @@ -7,10 +7,4 @@ Making new file ... Type Filter (Compression) Name ----------------------------------------- group / - dset UD (0.833:1) /dset1 - dset UD (0.833:1) /dset2 - dset UD (0.833:1) /dset3 - dset UD (0.833:1) /dset4 - dset UD (0.833:1) /dset_chunk - dset UD (0.833:1) /dset_compact - dset UD (0.833:1) /dset_contiguous + dset UD (1.000:1) /data diff --git a/BITGROOM/example/CMakeLists.txt b/BITGROOM/example/CMakeLists.txt index 8dc523d10..2062fe2ff 100755 --- a/BITGROOM/example/CMakeLists.txt +++ b/BITGROOM/example/CMakeLists.txt @@ -33,6 +33,16 @@ foreach (example ${dyn_examples}) endforeach () if (H5PL_BUILD_TESTING) + # h5repack test input generator. Target name is plugin-suffixed because the + # BITROUND and BITGROOM trees share the same CMake project and source file + # basename (issue #240). + add_executable (h5repack_floats_bitgroom ${PROJECT_SOURCE_DIR}/h5repack_floats.c) + TARGET_C_PROPERTIES (h5repack_floats_bitgroom ${LIB_TYPE}) + target_link_libraries (h5repack_floats_bitgroom PRIVATE ${H5PL_HDF5_LINK_LIBS}) + if (NOT WIN32) + target_link_libraries (h5repack_floats_bitgroom PRIVATE dl) + endif () + macro (ADD_H5_TEST testname) add_test ( NAME ${testname}-clearall @@ -126,6 +136,7 @@ if (H5PL_BUILD_TESTING) NAME H5BITGROOM_UD-${testname}-clearall-objects COMMAND ${CMAKE_COMMAND} -E remove + ${resultfile} out-${testname}.${resultfile} ${testname}.${resultfile}.out ${testname}.${resultfile}.out.err @@ -137,10 +148,33 @@ if (H5PL_BUILD_TESTING) endif () set (last_test "H5BITGROOM_UD-${testname}-clearall-objects") if (WIN32) - set (${TESTLIBDIR} "${HDF5_TOOLS_DIR}") + set (TESTLIBDIR "${HDF5_TOOLS_DIR}") else () - set (${TESTLIBDIR} "${HDF5_LIBRARY_PATH}") + set (TESTLIBDIR "${HDF5_LIBRARY_PATH}") endif () + # Generate the float-data input file for h5repack. The BitGroom filter's + # set_local() callback removes the filter from the pipeline on non-float + # datasets, so the input must contain float data for the test to actually + # exercise the filter (issue #240). Routed through runTest.cmake so + # PATH/DYLD_LIBRARY_PATH/LD_LIBRARY_PATH are set up for the helper to find + # the HDF5 runtime library on every platform (Windows in particular has + # no rpath equivalent and reports STATUS_DLL_NOT_FOUND if the HDF5 DLL is + # not on PATH). + add_test ( + NAME H5BITGROOM_UD-${testname}-gen-input + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_EXPECT=0" + -D "TEST_OUTPUT=h5repack_floats_bitgroom.out" + -D "TEST_SKIP_COMPARE=1" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -P "${H5BITGROOM_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5BITGROOM_UD-${testname}-gen-input PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS H5BITGROOM_UD-${testname}-clearall-objects) + set (last_test "H5BITGROOM_UD-${testname}-gen-input") if (HDF5_BUILD_TOOLS OR HDF5_PROVIDES_TOOLS) add_test ( NAME H5BITGROOM_UD-${testname} @@ -159,7 +193,7 @@ if (H5PL_BUILD_TESTING) ) set_tests_properties (H5BITGROOM_UD-${testname} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" - DEPENDS H5BITGROOM_UD-${testname}-clearall-objects) + DEPENDS H5BITGROOM_UD-${testname}-gen-input) add_test ( NAME H5BITGROOM_UD-h5dump-${testname} COMMAND "${CMAKE_COMMAND}" @@ -178,23 +212,46 @@ if (H5PL_BUILD_TESTING) WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" DEPENDS H5BITGROOM_UD-${testname}) set (last_test "H5BITGROOM_UD-h5dump-${testname}") + # Verify that the filter actually modifies the dataset values: dump the + # data (no -pH) and compare against a captured reference of the quantized + # output. BitGroom's encoding is bit-deterministic across platforms, so + # the captured reference is portable. The companion BITROUND test suite + # intentionally omits an equivalent check because the testing indicated + # the Granular BitRound encoding introduces platform-dependent + # 1-bit-different stored output. + add_test ( + NAME H5BITGROOM_UD-h5dump-data-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5PL_HDF5_DUMP_EXECUTABLE}" + -D "TEST_ARGS:STRING=out-${testname}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_OUTPUT=${resultfile}-${testname}-data.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}-${testname}-data.ddl" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -D "TEST_ENV_VAR=HDF5_PLUGIN_PATH" + -D "TEST_ENV_VALUE=${CMAKE_BINARY_DIR}/plugins" + -P "${H5BITGROOM_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5BITGROOM_UD-h5dump-data-${testname} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS H5BITGROOM_UD-h5dump-${testname}) + set (last_test "H5BITGROOM_UD-h5dump-data-${testname}") endif () endmacro () # -------------------------------------------------------------------- # Copy all the HDF5 files from the source directory into the test directory # -------------------------------------------------------------------- - set (LIST_HDF5_TEST_FILES - h5repack_layout.h5 - ) set (LIST_OTHER_TEST_FILES h5ex_d_bitgroom.ddl h5ex_d_bitgroom.tst - h5repack_layout.h5-ud_convert.ddl - ud_convert.h5repack_layout.h5.tst + h5repack_floats.h5-ud_convert.ddl + h5repack_floats.h5-ud_convert-data.ddl + ud_convert.h5repack_floats.h5.tst ) - foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) + foreach (h5_file ${LIST_OTHER_TEST_FILES}) HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/${h5_file}" "bitgroom_files") endforeach () add_custom_target(bitgroom_files ALL COMMENT "Copying files needed by bitgroom tests" DEPENDS ${bitgroom_files_list}) @@ -203,7 +260,7 @@ if (H5PL_BUILD_TESTING) if (NOT DISABLE_H5BITGROOM_ENCODER) #UD BITGROOM - ADD_H5_UD_TEST (ud_convert 0 h5repack_layout.h5 --enable-error-stack -v -f UD=32022,0,5,3,4,0,0,0 -l CHUNK=4x8) + ADD_H5_UD_TEST (ud_convert 0 h5repack_floats.h5 --enable-error-stack -v -f UD=32022,0,5,3,4,0,0,0 -l CHUNK=4x8) endif () endif () diff --git a/BITGROOM/example/h5repack_floats.c b/BITGROOM/example/h5repack_floats.c new file mode 100644 index 000000000..e4127da49 --- /dev/null +++ b/BITGROOM/example/h5repack_floats.c @@ -0,0 +1,87 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of the HDF5 BitGroom filter plugin source. The full * + * copyright notice, including terms governing use, modification, and * + * redistribution, is contained in the file COPYING, which can be found at * + * the root of the source code distribution tree. If you do not have access * + * to this file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Generates an HDF5 file with a chunked dataset of IEEE 32-bit floats and no + * filters applied. Used as input for the h5repack BitGroom test so the + * filter's set_local() callback retains the filter in the pipeline (the + * callback removes itself on non-float data). See issue #240. + */ + +#include "hdf5.h" +#include + +#define FILENAME "h5repack_floats.h5" +#define DATASET "data" +#define DIM0 32 +#define DIM1 64 +#define CHUNK0 4 +#define CHUNK1 8 + +int +main(void) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t space_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + hsize_t dims[2] = {DIM0, DIM1}; + hsize_t chunk[2] = {CHUNK0, CHUNK1}; + float wdata[DIM0][DIM1]; + int i, j; + int ret_value = 1; + + /* Non-integer floats: the 0.137f multiplier ensures every value carries + * non-trivial mantissa bits, so quantization by BitGroom is visible + * across the entire dataset rather than only on values that exceed the + * retained mantissa width. */ + for (i = 0; i < DIM0; i++) + for (j = 0; j < DIM1; j++) + wdata[i][j] = ((float)(i * j) - (float)j) * 0.137f; + + file_id = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if (file_id < 0) + goto done; + + space_id = H5Screate_simple(2, dims, NULL); + if (space_id < 0) + goto done; + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + if (dcpl_id < 0) + goto done; + if (H5Pset_chunk(dcpl_id, 2, chunk) < 0) + goto done; + + dset_id = H5Dcreate(file_id, DATASET, H5T_IEEE_F32LE, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + if (dset_id < 0) + goto done; + + if (H5Dwrite(dset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void *)wdata) < 0) + goto done; + + ret_value = 0; + +done: + if (dset_id >= 0) + H5Dclose(dset_id); + if (dcpl_id >= 0) + H5Pclose(dcpl_id); + if (space_id >= 0) + H5Sclose(space_id); + if (file_id >= 0) + H5Fclose(file_id); + + if (ret_value != 0) + fprintf(stderr, "h5repack_floats: failed to create %s\n", FILENAME); + + return ret_value; +} diff --git a/BITGROOM/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl b/BITGROOM/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl new file mode 100644 index 000000000..bfb9d303e --- /dev/null +++ b/BITGROOM/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl @@ -0,0 +1,318 @@ +HDF5 "out-ud_convert.h5repack_floats.h5" { +GROUP "/" { + DATASET "data" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + DATA { + (0,0): 0, -0.137024, -0.273926, -0.411011, -0.547852, -0.685059, + (0,6): -0.821777, -0.959228, -1.0957, -1.2334, -1.36963, -1.50732, + (0,12): -1.64355, -1.78125, -1.91797, -2.05566, -2.19141, -2.3291, + (0,18): -2.46582, -2.60352, -2.73926, -2.87793, -3.01367, -3.15137, + (0,24): -3.28711, -3.42578, -3.56152, -3.69922, -3.83594, -3.97363, + (0,30): -4.10938, -4.24805, -4.38281, -4.52148, -4.65625, -4.79687, + (0,36): -4.93164, -5.07031, -5.20508, -5.34375, -5.47852, -5.61719, + (0,42): -5.75391, -5.89258, -6.02734, -6.16602, -6.30078, -6.43945, + (0,48): -6.57422, -6.71484, -6.84961, -6.98828, -7.12305, -7.26172, + (0,54): -7.39648, -7.53516, -7.67188, -7.81055, -7.94531, -8.08594, + (0,60): -8.21875, -8.35937, -8.49219, -8.63281, + (1,0): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,21): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,42): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,63): 0, + (2,0): 0, 0.137024, 0.273926, 0.411011, 0.547852, 0.685059, 0.821777, + (2,7): 0.959228, 1.0957, 1.2334, 1.36963, 1.50732, 1.64355, 1.78125, + (2,14): 1.91797, 2.05566, 2.19141, 2.3291, 2.46582, 2.60352, 2.73926, + (2,21): 2.87793, 3.01367, 3.15137, 3.28711, 3.42578, 3.56152, 3.69922, + (2,28): 3.83594, 3.97363, 4.10938, 4.24805, 4.38281, 4.52148, 4.65625, + (2,35): 4.79687, 4.93164, 5.07031, 5.20508, 5.34375, 5.47852, 5.61719, + (2,42): 5.75391, 5.89258, 6.02734, 6.16602, 6.30078, 6.43945, 6.57422, + (2,49): 6.71484, 6.84961, 6.98828, 7.12305, 7.26172, 7.39648, 7.53516, + (2,56): 7.67188, 7.81055, 7.94531, 8.08594, 8.21875, 8.35937, 8.49219, + (2,63): 8.63281, + (3,0): 0, 0.274048, 0.547852, 0.822021, 1.0957, 1.37012, 1.64355, + (3,7): 1.91846, 2.19141, 2.4668, 2.73926, 3.01465, 3.28711, 3.5625, + (3,14): 3.83594, 4.11133, 4.38281, 4.6582, 4.93164, 5.20703, 5.47852, + (3,21): 5.75586, 6.02734, 6.30273, 6.57422, 6.85156, 7.12305, 7.39844, + (3,28): 7.67188, 7.94727, 8.21875, 8.49609, 8.76562, 9.04297, 9.3125, + (3,35): 9.59375, 9.86328, 10.1406, 10.4102, 10.6875, 10.957, 11.2344, + (3,42): 11.5078, 11.7852, 12.0547, 12.332, 12.6016, 12.8789, 13.1484, + (3,49): 13.4297, 13.6992, 13.9766, 14.2461, 14.5234, 14.793, 15.0703, + (3,56): 15.3438, 15.6211, 15.8906, 16.1719, 16.4375, 16.7187, 16.9844, + (3,63): 17.2656, + (4,0): 0, 0.411011, 0.821777, 1.2334, 1.64355, 2.05566, 2.46582, + (4,7): 2.87793, 3.28711, 3.69922, 4.10938, 4.52148, 4.93164, 5.34375, + (4,14): 5.75391, 6.16602, 6.57422, 6.98828, 7.39648, 7.81055, 8.21875, + (4,21): 8.63281, 9.03906, 9.45312, 9.86328, 10.2773, 10.6836, 11.0977, + (4,28): 11.5078, 11.9219, 12.3281, 12.7422, 13.1484, 13.5664, 13.9727, + (4,35): 14.3867, 14.793, 15.207, 15.6172, 16.0312, 16.4375, 16.8516, + (4,42): 17.2578, 17.6797, 18.0781, 18.5, 18.8984, 19.3203, 19.7266, + (4,49): 20.1406, 20.5469, 20.9687, 21.3672, 21.7891, 22.1875, 22.6094, + (4,56): 23.0156, 23.4297, 23.8359, 24.25, 24.6562, 25.0781, 25.4766, + (4,63): 25.8984, + (5,0): 0, 0.548096, 1.0957, 1.64404, 2.19141, 2.74023, 3.28711, + (5,7): 3.83691, 4.38281, 4.93359, 5.47852, 6.0293, 6.57422, 7.125, + (5,14): 7.67188, 8.22266, 8.76562, 9.31641, 9.86328, 10.4141, 10.957, + (5,21): 11.5117, 12.0547, 12.6055, 13.1484, 13.7031, 14.2461, 14.7969, + (5,28): 15.3438, 15.8945, 16.4375, 16.9922, 17.5312, 18.0859, 18.625, + (5,35): 19.1875, 19.7266, 20.2812, 20.8203, 21.375, 21.9141, 22.4687, + (5,42): 23.0156, 23.5703, 24.1094, 24.6641, 25.2031, 25.7578, 26.2969, + (5,49): 26.8594, 27.3984, 27.9531, 28.4922, 29.0469, 29.5859, 30.1406, + (5,56): 30.6875, 31.2422, 31.7812, 32.3437, 32.875, 33.4375, 33.9688, + (5,63): 34.5312, + (6,0): 0, 0.685059, 1.36963, 2.05566, 2.73926, 3.42578, 4.10938, + (6,7): 4.79687, 5.47852, 6.16602, 6.84961, 7.53516, 8.21875, 8.90625, + (6,14): 9.58984, 10.2773, 10.957, 11.6484, 12.3281, 13.0156, 13.6992, + (6,21): 14.3867, 15.0664, 15.7578, 16.4375, 17.1328, 17.8047, 18.5, + (6,28): 19.1797, 19.8672, 20.5469, 21.2422, 21.9141, 22.6094, 23.2891, + (6,35): 23.9766, 24.6562, 25.3516, 26.0234, 26.7187, 27.3984, 28.0859, + (6,42): 28.7656, 29.4609, 30.1328, 30.8281, 31.5078, 32.2031, 32.875, + (6,49): 33.5781, 34.25, 34.9375, 35.6094, 36.3125, 36.9844, 37.6875, + (6,56): 38.3594, 39.0469, 39.7188, 40.4219, 41.0938, 41.7969, 42.4688, + (6,63): 43.1562, + (7,0): 0, 0.822021, 1.64355, 2.4668, 3.28711, 4.11133, 4.93164, + (7,7): 5.75586, 6.57422, 7.39844, 8.21875, 9.04297, 9.86328, 10.6875, + (7,14): 11.5078, 12.332, 13.1484, 13.9766, 14.793, 15.6211, 16.4375, + (7,21): 17.2656, 18.0781, 18.9062, 19.7266, 20.5547, 21.3672, 22.1953, + (7,28): 23.0156, 23.8437, 24.6562, 25.4844, 26.2969, 27.1328, 27.9453, + (7,35): 28.7734, 29.5859, 30.4141, 31.2344, 32.0625, 32.875, 33.7031, + (7,42): 34.5156, 35.3594, 36.1562, 37, 37.7969, 38.6406, 39.4531, + (7,49): 40.2812, 41.0938, 41.9375, 42.7344, 43.5781, 44.375, 45.2187, + (7,56): 46.0312, 46.8594, 47.6719, 48.5, 49.3125, 50.1562, 50.9531, + (7,63): 51.7969, + (8,0): 0, 0.959228, 1.91797, 2.87793, 3.83594, 4.79687, 5.75391, + (8,7): 6.71484, 7.67188, 8.63281, 9.58984, 10.5508, 11.5078, 12.4687, + (8,14): 13.4258, 14.3867, 15.3438, 16.3047, 17.2578, 18.2266, 19.1797, + (8,21): 20.1406, 21.0938, 22.0625, 23.0156, 23.9766, 24.9297, 25.8984, + (8,28): 26.8516, 27.8125, 28.7656, 29.7344, 30.6875, 31.6484, 32.5938, + (8,35): 33.5781, 34.5156, 35.4844, 36.4375, 37.4062, 38.3594, 39.3281, + (8,42): 40.2656, 41.25, 42.1875, 43.1562, 44.1094, 45.0781, 46.0312, + (8,49): 47, 47.9375, 48.9219, 49.8594, 50.8281, 51.7812, 52.75, + (8,56): 53.7031, 54.6719, 55.6094, 56.5937, 57.5312, 58.5, 59.4531, + (8,63): 60.4219, + (9,0): 0, 1.09619, 2.19141, 3.28809, 4.38281, 5.48047, 6.57422, + (9,7): 7.67383, 8.76562, 9.86719, 10.957, 12.0586, 13.1484, 14.25, + (9,14): 15.3438, 16.4453, 17.5312, 18.6328, 19.7266, 20.8281, 21.9141, + (9,21): 23.0234, 24.1094, 25.2109, 26.2969, 27.4062, 28.4922, 29.5937, + (9,28): 30.6875, 31.7891, 32.875, 33.9844, 35.0625, 36.1719, 37.25, + (9,35): 38.375, 39.4531, 40.5625, 41.6406, 42.75, 43.8281, 44.9375, + (9,42): 46.0312, 47.1406, 48.2188, 49.3281, 50.4062, 51.5156, 52.5938, + (9,49): 53.7187, 54.7969, 55.9062, 56.9844, 58.0937, 59.1719, 60.2812, + (9,56): 61.375, 62.4844, 63.5625, 64.6875, 65.75, 66.875, 67.9375, + (9,63): 69.0625, + (10,0): 0, 1.2334, 2.46582, 3.69922, 4.93164, 6.16602, 7.39648, + (10,7): 8.63281, 9.86328, 11.0977, 12.3281, 13.5664, 14.793, 16.0312, + (10,14): 17.2578, 18.5, 19.7266, 20.9687, 22.1875, 23.4297, 24.6562, + (10,21): 25.8984, 27.125, 28.3594, 29.5859, 30.8281, 32.0469, 33.2969, + (10,28): 34.5156, 35.7656, 36.9844, 38.2344, 39.4531, 40.7031, 41.9219, + (10,35): 43.1562, 44.375, 45.625, 46.8438, 48.0937, 49.3125, 50.5625, + (10,42): 51.7812, 53.0312, 54.25, 55.5, 56.7031, 57.9531, 59.1719, + (10,49): 60.4219, 61.6406, 62.8906, 64.0938, 65.375, 66.5625, 67.8437, + (10,56): 69.0312, 70.2812, 71.5, 72.75, 73.9688, 75.2187, 76.4375, + (10,63): 77.6875, + (11,0): 0, 1.37012, 2.73926, 4.11133, 5.47852, 6.85156, 8.21875, + (11,7): 9.59375, 10.957, 12.332, 13.6992, 15.0703, 16.4375, 17.8125, + (11,14): 19.1797, 20.5547, 21.9141, 23.2969, 24.6562, 26.0312, 27.3984, + (11,21): 28.7734, 30.1328, 31.5156, 32.875, 34.2656, 35.6094, 37, + (11,28): 38.3594, 39.7344, 41.0938, 42.4844, 43.8281, 45.2187, 46.5781, + (11,35): 47.9531, 49.3125, 50.7031, 52.0469, 53.4375, 54.7969, 56.1719, + (11,42): 57.5312, 58.9219, 60.2656, 61.6562, 63.0156, 64.4062, 65.75, + (11,49): 67.1562, 68.5, 69.875, 71.2188, 72.625, 73.9688, 75.375, + (11,56): 76.7188, 78.0937, 79.4375, 80.8437, 82.1875, 83.5937, 84.9375, + (11,63): 86.3125, + (12,0): 0, 1.50732, 3.01367, 4.52148, 6.02734, 7.53516, 9.03906, + (12,7): 10.5508, 12.0547, 13.5664, 15.0664, 16.5781, 18.0781, 19.5937, + (12,14): 21.0938, 22.6094, 24.1094, 25.625, 27.125, 28.6406, 30.1328, + (12,21): 31.6484, 33.1406, 34.6719, 36.1562, 37.6875, 39.1719, 40.7031, + (12,28): 42.1875, 43.7031, 45.2031, 46.7187, 48.2188, 49.7344, 51.2344, + (12,35): 52.75, 54.25, 55.7656, 57.2656, 58.7812, 60.2656, 61.7969, + (12,42): 63.2812, 64.8125, 66.2812, 67.8437, 69.3125, 70.8437, 72.3125, + (12,49): 73.8437, 75.3438, 76.875, 78.3438, 79.875, 81.375, 82.9062, + (12,56): 84.375, 85.9062, 87.375, 88.9375, 90.4062, 91.9375, 93.4062, + (12,63): 94.9687, + (13,0): 0, 1.64404, 3.28711, 4.93359, 6.57422, 8.22266, 9.86328, + (13,7): 11.5117, 13.1484, 14.7969, 16.4375, 18.0859, 19.7266, 21.375, + (13,14): 23.0156, 24.6641, 26.2969, 27.9531, 29.5859, 31.2422, 32.875, + (13,21): 34.5312, 36.1562, 37.8125, 39.4531, 41.1094, 42.7344, 44.3906, + (13,28): 46.0312, 47.6875, 49.3125, 50.9687, 52.5938, 54.2656, 55.8906, + (13,35): 57.5469, 59.1719, 60.8281, 62.4688, 64.125, 65.75, 67.4062, + (13,42): 69.0312, 70.7187, 72.3125, 74, 75.5938, 77.2812, 78.9062, + (13,49): 80.5625, 82.1875, 83.875, 85.4688, 87.1562, 88.75, 90.4375, + (13,56): 92.0625, 93.7187, 95.3438, 97, 98.625, 100.312, 101.906, + (13,63): 103.594, + (14,0): 0, 1.78125, 3.56152, 5.34375, 7.12305, 8.90625, 10.6836, + (14,7): 12.4687, 14.2461, 16.0312, 17.8047, 19.5937, 21.3672, 23.1562, + (14,14): 24.9297, 26.7187, 28.4922, 30.2812, 32.0469, 33.8437, 35.6094, + (14,21): 37.4062, 39.1719, 40.9687, 42.7344, 44.5312, 46.2969, 48.0937, + (14,28): 49.8594, 51.6562, 53.4219, 55.2187, 56.9844, 58.7812, 60.5469, + (14,35): 62.3437, 64.0938, 65.9062, 67.6562, 69.4687, 71.2188, 73.0312, + (14,42): 74.7812, 76.5937, 78.3438, 80.1562, 81.9062, 83.7187, 85.4688, + (14,49): 87.2812, 89.0312, 90.8437, 92.5938, 94.4062, 96.1562, 97.9687, + (14,56): 99.7188, 101.531, 103.281, 105.094, 106.844, 108.656, 110.406, + (14,63): 112.219, + (15,0): 0, 1.91846, 3.83594, 5.75586, 7.67188, 9.59375, 11.5078, + (15,7): 13.4297, 15.3438, 17.2656, 19.1797, 21.1016, 23.0156, 24.9375, + (15,14): 26.8516, 28.7734, 30.6875, 32.6094, 34.5156, 36.4531, 38.3594, + (15,21): 40.2812, 42.1875, 44.125, 46.0312, 47.9531, 49.8594, 51.7969, + (15,28): 53.7031, 55.625, 57.5312, 59.4687, 61.375, 63.2969, 65.1875, + (15,35): 67.1562, 69.0312, 70.9687, 72.875, 74.8125, 76.7188, 78.6562, + (15,42): 80.5312, 82.5, 84.375, 86.3125, 88.2188, 90.1562, 92.0625, 94, + (15,50): 95.875, 97.8437, 99.7188, 101.656, 103.562, 105.5, 107.406, + (15,57): 109.344, 111.219, 113.187, 115.062, 117, 118.906, 120.844, + (16,0): 0, 2.05566, 4.10938, 6.16602, 8.21875, 10.2773, 12.3281, + (16,7): 14.3867, 16.4375, 18.5, 20.5469, 22.6094, 24.6562, 26.7187, + (16,14): 28.7656, 30.8281, 32.875, 34.9375, 36.9844, 39.0469, 41.0938, + (16,21): 43.1562, 45.2031, 47.2656, 49.3125, 51.375, 53.4219, 55.5, + (16,28): 57.5312, 59.6094, 61.6406, 63.7187, 65.75, 67.8437, 69.8438, + (16,35): 71.9375, 73.9688, 76.0625, 78.0625, 80.1562, 82.1875, 84.2812, + (16,42): 86.2812, 88.375, 90.4062, 92.5, 94.5, 96.5937, 98.625, + (16,49): 100.719, 102.719, 104.812, 106.844, 108.937, 110.969, 113.031, + (16,56): 115.062, 117.156, 119.188, 121.25, 123.281, 125.375, 127.406, + (16,63): 129.5, + (17,0): 0, 2.19238, 4.38281, 6.57617, 8.76562, 10.9609, 13.1484, + (17,7): 15.3477, 17.5312, 19.7344, 21.9141, 24.1172, 26.2969, 28.5, + (17,14): 30.6875, 32.8906, 35.0625, 37.2656, 39.4531, 41.6562, 43.8281, + (17,21): 46.0469, 48.2188, 50.4219, 52.5938, 54.8125, 56.9844, 59.1875, + (17,28): 61.375, 63.5781, 65.75, 67.9687, 70.125, 72.3437, 74.5, 76.75, + (17,36): 78.9062, 81.125, 83.2812, 85.5, 87.6562, 89.875, 92.0625, + (17,43): 94.2812, 96.4375, 98.6562, 100.812, 103.031, 105.188, 107.437, + (17,50): 109.594, 111.812, 113.969, 116.187, 118.344, 120.562, 122.75, + (17,57): 124.969, 127.125, 129.375, 131.5, 133.75, 135.875, 138.125, + (18,0): 0, 2.3291, 4.65625, 6.98828, 9.3125, 11.6484, 13.9727, 16.3047, + (18,8): 18.625, 20.9687, 23.2891, 25.625, 27.9453, 30.2812, 32.5938, + (18,15): 34.9375, 37.25, 39.5937, 41.9219, 44.2656, 46.5781, 48.9219, + (18,22): 51.2344, 53.5781, 55.8906, 58.2344, 60.5469, 62.8906, 65.1875, + (18,29): 67.5625, 69.8438, 72.2187, 74.5, 76.875, 79.1562, 81.5312, + (18,36): 83.8438, 86.1875, 88.5, 90.8437, 93.1562, 95.5, 97.8125, + (18,43): 100.156, 102.469, 104.812, 107.125, 109.469, 111.781, 114.125, + (18,50): 116.438, 118.781, 121.094, 123.437, 125.75, 128.125, 130.375, + (18,57): 132.812, 135.062, 137.437, 139.688, 142.125, 144.375, 146.75, + (19,0): 0, 2.4668, 4.93164, 7.39844, 9.86328, 12.332, 14.793, 17.2656, + (19,8): 19.7266, 22.1953, 24.6562, 27.1328, 29.5859, 32.0625, 34.5156, + (19,15): 37, 39.4531, 41.9375, 44.375, 46.8594, 49.3125, 51.7969, + (19,22): 54.25, 56.7187, 59.1719, 61.6562, 64.0938, 66.5937, 69.0312, + (19,29): 71.5312, 73.9688, 76.4687, 78.9062, 81.4062, 83.8438, 86.3125, + (19,36): 88.75, 91.25, 93.6875, 96.1875, 98.625, 101.125, 103.562, + (19,43): 106.062, 108.5, 111, 113.406, 115.906, 118.344, 120.844, + (19,50): 123.281, 125.781, 128.188, 130.75, 133.125, 135.687, 138.062, + (19,57): 140.562, 143, 145.5, 147.938, 150.437, 152.875, 155.375, + (20,0): 0, 2.60352, 5.20508, 7.81055, 10.4102, 13.0156, 15.6172, + (20,7): 18.2266, 20.8203, 23.4297, 26.0234, 28.6406, 31.2344, 33.8437, + (20,14): 36.4375, 39.0469, 41.6406, 44.2656, 46.8438, 49.4687, 52.0469, + (20,21): 54.6719, 57.2656, 59.875, 62.4688, 65.0937, 67.6562, 70.2812, + (20,28): 72.875, 75.5, 78.0625, 80.7187, 83.2812, 85.9062, 88.5, + (20,35): 91.125, 93.6875, 96.3125, 98.9062, 101.531, 104.094, 106.75, + (20,42): 109.312, 111.937, 114.531, 117.156, 119.719, 122.344, 124.938, + (20,49): 127.562, 130.125, 132.812, 135.312, 138, 140.5, 143.187, + (20,56): 145.75, 148.375, 150.938, 153.625, 156.125, 158.812, 161.375, + (20,63): 164, + (21,0): 0, 2.74023, 5.47852, 8.22266, 10.957, 13.7031, 16.4375, + (21,7): 19.1875, 21.9141, 24.6641, 27.3984, 30.1406, 32.875, 35.625, + (21,14): 38.3594, 41.1094, 43.8281, 46.5937, 49.3125, 52.0625, 54.7969, + (21,21): 57.5469, 60.2656, 63.0312, 65.75, 68.5312, 71.2188, 74, + (21,28): 76.7188, 79.4687, 82.1875, 84.9687, 87.6562, 90.4375, 93.1562, + (21,35): 95.9062, 98.625, 101.406, 104.094, 106.875, 109.594, 112.344, + (21,42): 115.062, 117.844, 120.531, 123.312, 126.031, 128.812, 131.5, + (21,49): 134.312, 137, 139.75, 142.438, 145.25, 147.938, 150.75, + (21,56): 153.438, 156.187, 158.875, 161.687, 164.375, 167.187, 169.875, + (21,63): 172.625, + (22,0): 0, 2.87793, 5.75391, 8.63281, 11.5078, 14.3867, 17.2578, + (22,7): 20.1406, 23.0156, 25.8984, 28.7656, 31.6484, 34.5156, 37.4062, + (22,14): 40.2656, 43.1562, 46.0312, 48.9219, 51.7812, 54.6719, 57.5312, + (22,21): 60.4219, 63.2812, 66.1875, 69.0312, 71.9375, 74.7812, 77.6875, + (22,28): 80.5312, 83.4375, 86.2812, 89.1875, 92.0625, 94.9687, 97.8125, + (22,35): 100.719, 103.562, 106.469, 109.312, 112.219, 115.062, 117.969, + (22,42): 120.812, 123.719, 126.562, 129.5, 132.312, 135.25, 138.062, + (22,49): 141, 143.812, 146.75, 149.562, 152.5, 155.312, 158.25, + (22,56): 161.062, 164, 166.812, 169.75, 172.562, 175.5, 178.312, + (22,63): 181.312, + (23,0): 0, 3.01465, 6.02734, 9.04297, 12.0547, 15.0703, 18.0781, + (23,7): 21.1016, 24.1094, 27.1328, 30.1328, 33.1562, 36.1562, 39.1875, + (23,14): 42.1875, 45.2187, 48.2188, 51.25, 54.25, 57.2812, 60.2656, + (23,21): 63.2969, 66.2812, 69.3437, 72.3125, 75.375, 78.3438, 81.4062, + (23,28): 84.375, 87.4062, 90.4062, 93.4375, 96.4375, 99.4687, 102.469, + (23,35): 105.5, 108.5, 111.531, 114.531, 117.562, 120.531, 123.594, + (23,42): 126.562, 129.625, 132.562, 135.687, 138.625, 141.687, 144.625, + (23,49): 147.687, 150.688, 153.75, 156.688, 159.75, 162.75, 165.812, + (23,56): 168.75, 171.812, 174.75, 177.875, 180.812, 183.875, 186.812, + (23,63): 189.937, + (24,0): 0, 3.15137, 6.30078, 9.45312, 12.6016, 15.7578, 18.8984, + (24,7): 22.0625, 25.2031, 28.3594, 31.5078, 34.6719, 37.7969, 40.9687, + (24,14): 44.1094, 47.2656, 50.4062, 53.5781, 56.7031, 59.875, 63.0156, + (24,21): 66.1875, 69.3125, 72.5, 75.5938, 78.7812, 81.9062, 85.0937, + (24,28): 88.2188, 91.4062, 94.5, 97.6875, 100.812, 104, 107.125, + (24,35): 110.312, 113.406, 116.594, 119.719, 122.906, 126.031, 129.25, + (24,42): 132.312, 135.5, 138.625, 141.812, 144.938, 148.125, 151.188, + (24,49): 154.437, 157.5, 160.75, 163.812, 167.062, 170.125, 173.312, + (24,56): 176.438, 179.625, 182.75, 185.937, 189, 192.25, 195.312, + (24,63): 198.562, + (25,0): 0, 3.28809, 6.57422, 9.86719, 13.1484, 16.4453, 19.7266, + (25,7): 23.0234, 26.2969, 29.5937, 32.875, 36.1719, 39.4531, 42.75, + (25,14): 46.0312, 49.3281, 52.5938, 55.9062, 59.1719, 62.4844, 65.75, + (25,21): 69.0625, 72.3125, 75.625, 78.9062, 82.2187, 85.4688, 88.7812, + (25,28): 92.0625, 95.375, 98.625, 101.937, 105.188, 108.531, 111.781, + (25,35): 115.094, 118.344, 121.656, 124.938, 128.25, 131.5, 134.812, + (25,42): 138.062, 141.437, 144.625, 148, 151.188, 154.562, 157.812, + (25,49): 161.125, 164.375, 167.75, 170.938, 174.312, 177.5, 180.875, + (25,56): 184.125, 187.437, 190.688, 194, 197.25, 200.625, 203.812, + (25,63): 207.187, + (26,0): 0, 3.42578, 6.84961, 10.2773, 13.6992, 17.1328, 20.5469, + (26,7): 23.9766, 27.3984, 30.8281, 34.25, 37.6875, 41.0938, 44.5312, + (26,14): 47.9375, 51.375, 54.7969, 58.2344, 61.6406, 65.0937, 68.5, + (26,21): 71.9375, 75.3438, 78.7812, 82.1875, 85.6562, 89.0312, 92.5, + (26,28): 95.875, 99.3437, 102.719, 106.187, 109.594, 113.031, 116.438, + (26,35): 119.875, 123.281, 126.75, 130.125, 133.625, 137, 140.437, + (26,42): 143.812, 147.312, 150.688, 154.187, 157.5, 161, 164.375, + (26,49): 167.875, 171.25, 174.687, 178.062, 181.562, 184.938, 188.437, + (26,56): 191.75, 195.25, 198.625, 202.125, 205.438, 208.937, 212.312, + (26,63): 215.812, + (27,0): 0, 3.5625, 7.12305, 10.6875, 14.2461, 17.8125, 21.3672, + (27,7): 24.9375, 28.4922, 32.0625, 35.6094, 39.1875, 42.7344, 46.3125, + (27,14): 49.8594, 53.4375, 56.9844, 60.5625, 64.0938, 67.6875, 71.2188, + (27,21): 74.8125, 78.3438, 81.9375, 85.4688, 89.0625, 92.5938, 96.1875, + (27,28): 99.7188, 103.312, 106.844, 110.437, 113.969, 117.562, 121.094, + (27,35): 124.687, 128.188, 131.812, 135.312, 138.937, 142.438, 146.062, + (27,42): 149.562, 153.187, 156.688, 160.312, 163.812, 167.437, 170.938, + (27,49): 174.562, 178.062, 181.687, 185.188, 188.812, 192.312, 195.937, + (27,56): 199.438, 203.062, 206.562, 210.187, 213.688, 217.312, 220.812, + (27,63): 224.437, + (28,0): 0, 3.69922, 7.39648, 11.0977, 14.793, 18.5, 22.1875, 25.8984, + (28,8): 29.5859, 33.2969, 36.9844, 40.7031, 44.375, 48.0937, 51.7812, + (28,15): 55.5, 59.1719, 62.8906, 66.5625, 70.2812, 73.9688, 77.6875, + (28,22): 81.375, 85.0937, 88.75, 92.5, 96.1562, 99.875, 103.562, + (28,29): 107.281, 110.969, 114.687, 118.344, 122.094, 125.75, 129.5, + (28,36): 133.125, 136.875, 140.5, 144.312, 147.938, 151.687, 155.312, + (28,43): 159.062, 162.75, 166.5, 170.125, 173.875, 177.5, 181.312, + (28,50): 184.938, 188.687, 192.312, 196.062, 199.688, 203.5, 207.125, + (28,57): 210.875, 214.5, 218.25, 221.938, 225.687, 229.312, 233.062, + (29,0): 0, 3.83691, 7.67188, 11.5117, 15.3438, 19.1875, 23.0156, + (29,7): 26.8594, 30.6875, 34.5312, 38.3594, 42.2031, 46.0312, 49.875, + (29,14): 53.7031, 57.5469, 61.375, 65.2187, 69.0312, 72.9062, 76.7188, + (29,21): 80.5625, 84.375, 88.25, 92.0625, 95.9062, 99.7188, 103.594, + (29,28): 107.406, 111.25, 115.062, 118.937, 122.75, 126.594, 130.375, + (29,35): 134.312, 138.062, 141.937, 145.75, 149.625, 153.438, 157.312, + (29,42): 161.062, 165, 168.75, 172.625, 176.438, 180.312, 184.125, 188, + (29,50): 191.75, 195.687, 199.438, 203.312, 207.125, 211, 214.812, + (29,57): 218.687, 222.438, 226.375, 230.125, 234, 237.812, 241.687, + (30,0): 0, 3.97363, 7.94531, 11.9219, 15.8906, 19.8672, 23.8359, + (30,7): 27.8125, 31.7812, 35.7656, 39.7188, 43.7031, 47.6719, 51.6562, + (30,14): 55.6094, 59.6094, 63.5625, 67.5625, 71.5, 75.5, 79.4375, + (30,21): 83.4375, 87.375, 91.4062, 95.3438, 99.3437, 103.281, 107.281, + (30,28): 111.219, 115.219, 119.188, 123.187, 127.125, 131.125, 135.062, + (30,35): 139.062, 143, 147.062, 150.938, 155, 158.875, 162.937, + (30,42): 166.812, 170.875, 174.75, 178.812, 182.75, 186.75, 190.688, + (30,49): 194.687, 198.625, 202.625, 206.562, 210.625, 214.5, 218.562, + (30,56): 222.438, 226.5, 230.375, 234.437, 238.375, 242.375, 246.312, + (30,63): 250.312, + (31,0): 0, 4.11133, 8.21875, 12.332, 16.4375, 20.5547, 24.6562, + (31,7): 28.7734, 32.875, 37, 41.0938, 45.2187, 49.3125, 53.4375, + (31,14): 57.5312, 61.6562, 65.75, 69.875, 73.9688, 78.0937, 82.1875, + (31,21): 86.3125, 90.4062, 94.5312, 98.625, 102.75, 106.844, 111, + (31,28): 115.062, 119.219, 123.281, 127.437, 131.5, 135.687, 139.688, + (31,35): 143.875, 147.938, 152.125, 156.125, 160.312, 164.375, 168.562, + (31,42): 172.562, 176.75, 180.812, 185, 189, 193.187, 197.25, 201.437, + (31,50): 205.438, 209.625, 213.688, 217.875, 221.938, 226.062, 230.125, + (31,57): 234.312, 238.375, 242.5, 246.562, 250.75, 254.812, 259 + } + } +} +} diff --git a/BITGROOM/example/testfiles/h5repack_floats.h5-ud_convert.ddl b/BITGROOM/example/testfiles/h5repack_floats.h5-ud_convert.ddl new file mode 100644 index 000000000..f4d34f36c --- /dev/null +++ b/BITGROOM/example/testfiles/h5repack_floats.h5-ud_convert.ddl @@ -0,0 +1,26 @@ +HDF5 "out-ud_convert.h5repack_floats.h5" { +GROUP "/" { + DATASET "data" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + STORAGE_LAYOUT { + CHUNKED ( 4, 8 ) + SIZE 8192 (1.000:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32022 + COMMENT BitGroom filter (Zender, 2016 GMD: http://www.geosci-model-dev.net/9/3199/2016) + PARAMS { 3 4 0 0 0 } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } +} +} diff --git a/BITGROOM/example/testfiles/h5repack_layout.h5 b/BITGROOM/example/testfiles/h5repack_layout.h5 deleted file mode 100644 index 2cc44b140..000000000 Binary files a/BITGROOM/example/testfiles/h5repack_layout.h5 and /dev/null differ diff --git a/BITGROOM/example/testfiles/h5repack_layout.h5-ud_convert.ddl b/BITGROOM/example/testfiles/h5repack_layout.h5-ud_convert.ddl deleted file mode 100644 index dd4ae1308..000000000 --- a/BITGROOM/example/testfiles/h5repack_layout.h5-ud_convert.ddl +++ /dev/null @@ -1,130 +0,0 @@ -HDF5 "out-ud_convert.h5repack_layout.h5" { -GROUP "/" { - DATASET "dset1" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset2" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset3" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset4" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_chunk" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, H5S_UNLIMITED ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_compact" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_contiguous" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } -} -} diff --git a/BITGROOM/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst b/BITGROOM/example/testfiles/ud_convert.h5repack_floats.h5.tst similarity index 53% rename from BITGROOM/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst rename to BITGROOM/example/testfiles/ud_convert.h5repack_floats.h5.tst index afcd00550..e658a2828 100644 --- a/BITGROOM/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst +++ b/BITGROOM/example/testfiles/ud_convert.h5repack_floats.h5.tst @@ -7,10 +7,4 @@ Making new file ... Type Filter (Compression) Name ----------------------------------------- group / - dset UD (0.833:1) /dset1 - dset UD (0.833:1) /dset2 - dset UD (0.833:1) /dset3 - dset UD (0.833:1) /dset4 - dset UD (0.833:1) /dset_chunk - dset UD (0.833:1) /dset_compact - dset UD (0.833:1) /dset_contiguous + dset UD (1.000:1) /data diff --git a/BITROUND/config/cmake/binex/example/CMakeLists.txt b/BITROUND/config/cmake/binex/example/CMakeLists.txt index e5ece574a..807ff361c 100644 --- a/BITROUND/config/cmake/binex/example/CMakeLists.txt +++ b/BITROUND/config/cmake/binex/example/CMakeLists.txt @@ -34,6 +34,17 @@ foreach (example ${dyn_examples}) endforeach () if (H5PL_BUILD_TESTING) + # h5repack test input generator. Target name is plugin-suffixed because the + # BITROUND and BITGROOM trees share the same CMake project and source file + # basename (issue #240). + add_executable (h5repack_floats_bitround ${PROJECT_SOURCE_DIR}/h5repack_floats.c) + target_include_directories(h5repack_floats_bitround PRIVATE ${H5PL_HDF5_INCLUDE_DIRS}) + TARGET_C_PROPERTIES (h5repack_floats_bitround ${LIB_TYPE}) + target_link_libraries (h5repack_floats_bitround PRIVATE ${H5PL_HDF5_LINK_LIBS}) + if (NOT WIN32) + target_link_libraries (h5repack_floats_bitround PRIVATE ${CMAKE_DL_LIBS}) + endif () + macro (ADD_H5_TEST testname) add_test ( NAME ${testname}-clearall @@ -127,21 +138,47 @@ if (H5PL_BUILD_TESTING) NAME H5BITROUND_UD-${testname}-clearall-objects COMMAND ${CMAKE_COMMAND} -E remove + ${resultfile} out-${testname}.${resultfile} ${testname}.${resultfile}.out ${testname}.${resultfile}.out.err ${resultfile}-${testname}.out ${resultfile}-${testname}.out.err + ${resultfile}-${testname}-data.out + ${resultfile}-${testname}-data.out.err ) if (NOT "${last_test}" STREQUAL "") set_tests_properties (H5BITROUND_UD-${testname}-clearall-objects PROPERTIES DEPENDS ${last_test}) endif () set (last_test "H5BITROUND_UD-${testname}-clearall-objects") if (WIN32) - set (${TESTLIBDIR} "${HDF5_TOOLS_DIR}") + set (TESTLIBDIR "${HDF5_TOOLS_DIR}") else () - set (${TESTLIBDIR} "${HDF5_LIBRARY_PATH}") + set (TESTLIBDIR "${HDF5_LIBRARY_PATH}") endif () + # Generate the float-data input file for h5repack. The Granular BitRound + # filter's set_local() callback removes the filter from the pipeline on + # non-float datasets, so the input must contain float data for the test + # to actually exercise the filter (issue #240). Routed through + # runTest.cmake so PATH/DYLD_LIBRARY_PATH/LD_LIBRARY_PATH are set up for + # the helper to find the HDF5 runtime library on every platform + # (Windows in particular has no rpath equivalent and reports + # STATUS_DLL_NOT_FOUND if the HDF5 DLL is not on PATH). + add_test ( + NAME H5BITROUND_UD-${testname}-gen-input + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_EXPECT=0" + -D "TEST_OUTPUT=h5repack_floats_bitround.out" + -D "TEST_SKIP_COMPARE=1" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -P "${H5PL_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5BITROUND_UD-${testname}-gen-input PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS H5BITROUND_UD-${testname}-clearall-objects) + set (last_test "H5BITROUND_UD-${testname}-gen-input") if (HDF5_BUILD_TOOLS OR HDF5_PROVIDES_TOOLS) add_test ( NAME H5BITROUND_UD-${testname} @@ -156,7 +193,7 @@ if (H5PL_BUILD_TESTING) -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" -P "${H5PL_RESOURCES_DIR}/runTest.cmake" ) - set_tests_properties (H5BITROUND_UD-${testname} PROPERTIES DEPENDS H5BITROUND_UD-${testname}-clearall-objects) + set_tests_properties (H5BITROUND_UD-${testname} PROPERTIES DEPENDS H5BITROUND_UD-${testname}-gen-input) add_test ( NAME H5BITROUND_UD-h5dump-${testname} COMMAND "${CMAKE_COMMAND}" @@ -173,23 +210,43 @@ if (H5PL_BUILD_TESTING) WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" DEPENDS H5BITROUND_UD-${testname}) set (last_test "H5BITROUND_UD-h5dump-${testname}") + # Verify that the filter actually modifies the dataset values: dump the + # data (no -pH) and compare against a captured reference of the + # quantized output. Granular BitRound's encoding was made bit- + # reproducible across platforms by replacing the libm log10()/floor() + # chain in ccr_gbr() with a deterministic power-of-10 table lookup + # (see comment in src/H5Zgranularbr.c above ccr_gbr_dgt_nbr()). + add_test ( + NAME H5BITROUND_UD-h5dump-data-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5PL_HDF5_DUMP_EXECUTABLE}" + -D "TEST_ARGS:STRING=out-${testname}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_OUTPUT=${resultfile}-${testname}-data.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}-${testname}-data.ddl" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -P "${H5PL_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5BITROUND_UD-h5dump-data-${testname} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS H5BITROUND_UD-h5dump-${testname}) + set (last_test "H5BITROUND_UD-h5dump-data-${testname}") endif () endmacro () # -------------------------------------------------------------------- # Copy all the HDF5 files from the source directory into the test directory # -------------------------------------------------------------------- - set (LIST_HDF5_TEST_FILES - h5repack_layout.h5 - ) set (LIST_OTHER_TEST_FILES h5ex_d_granularbr.ddl h5ex_d_granularbr.tst - h5repack_layout.h5-ud_convert.ddl - ud_convert.h5repack_layout.h5.tst + h5repack_floats.h5-ud_convert.ddl + h5repack_floats.h5-ud_convert-data.ddl + ud_convert.h5repack_floats.h5.tst ) - foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) + foreach (h5_file ${LIST_OTHER_TEST_FILES}) HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/${h5_file}" "example_files") endforeach () add_custom_target(example_files ALL COMMENT "Copying files needed by example tests" DEPENDS ${example_files_list}) @@ -198,7 +255,7 @@ if (H5PL_BUILD_TESTING) if (NOT DISABLE_H5PL_ENCODER) #UD BITROUND - ADD_H5_UD_TEST (ud_convert 0 h5repack_layout.h5 --enable-error-stack -v -f UD=32023,0,5,3,4,0,0,0 -l CHUNK=4x8) + ADD_H5_UD_TEST (ud_convert 0 h5repack_floats.h5 --enable-error-stack -v -f UD=32023,0,5,3,4,0,0,0 -l CHUNK=4x8) endif () endif () diff --git a/BITROUND/config/cmake/binex/example/h5repack_floats.c b/BITROUND/config/cmake/binex/example/h5repack_floats.c new file mode 100644 index 000000000..b01cc161e --- /dev/null +++ b/BITROUND/config/cmake/binex/example/h5repack_floats.c @@ -0,0 +1,87 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of the HDF5 Granular BitRound filter plugin source. * + * The full copyright notice, including terms governing use, modification, * + * and redistribution, is contained in the file COPYING, which can be found * + * at the root of the source code distribution tree. If you do not have * + * access to this file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Generates an HDF5 file with a chunked dataset of IEEE 32-bit floats and no + * filters applied. Used as input for the h5repack Granular BitRound test so + * the filter's set_local() callback retains the filter in the pipeline (the + * callback removes itself on non-float data). See issue #240. + */ + +#include "hdf5.h" +#include + +#define FILENAME "h5repack_floats.h5" +#define DATASET "data" +#define DIM0 32 +#define DIM1 64 +#define CHUNK0 4 +#define CHUNK1 8 + +int +main(void) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t space_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + hsize_t dims[2] = {DIM0, DIM1}; + hsize_t chunk[2] = {CHUNK0, CHUNK1}; + float wdata[DIM0][DIM1]; + int i, j; + int ret_value = 1; + + /* Non-integer floats: the 0.137f multiplier ensures every value carries + * non-trivial mantissa bits, so quantization by Granular BitRound is + * visible across the entire dataset rather than only on values that + * exceed the retained mantissa width. */ + for (i = 0; i < DIM0; i++) + for (j = 0; j < DIM1; j++) + wdata[i][j] = ((float)(i * j) - (float)j) * 0.137f; + + file_id = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if (file_id < 0) + goto done; + + space_id = H5Screate_simple(2, dims, NULL); + if (space_id < 0) + goto done; + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + if (dcpl_id < 0) + goto done; + if (H5Pset_chunk(dcpl_id, 2, chunk) < 0) + goto done; + + dset_id = H5Dcreate(file_id, DATASET, H5T_IEEE_F32LE, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + if (dset_id < 0) + goto done; + + if (H5Dwrite(dset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void *)wdata) < 0) + goto done; + + ret_value = 0; + +done: + if (dset_id >= 0) + H5Dclose(dset_id); + if (dcpl_id >= 0) + H5Pclose(dcpl_id); + if (space_id >= 0) + H5Sclose(space_id); + if (file_id >= 0) + H5Fclose(file_id); + + if (ret_value != 0) + fprintf(stderr, "h5repack_floats: failed to create %s\n", FILENAME); + + return ret_value; +} diff --git a/BITROUND/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl b/BITROUND/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl new file mode 100644 index 000000000..22d6e9129 --- /dev/null +++ b/BITROUND/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl @@ -0,0 +1,270 @@ +HDF5 "out-ud_convert.h5repack_floats.h5" { +GROUP "/" { + DATASET "data" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + DATA { + (0,0): 0, -0.136719, -0.274414, -0.411133, -0.547852, -0.68457, + (0,6): -0.822266, -0.958984, -1.09375, -1.23438, -1.36719, -1.50781, + (0,12): -1.64062, -1.78125, -1.92188, -2.05469, -2.19531, -2.32812, + (0,18): -2.46875, -2.60156, -2.74219, -2.875, -3.01562, -3.14844, + (0,24): -3.28906, -3.42188, -3.5625, -3.69531, -3.83594, -3.97656, + (0,30): -4.10938, -4.25, -4.38281, -4.52344, -4.65625, -4.79688, + (0,36): -4.92969, -5.07031, -5.20312, -5.34375, -5.47656, -5.61719, + (0,42): -5.75781, -5.89062, -6.03125, -6.16406, -6.30469, -6.4375, + (0,48): -6.57812, -6.71094, -6.85156, -6.98438, -7.125, -7.25781, + (0,54): -7.39844, -7.53125, -7.67188, -7.8125, -7.94531, -8.08594, + (0,60): -8.21875, -8.35938, -8.49219, -8.63281, + (1,0): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,21): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,42): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,63): 0, + (2,0): 0, 0.136719, 0.274414, 0.411133, 0.547852, 0.68457, 0.822266, + (2,7): 0.958984, 1.09375, 1.23438, 1.36719, 1.50781, 1.64062, 1.78125, + (2,14): 1.92188, 2.05469, 2.19531, 2.32812, 2.46875, 2.60156, 2.74219, + (2,21): 2.875, 3.01562, 3.14844, 3.28906, 3.42188, 3.5625, 3.69531, + (2,28): 3.83594, 3.97656, 4.10938, 4.25, 4.38281, 4.52344, 4.65625, + (2,35): 4.79688, 4.92969, 5.07031, 5.20312, 5.34375, 5.47656, 5.61719, + (2,42): 5.75781, 5.89062, 6.03125, 6.16406, 6.30469, 6.4375, 6.57812, + (2,49): 6.71094, 6.85156, 6.98438, 7.125, 7.25781, 7.39844, 7.53125, + (2,56): 7.67188, 7.8125, 7.94531, 8.08594, 8.21875, 8.35938, 8.49219, + (2,63): 8.63281, + (3,0): 0, 0.274414, 0.547852, 0.822266, 1.09375, 1.36719, 1.64062, + (3,7): 1.92188, 2.19531, 2.46875, 2.74219, 3.01562, 3.28906, 3.5625, + (3,14): 3.83594, 4.10938, 4.38281, 4.65625, 4.92969, 5.20312, 5.47656, + (3,21): 5.75781, 6.03125, 6.30469, 6.57812, 6.85156, 7.125, 7.39844, + (3,28): 7.67188, 7.94531, 8.21875, 8.49219, 8.76562, 9.03906, 9.3125, + (3,35): 9.59375, 9.86719, 10.125, 10.4375, 10.6875, 10.9375, 11.25, + (3,42): 11.5, 11.8125, 12.0625, 12.3125, 12.625, 12.875, 13.125, + (3,49): 13.4375, 13.6875, 14, 14.25, 14.5, 14.8125, 15.0625, 15.375, + (3,57): 15.625, 15.875, 16.1875, 16.4375, 16.6875, 17, 17.25, + (4,0): 0, 0.411133, 0.822266, 1.23438, 1.64062, 2.05469, 2.46875, + (4,7): 2.875, 3.28906, 3.69531, 4.10938, 4.52344, 4.92969, 5.34375, + (4,14): 5.75781, 6.16406, 6.57812, 6.98438, 7.39844, 7.8125, 8.21875, + (4,21): 8.63281, 9.03906, 9.45312, 9.86719, 10.25, 10.6875, 11.125, + (4,28): 11.5, 11.9375, 12.3125, 12.75, 13.125, 13.5625, 14, 14.375, + (4,36): 14.8125, 15.1875, 15.625, 16, 16.4375, 16.875, 17.25, 17.6875, + (4,44): 18.0625, 18.5, 18.875, 19.3125, 19.75, 20.125, 20.5625, + (4,51): 20.9375, 21.375, 21.8125, 22.1875, 22.625, 23, 23.4375, + (4,58): 23.8125, 24.25, 24.6875, 25.0625, 25.5, 25.875, + (5,0): 0, 0.547852, 1.09375, 1.64062, 2.19531, 2.74219, 3.28906, + (5,7): 3.83594, 4.38281, 4.92969, 5.47656, 6.03125, 6.57812, 7.125, + (5,14): 7.67188, 8.21875, 8.76562, 9.3125, 9.86719, 10.4375, 10.9375, + (5,21): 11.5, 12.0625, 12.625, 13.125, 13.6875, 14.25, 14.8125, 15.375, + (5,29): 15.875, 16.4375, 17, 17.5625, 18.0625, 18.625, 19.1875, 19.75, + (5,37): 20.25, 20.8125, 21.375, 21.9375, 22.4375, 23, 23.5625, 24.125, + (5,45): 24.6875, 25.1875, 25.75, 26.3125, 26.875, 27.375, 27.9375, + (5,52): 28.5, 29.0625, 29.5625, 30.125, 30.6875, 31.25, 31.8125, + (5,59): 32.3125, 32.875, 33.4375, 34, 34.5, + (6,0): 0, 0.68457, 1.36719, 2.05469, 2.74219, 3.42188, 4.10938, + (6,7): 4.79688, 5.47656, 6.16406, 6.85156, 7.53125, 8.21875, 8.90625, + (6,14): 9.59375, 10.25, 10.9375, 11.625, 12.3125, 13, 13.6875, 14.375, + (6,22): 15.0625, 15.75, 16.4375, 17.125, 17.8125, 18.5, 19.1875, + (6,29): 19.875, 20.5625, 21.25, 21.9375, 22.625, 23.3125, 24, 24.6875, + (6,37): 25.375, 26, 26.6875, 27.375, 28.0625, 28.75, 29.4375, 30.125, + (6,45): 30.8125, 31.5, 32.1875, 32.875, 33.5625, 34.25, 34.9375, + (6,52): 35.625, 36.3125, 37, 37.6875, 38.375, 39.0625, 39.75, 40.4375, + (6,60): 41.125, 41.8125, 42.5, 43.125, + (7,0): 0, 0.822266, 1.64062, 2.46875, 3.28906, 4.10938, 4.92969, + (7,7): 5.75781, 6.57812, 7.39844, 8.21875, 9.03906, 9.86719, 10.6875, + (7,14): 11.5, 12.3125, 13.125, 14, 14.8125, 15.625, 16.4375, 17.25, + (7,22): 18.0625, 18.875, 19.75, 20.5625, 21.375, 22.1875, 23, 23.8125, + (7,30): 24.6875, 25.5, 26.3125, 27.125, 27.9375, 28.75, 29.5625, + (7,37): 30.4375, 31.25, 32.0625, 32.875, 33.6875, 34.5, 35.375, + (7,44): 36.1875, 37, 37.8125, 38.625, 39.4375, 40.25, 41.125, 41.9375, + (7,52): 42.75, 43.5625, 44.375, 45.1875, 46.0625, 46.875, 47.6875, + (7,59): 48.5, 49.3125, 50.125, 50.9375, 51.8125, + (8,0): 0, 0.958984, 1.92188, 2.875, 3.83594, 4.79688, 5.75781, 6.71094, + (8,8): 7.67188, 8.63281, 9.59375, 10.5625, 11.5, 12.4375, 13.4375, + (8,15): 14.375, 15.375, 16.3125, 17.25, 18.25, 19.1875, 20.125, 21.125, + (8,23): 22.0625, 23, 24, 24.9375, 25.875, 26.875, 27.8125, 28.75, + (8,31): 29.75, 30.6875, 31.625, 32.625, 33.5625, 34.5, 35.5, 36.4375, + (8,39): 37.375, 38.375, 39.3125, 40.25, 41.25, 42.1875, 43.125, 44.125, + (8,47): 45.0625, 46.0625, 47, 47.9375, 48.9375, 49.875, 50.8125, + (8,54): 51.8125, 52.75, 53.6875, 54.6875, 55.625, 56.5625, 57.5625, + (8,61): 58.5, 59.4375, 60.4375, + (9,0): 0, 1.09375, 2.19531, 3.28906, 4.38281, 5.47656, 6.57812, + (9,7): 7.67188, 8.76562, 9.86719, 10.9375, 12.0625, 13.125, 14.25, + (9,14): 15.375, 16.4375, 17.5625, 18.625, 19.75, 20.8125, 21.9375, 23, + (9,22): 24.125, 25.1875, 26.3125, 27.375, 28.5, 29.5625, 30.6875, + (9,29): 31.8125, 32.875, 34, 35.0625, 36.1875, 37.25, 38.375, 39.4375, + (9,37): 40.5625, 41.625, 42.75, 43.8125, 44.9375, 46.0625, 47.125, + (9,44): 48.25, 49.3125, 50.4375, 51.5, 52.625, 53.6875, 54.8125, + (9,51): 55.875, 57, 58.0625, 59.1875, 60.25, 61.375, 62.5, 63.5625, + (9,59): 64.6875, 65.75, 66.875, 67.9375, 69.0625, + (10,0): 0, 1.23438, 2.46875, 3.69531, 4.92969, 6.16406, 7.39844, + (10,7): 8.63281, 9.86719, 11.125, 12.3125, 13.5625, 14.8125, 16, 17.25, + (10,15): 18.5, 19.75, 20.9375, 22.1875, 23.4375, 24.6875, 25.875, + (10,22): 27.125, 28.375, 29.5625, 30.8125, 32.0625, 33.3125, 34.5, + (10,29): 35.75, 37, 38.25, 39.4375, 40.6875, 41.9375, 43.125, 44.375, + (10,37): 45.625, 46.875, 48.0625, 49.3125, 50.5625, 51.8125, 53, 54.25, + (10,45): 55.5, 56.6875, 57.9375, 59.1875, 60.4375, 61.625, 62.875, + (10,52): 64.125, 65.375, 66.5625, 67.8125, 69.0625, 70.25, 71.5, 72.75, + (10,60): 74, 75.1875, 76.4375, 77.6875, + (11,0): 0, 1.36719, 2.74219, 4.10938, 5.47656, 6.85156, 8.21875, + (11,7): 9.59375, 10.9375, 12.3125, 13.6875, 15.0625, 16.4375, 17.8125, + (11,14): 19.1875, 20.5625, 21.9375, 23.3125, 24.6875, 26, 27.375, + (11,21): 28.75, 30.125, 31.5, 32.875, 34.25, 35.625, 37, 38.375, 39.75, + (11,30): 41.125, 42.5, 43.8125, 45.1875, 46.5625, 47.9375, 49.3125, + (11,37): 50.6875, 52.0625, 53.4375, 54.8125, 56.1875, 57.5625, 58.9375, + (11,44): 60.25, 61.625, 63, 64.375, 65.75, 67.125, 68.5, 69.875, 71.25, + (11,53): 72.625, 74, 75.375, 76.75, 78.0625, 79.4375, 80.8125, 82.1875, + (11,61): 83.5625, 84.9375, 86.3125, + (12,0): 0, 1.50781, 3.01562, 4.52344, 6.03125, 7.53125, 9.03906, + (12,7): 10.5625, 12.0625, 13.5625, 15.0625, 16.5625, 18.0625, 19.5625, + (12,14): 21.125, 22.625, 24.125, 25.625, 27.125, 28.625, 30.125, + (12,21): 31.625, 33.125, 34.6875, 36.1875, 37.6875, 39.1875, 40.6875, + (12,28): 42.1875, 43.6875, 45.1875, 46.6875, 48.25, 49.75, 51.25, + (12,35): 52.75, 54.25, 55.75, 57.25, 58.75, 60.25, 61.8125, 63.3125, + (12,43): 64.8125, 66.3125, 67.8125, 69.3125, 70.8125, 72.3125, 73.8125, + (12,50): 75.375, 76.875, 78.375, 79.875, 81.375, 82.875, 84.375, + (12,57): 85.875, 87.375, 88.9375, 90.4375, 91.9375, 93.4375, 94.9375, + (13,0): 0, 1.64062, 3.28906, 4.92969, 6.57812, 8.21875, 9.86719, 11.5, + (13,8): 13.125, 14.8125, 16.4375, 18.0625, 19.75, 21.375, 23, 24.6875, + (13,16): 26.3125, 27.9375, 29.5625, 31.25, 32.875, 34.5, 36.1875, + (13,23): 37.8125, 39.4375, 41.125, 42.75, 44.375, 46.0625, 47.6875, + (13,30): 49.3125, 50.9375, 52.625, 54.25, 55.875, 57.5625, 59.1875, + (13,37): 60.8125, 62.5, 64.125, 65.75, 67.375, 69.0625, 70.6875, + (13,44): 72.3125, 74, 75.625, 77.25, 78.9375, 80.5625, 82.1875, 83.875, + (13,52): 85.5, 87.125, 88.75, 90.4375, 92.0625, 93.6875, 95.375, 97, + (13,60): 98.625, 100, 102, 104, + (14,0): 0, 1.78125, 3.5625, 5.34375, 7.125, 8.90625, 10.6875, 12.4375, + (14,8): 14.25, 16, 17.8125, 19.5625, 21.375, 23.125, 24.9375, 26.6875, + (14,16): 28.5, 30.25, 32.0625, 33.8125, 35.625, 37.375, 39.1875, + (14,23): 40.9375, 42.75, 44.5, 46.3125, 48.0625, 49.875, 51.625, + (14,30): 53.4375, 55.1875, 57, 58.75, 60.5625, 62.3125, 64.125, 65.875, + (14,38): 67.6875, 69.4375, 71.25, 73, 74.8125, 76.5625, 78.375, 80.125, + (14,46): 81.9375, 83.6875, 85.5, 87.25, 89.0625, 90.8125, 92.625, + (14,53): 94.375, 96.1875, 97.9375, 99.75, 102, 103, 105, 107, 109, 110, + (14,63): 112, + (15,0): 0, 1.92188, 3.83594, 5.75781, 7.67188, 9.59375, 11.5, 13.4375, + (15,8): 15.375, 17.25, 19.1875, 21.125, 23, 24.9375, 26.875, 28.75, + (15,16): 30.6875, 32.625, 34.5, 36.4375, 38.375, 40.25, 42.1875, + (15,23): 44.125, 46.0625, 47.9375, 49.875, 51.8125, 53.6875, 55.625, + (15,30): 57.5625, 59.4375, 61.375, 63.3125, 65.1875, 67.125, 69.0625, + (15,37): 70.9375, 72.875, 74.8125, 76.75, 78.625, 80.5625, 82.5, + (15,44): 84.375, 86.3125, 88.25, 90.125, 92.0625, 94, 95.875, 97.8125, + (15,52): 99.75, 102, 104, 105, 107, 109, 111, 113, 115, 117, 119, 121, + (16,0): 0, 2.05469, 4.10938, 6.16406, 8.21875, 10.25, 12.3125, 14.375, + (16,8): 16.4375, 18.5, 20.5625, 22.625, 24.6875, 26.6875, 28.75, + (16,15): 30.8125, 32.875, 34.9375, 37, 39.0625, 41.125, 43.125, + (16,22): 45.1875, 47.25, 49.3125, 51.375, 53.4375, 55.5, 57.5625, + (16,29): 59.625, 61.625, 63.6875, 65.75, 67.8125, 69.875, 71.9375, 74, + (16,37): 76.0625, 78.0625, 80.125, 82.1875, 84.25, 86.3125, 88.375, + (16,44): 90.4375, 92.5, 94.5, 96.5625, 98.625, 101, 103, 105, 107, 109, + (16,54): 111, 113, 115, 117, 119, 121, 123, 125, 127, 129, + (17,0): 0, 2.19531, 4.38281, 6.57812, 8.76562, 10.9375, 13.125, 15.375, + (17,8): 17.5625, 19.75, 21.9375, 24.125, 26.3125, 28.5, 30.6875, + (17,15): 32.875, 35.0625, 37.25, 39.4375, 41.625, 43.8125, 46.0625, + (17,22): 48.25, 50.4375, 52.625, 54.8125, 57, 59.1875, 61.375, 63.5625, + (17,30): 65.75, 67.9375, 70.125, 72.3125, 74.5, 76.75, 78.9375, 81.125, + (17,38): 83.3125, 85.5, 87.6875, 89.875, 92.0625, 94.25, 96.4375, + (17,45): 98.625, 101, 103, 105, 107, 110, 112, 114, 116, 118, 121, 123, + (17,57): 125, 127, 129, 132, 134, 136, 138, + (18,0): 0, 2.32812, 4.65625, 6.98438, 9.3125, 11.625, 14, 16.3125, + (18,8): 18.625, 20.9375, 23.3125, 25.625, 27.9375, 30.25, 32.625, + (18,15): 34.9375, 37.25, 39.5625, 41.9375, 44.25, 46.5625, 48.9375, + (18,22): 51.25, 53.5625, 55.875, 58.25, 60.5625, 62.875, 65.1875, + (18,29): 67.5625, 69.875, 72.1875, 74.5, 76.875, 79.1875, 81.5, 83.875, + (18,37): 86.1875, 88.5, 90.8125, 93.1875, 95.5, 97.8125, 100, 102, 105, + (18,46): 107, 109, 112, 114, 116, 119, 121, 123, 126, 128, 130, 133, + (18,58): 135, 137, 140, 142, 144, 147, + (19,0): 0, 2.46875, 4.92969, 7.39844, 9.86719, 12.3125, 14.8125, 17.25, + (19,8): 19.75, 22.1875, 24.6875, 27.125, 29.5625, 32.0625, 34.5, 37, + (19,16): 39.4375, 41.9375, 44.375, 46.875, 49.3125, 51.8125, 54.25, + (19,23): 56.6875, 59.1875, 61.625, 64.125, 66.5625, 69.0625, 71.5, 74, + (19,31): 76.4375, 78.9375, 81.375, 83.875, 86.3125, 88.75, 91.25, + (19,38): 93.6875, 96.1875, 98.625, 101, 104, 106, 109, 111, 113, 116, + (19,48): 118, 121, 123, 126, 128, 131, 133, 136, 138, 141, 143, 145, + (19,60): 148, 150, 153, 155, + (20,0): 0, 2.60156, 5.20312, 7.8125, 10.4375, 13, 15.625, 18.25, + (20,8): 20.8125, 23.4375, 26, 28.625, 31.25, 33.8125, 36.4375, 39.0625, + (20,16): 41.625, 44.25, 46.875, 49.4375, 52.0625, 54.6875, 57.25, + (20,23): 59.875, 62.5, 65.0625, 67.6875, 70.25, 72.875, 75.5, 78.0625, + (20,31): 80.6875, 83.3125, 85.875, 88.5, 91.125, 93.6875, 96.3125, + (20,38): 98.9375, 102, 104, 107, 109, 112, 115, 117, 120, 122, 125, + (20,49): 128, 130, 133, 135, 138, 141, 143, 146, 148, 151, 154, 156, + (20,61): 159, 161, 164, + (21,0): 0, 2.74219, 5.47656, 8.21875, 10.9375, 13.6875, 16.4375, + (21,7): 19.1875, 21.9375, 24.6875, 27.375, 30.125, 32.875, 35.625, + (21,14): 38.375, 41.125, 43.8125, 46.5625, 49.3125, 52.0625, 54.8125, + (21,21): 57.5625, 60.25, 63, 65.75, 68.5, 71.25, 74, 76.75, 79.4375, + (21,30): 82.1875, 84.9375, 87.6875, 90.4375, 93.1875, 95.875, 98.625, + (21,37): 101, 104, 107, 110, 112, 115, 118, 121, 123, 126, 129, 132, + (21,49): 134, 137, 140, 142, 145, 148, 151, 153, 156, 159, 162, 164, + (21,61): 167, 170, 173, + (22,0): 0, 2.875, 5.75781, 8.63281, 11.5, 14.375, 17.25, 20.125, 23, + (22,9): 25.875, 28.75, 31.625, 34.5, 37.375, 40.25, 43.125, 46.0625, + (22,17): 48.9375, 51.8125, 54.6875, 57.5625, 60.4375, 63.3125, 66.1875, + (22,24): 69.0625, 71.9375, 74.8125, 77.6875, 80.5625, 83.4375, 86.3125, + (22,31): 89.1875, 92.0625, 94.9375, 97.8125, 101, 104, 106, 109, 112, + (22,40): 115, 118, 121, 124, 127, 129, 132, 135, 138, 141, 144, 147, + (22,52): 150, 152, 155, 158, 161, 164, 167, 170, 173, 175, 178, 181, + (23,0): 0, 3.01562, 6.03125, 9.03906, 12.0625, 15.0625, 18.0625, + (23,7): 21.125, 24.125, 27.125, 30.125, 33.125, 36.1875, 39.1875, + (23,14): 42.1875, 45.1875, 48.25, 51.25, 54.25, 57.25, 60.25, 63.3125, + (23,22): 66.3125, 69.3125, 72.3125, 75.375, 78.375, 81.375, 84.375, + (23,29): 87.375, 90.4375, 93.4375, 96.4375, 99.4375, 102, 105, 109, + (23,37): 112, 115, 118, 121, 124, 127, 130, 133, 136, 139, 142, 145, + (23,49): 148, 151, 154, 157, 160, 163, 166, 169, 172, 175, 178, 181, + (23,61): 184, 187, 190, + (24,0): 0, 3.14844, 6.30469, 9.45312, 12.625, 15.75, 18.875, 22.0625, + (24,8): 25.1875, 28.375, 31.5, 34.6875, 37.8125, 40.9375, 44.125, + (24,15): 47.25, 50.4375, 53.5625, 56.6875, 59.875, 63, 66.1875, + (24,22): 69.3125, 72.5, 75.625, 78.75, 81.9375, 85.0625, 88.25, 91.375, + (24,30): 94.5, 97.6875, 101, 104, 107, 110, 113, 117, 120, 123, 126, + (24,41): 129, 132, 135, 139, 142, 145, 148, 151, 154, 158, 161, 164, + (24,53): 167, 170, 173, 176, 180, 183, 186, 189, 192, 195, 199, + (25,0): 0, 3.28906, 6.57812, 9.86719, 13.125, 16.4375, 19.75, 23, + (25,8): 26.3125, 29.5625, 32.875, 36.1875, 39.4375, 42.75, 46.0625, + (25,15): 49.3125, 52.625, 55.875, 59.1875, 62.5, 65.75, 69.0625, + (25,22): 72.3125, 75.625, 78.9375, 82.1875, 85.5, 88.75, 92.0625, + (25,29): 95.375, 98.625, 102, 105, 109, 112, 115, 118, 122, 125, 128, + (25,40): 132, 135, 138, 141, 145, 148, 151, 155, 158, 161, 164, 168, + (25,52): 171, 174, 178, 181, 184, 187, 191, 194, 197, 201, 204, 207, + (26,0): 0, 3.42188, 6.85156, 10.25, 13.6875, 17.125, 20.5625, 24, + (26,8): 27.375, 30.8125, 34.25, 37.6875, 41.125, 44.5, 47.9375, 51.375, + (26,16): 54.8125, 58.25, 61.625, 65.0625, 68.5, 71.9375, 75.375, 78.75, + (26,24): 82.1875, 85.625, 89.0625, 92.5, 95.875, 99.3125, 103, 106, + (26,32): 110, 113, 116, 120, 123, 127, 130, 134, 137, 140, 144, 147, + (26,44): 151, 154, 158, 161, 164, 168, 171, 175, 178, 182, 185, 188, + (26,56): 192, 195, 199, 202, 205, 209, 212, 216, + (27,0): 0, 3.5625, 7.125, 10.6875, 14.25, 17.8125, 21.375, 24.9375, + (27,8): 28.5, 32.0625, 35.625, 39.1875, 42.75, 46.3125, 49.875, + (27,15): 53.4375, 57, 60.5625, 64.125, 67.6875, 71.25, 74.8125, 78.375, + (27,23): 81.9375, 85.5, 89.0625, 92.625, 96.1875, 99.75, 103, 107, 110, + (27,32): 114, 118, 121, 125, 128, 132, 135, 139, 142, 146, 150, 153, + (27,44): 157, 160, 164, 167, 171, 175, 178, 182, 185, 189, 192, 196, + (27,56): 199, 203, 207, 210, 214, 217, 221, 224, + (28,0): 0, 3.69531, 7.39844, 11.125, 14.8125, 18.5, 22.1875, 25.875, + (28,8): 29.5625, 33.3125, 37, 40.6875, 44.375, 48.0625, 51.8125, 55.5, + (28,16): 59.1875, 62.875, 66.5625, 70.25, 74, 77.6875, 81.375, 85.0625, + (28,24): 88.75, 92.5, 96.1875, 99.875, 104, 107, 111, 115, 118, 122, + (28,34): 126, 129, 133, 137, 141, 144, 148, 152, 155, 159, 163, 166, + (28,46): 170, 174, 178, 181, 185, 189, 192, 196, 200, 203, 207, 211, + (28,58): 215, 218, 222, 226, 229, 233, + (29,0): 0, 3.83594, 7.67188, 11.5, 15.375, 19.1875, 23, 26.875, + (29,8): 30.6875, 34.5, 38.375, 42.1875, 46.0625, 49.875, 53.6875, + (29,15): 57.5625, 61.375, 65.1875, 69.0625, 72.875, 76.75, 80.5625, + (29,22): 84.375, 88.25, 92.0625, 95.875, 99.75, 104, 107, 111, 115, + (29,31): 119, 123, 127, 130, 134, 138, 142, 146, 150, 153, 157, 161, + (29,43): 165, 169, 173, 176, 180, 184, 188, 192, 196, 199, 203, 207, + (29,55): 211, 215, 219, 222, 226, 230, 234, 238, 242, + (30,0): 0, 3.97656, 7.94531, 11.9375, 15.875, 19.875, 23.8125, 27.8125, + (30,8): 31.8125, 35.75, 39.75, 43.6875, 47.6875, 51.625, 55.625, + (30,15): 59.625, 63.5625, 67.5625, 71.5, 75.5, 79.4375, 83.4375, + (30,22): 87.375, 91.375, 95.375, 99.3125, 103, 107, 111, 115, 119, 123, + (30,32): 127, 131, 135, 139, 143, 147, 151, 155, 159, 163, 167, 171, + (30,44): 175, 179, 183, 187, 191, 195, 199, 203, 207, 211, 215, 219, + (30,56): 222, 226, 230, 234, 238, 242, 246, 250, + (31,0): 0, 4.10938, 8.21875, 12.3125, 16.4375, 20.5625, 24.6875, 28.75, + (31,8): 32.875, 37, 41.125, 45.1875, 49.3125, 53.4375, 57.5625, 61.625, + (31,16): 65.75, 69.875, 74, 78.0625, 82.1875, 86.3125, 90.4375, 94.5, + (31,24): 98.625, 103, 107, 111, 115, 119, 123, 127, 132, 136, 140, 144, + (31,36): 148, 152, 156, 160, 164, 169, 173, 177, 181, 185, 189, 193, + (31,48): 197, 201, 205, 210, 214, 218, 222, 226, 230, 234, 238, 242, + (31,60): 247, 251, 255, 259 + } + } +} +} diff --git a/BITROUND/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert.ddl b/BITROUND/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert.ddl new file mode 100644 index 000000000..070455a43 --- /dev/null +++ b/BITROUND/config/cmake/binex/example/testfiles/h5repack_floats.h5-ud_convert.ddl @@ -0,0 +1,26 @@ +HDF5 "out-ud_convert.h5repack_floats.h5" { +GROUP "/" { + DATASET "data" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + STORAGE_LAYOUT { + CHUNKED ( 4, 8 ) + SIZE 8192 (1.000:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32023 + COMMENT Granular BitRound filter + PARAMS { 3 4 0 0 0 } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } +} +} diff --git a/BITROUND/config/cmake/binex/example/testfiles/h5repack_layout.h5 b/BITROUND/config/cmake/binex/example/testfiles/h5repack_layout.h5 deleted file mode 100644 index 2cc44b140..000000000 Binary files a/BITROUND/config/cmake/binex/example/testfiles/h5repack_layout.h5 and /dev/null differ diff --git a/BITROUND/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl b/BITROUND/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl deleted file mode 100644 index dd4ae1308..000000000 --- a/BITROUND/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl +++ /dev/null @@ -1,130 +0,0 @@ -HDF5 "out-ud_convert.h5repack_layout.h5" { -GROUP "/" { - DATASET "dset1" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset2" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset3" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset4" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_chunk" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, H5S_UNLIMITED ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_compact" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_contiguous" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } -} -} diff --git a/BITROUND/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst b/BITROUND/config/cmake/binex/example/testfiles/ud_convert.h5repack_floats.h5.tst similarity index 53% rename from BITROUND/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst rename to BITROUND/config/cmake/binex/example/testfiles/ud_convert.h5repack_floats.h5.tst index 2674f6d31..3a41f88bd 100644 --- a/BITROUND/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst +++ b/BITROUND/config/cmake/binex/example/testfiles/ud_convert.h5repack_floats.h5.tst @@ -7,10 +7,4 @@ Making new file ... Type Filter (Compression) Name ----------------------------------------- group / - dset UD (0.833:1) /dset1 - dset UD (0.833:1) /dset2 - dset UD (0.833:1) /dset3 - dset UD (0.833:1) /dset4 - dset UD (0.833:1) /dset_chunk - dset UD (0.833:1) /dset_compact - dset UD (0.833:1) /dset_contiguous + dset UD (1.000:1) /data diff --git a/BITROUND/example/CMakeLists.txt b/BITROUND/example/CMakeLists.txt index 2d2171645..9be01cc55 100644 --- a/BITROUND/example/CMakeLists.txt +++ b/BITROUND/example/CMakeLists.txt @@ -33,6 +33,16 @@ foreach (example ${dyn_examples}) endforeach () if (H5PL_BUILD_TESTING) + # h5repack test input generator. Target name is plugin-suffixed because the + # BITROUND and BITGROOM trees share the same CMake project and source file + # basename (issue #240). + add_executable (h5repack_floats_bitround ${PROJECT_SOURCE_DIR}/h5repack_floats.c) + TARGET_C_PROPERTIES (h5repack_floats_bitround ${LIB_TYPE}) + target_link_libraries (h5repack_floats_bitround PRIVATE ${H5PL_HDF5_LINK_LIBS}) + if (NOT WIN32) + target_link_libraries (h5repack_floats_bitround PRIVATE dl) + endif () + macro (ADD_H5_TEST testname) add_test ( NAME ${testname}-clearall @@ -132,6 +142,7 @@ if (H5PL_BUILD_TESTING) NAME H5BITROUND_UD-${testname}-clearall-objects COMMAND ${CMAKE_COMMAND} -E remove + ${resultfile} out-${testname}.${resultfile} ${testname}.${resultfile}.out ${testname}.${resultfile}.out.err @@ -143,10 +154,33 @@ if (H5PL_BUILD_TESTING) endif () set (last_test "H5BITROUND_UD-${testname}-clearall-objects") if (WIN32) - set (${TESTLIBDIR} "${HDF5_TOOLS_DIR}") + set (TESTLIBDIR "${HDF5_TOOLS_DIR}") else () - set (${TESTLIBDIR} "${HDF5_LIBRARY_PATH}") + set (TESTLIBDIR "${HDF5_LIBRARY_PATH}") endif () + # Generate the float-data input file for h5repack. The Granular BitRound + # filter's set_local() callback removes the filter from the pipeline on + # non-float datasets, so the input must contain float data for the test + # to actually exercise the filter (issue #240). Routed through + # runTest.cmake so PATH/DYLD_LIBRARY_PATH/LD_LIBRARY_PATH are set up for + # the helper to find the HDF5 runtime library on every platform + # (Windows in particular has no rpath equivalent and reports + # STATUS_DLL_NOT_FOUND if the HDF5 DLL is not on PATH). + add_test ( + NAME H5BITROUND_UD-${testname}-gen-input + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_EXPECT=0" + -D "TEST_OUTPUT=h5repack_floats_bitround.out" + -D "TEST_SKIP_COMPARE=1" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -P "${H5BITROUND_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5BITROUND_UD-${testname}-gen-input PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS H5BITROUND_UD-${testname}-clearall-objects) + set (last_test "H5BITROUND_UD-${testname}-gen-input") if (HDF5_BUILD_TOOLS OR HDF5_PROVIDES_TOOLS) add_test ( NAME H5BITROUND_UD-${testname} @@ -165,7 +199,7 @@ if (H5PL_BUILD_TESTING) ) set_tests_properties (H5BITROUND_UD-${testname} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" - DEPENDS H5BITROUND_UD-${testname}-clearall-objects) + DEPENDS H5BITROUND_UD-${testname}-gen-input) add_test ( NAME H5BITROUND_UD-h5dump-${testname} COMMAND "${CMAKE_COMMAND}" @@ -184,23 +218,45 @@ if (H5PL_BUILD_TESTING) WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" DEPENDS H5BITROUND_UD-${testname}) set (last_test "H5BITROUND_UD-h5dump-${testname}") + # Verify that the filter actually modifies the dataset values: dump the + # data (no -pH) and compare against a captured reference of the + # quantized output. Granular BitRound's encoding was made bit- + # reproducible across platforms by replacing the libm log10()/floor() + # chain in ccr_gbr() with a deterministic power-of-10 table lookup + # (see comment in src/H5Zgranularbr.c above ccr_gbr_dgt_nbr()). + add_test ( + NAME H5BITROUND_UD-h5dump-data-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5PL_HDF5_DUMP_EXECUTABLE}" + -D "TEST_ARGS:STRING=out-${testname}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_OUTPUT=${resultfile}-${testname}-data.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}-${testname}-data.ddl" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -D "TEST_ENV_VAR=HDF5_PLUGIN_PATH" + -D "TEST_ENV_VALUE=${CMAKE_BINARY_DIR}/plugins" + -P "${H5BITROUND_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5BITROUND_UD-h5dump-data-${testname} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS H5BITROUND_UD-h5dump-${testname}) + set (last_test "H5BITROUND_UD-h5dump-data-${testname}") endif () endmacro () # -------------------------------------------------------------------- # Copy all the HDF5 files from the source directory into the test directory # -------------------------------------------------------------------- - set (LIST_HDF5_TEST_FILES - h5repack_layout.h5 - ) set (LIST_OTHER_TEST_FILES h5ex_d_granularbr.ddl h5ex_d_granularbr.tst - h5repack_layout.h5-ud_convert.ddl - ud_convert.h5repack_layout.h5.tst + h5repack_floats.h5-ud_convert.ddl + h5repack_floats.h5-ud_convert-data.ddl + ud_convert.h5repack_floats.h5.tst ) - foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) + foreach (h5_file ${LIST_OTHER_TEST_FILES}) HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/${h5_file}" "granularbr_files") endforeach () add_custom_target(granularbr_files ALL COMMENT "Copying files needed by granularbr tests" DEPENDS ${granularbr_files_list}) @@ -209,7 +265,7 @@ if (H5PL_BUILD_TESTING) if (NOT DISABLE_H5BITROUND_ENCODER) #UD BITROUND - ADD_H5_UD_TEST (ud_convert 0 h5repack_layout.h5 --enable-error-stack -v -f UD=32023,0,5,3,4,0,0,0 -l CHUNK=4x8) + ADD_H5_UD_TEST (ud_convert 0 h5repack_floats.h5 --enable-error-stack -v -f UD=32023,0,5,3,4,0,0,0 -l CHUNK=4x8) endif () endif () diff --git a/BITROUND/example/h5repack_floats.c b/BITROUND/example/h5repack_floats.c new file mode 100644 index 000000000..b01cc161e --- /dev/null +++ b/BITROUND/example/h5repack_floats.c @@ -0,0 +1,87 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of the HDF5 Granular BitRound filter plugin source. * + * The full copyright notice, including terms governing use, modification, * + * and redistribution, is contained in the file COPYING, which can be found * + * at the root of the source code distribution tree. If you do not have * + * access to this file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Generates an HDF5 file with a chunked dataset of IEEE 32-bit floats and no + * filters applied. Used as input for the h5repack Granular BitRound test so + * the filter's set_local() callback retains the filter in the pipeline (the + * callback removes itself on non-float data). See issue #240. + */ + +#include "hdf5.h" +#include + +#define FILENAME "h5repack_floats.h5" +#define DATASET "data" +#define DIM0 32 +#define DIM1 64 +#define CHUNK0 4 +#define CHUNK1 8 + +int +main(void) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t space_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + hsize_t dims[2] = {DIM0, DIM1}; + hsize_t chunk[2] = {CHUNK0, CHUNK1}; + float wdata[DIM0][DIM1]; + int i, j; + int ret_value = 1; + + /* Non-integer floats: the 0.137f multiplier ensures every value carries + * non-trivial mantissa bits, so quantization by Granular BitRound is + * visible across the entire dataset rather than only on values that + * exceed the retained mantissa width. */ + for (i = 0; i < DIM0; i++) + for (j = 0; j < DIM1; j++) + wdata[i][j] = ((float)(i * j) - (float)j) * 0.137f; + + file_id = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if (file_id < 0) + goto done; + + space_id = H5Screate_simple(2, dims, NULL); + if (space_id < 0) + goto done; + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + if (dcpl_id < 0) + goto done; + if (H5Pset_chunk(dcpl_id, 2, chunk) < 0) + goto done; + + dset_id = H5Dcreate(file_id, DATASET, H5T_IEEE_F32LE, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + if (dset_id < 0) + goto done; + + if (H5Dwrite(dset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void *)wdata) < 0) + goto done; + + ret_value = 0; + +done: + if (dset_id >= 0) + H5Dclose(dset_id); + if (dcpl_id >= 0) + H5Pclose(dcpl_id); + if (space_id >= 0) + H5Sclose(space_id); + if (file_id >= 0) + H5Fclose(file_id); + + if (ret_value != 0) + fprintf(stderr, "h5repack_floats: failed to create %s\n", FILENAME); + + return ret_value; +} diff --git a/BITROUND/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl b/BITROUND/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl new file mode 100644 index 000000000..22d6e9129 --- /dev/null +++ b/BITROUND/example/testfiles/h5repack_floats.h5-ud_convert-data.ddl @@ -0,0 +1,270 @@ +HDF5 "out-ud_convert.h5repack_floats.h5" { +GROUP "/" { + DATASET "data" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + DATA { + (0,0): 0, -0.136719, -0.274414, -0.411133, -0.547852, -0.68457, + (0,6): -0.822266, -0.958984, -1.09375, -1.23438, -1.36719, -1.50781, + (0,12): -1.64062, -1.78125, -1.92188, -2.05469, -2.19531, -2.32812, + (0,18): -2.46875, -2.60156, -2.74219, -2.875, -3.01562, -3.14844, + (0,24): -3.28906, -3.42188, -3.5625, -3.69531, -3.83594, -3.97656, + (0,30): -4.10938, -4.25, -4.38281, -4.52344, -4.65625, -4.79688, + (0,36): -4.92969, -5.07031, -5.20312, -5.34375, -5.47656, -5.61719, + (0,42): -5.75781, -5.89062, -6.03125, -6.16406, -6.30469, -6.4375, + (0,48): -6.57812, -6.71094, -6.85156, -6.98438, -7.125, -7.25781, + (0,54): -7.39844, -7.53125, -7.67188, -7.8125, -7.94531, -8.08594, + (0,60): -8.21875, -8.35938, -8.49219, -8.63281, + (1,0): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,21): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,42): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,63): 0, + (2,0): 0, 0.136719, 0.274414, 0.411133, 0.547852, 0.68457, 0.822266, + (2,7): 0.958984, 1.09375, 1.23438, 1.36719, 1.50781, 1.64062, 1.78125, + (2,14): 1.92188, 2.05469, 2.19531, 2.32812, 2.46875, 2.60156, 2.74219, + (2,21): 2.875, 3.01562, 3.14844, 3.28906, 3.42188, 3.5625, 3.69531, + (2,28): 3.83594, 3.97656, 4.10938, 4.25, 4.38281, 4.52344, 4.65625, + (2,35): 4.79688, 4.92969, 5.07031, 5.20312, 5.34375, 5.47656, 5.61719, + (2,42): 5.75781, 5.89062, 6.03125, 6.16406, 6.30469, 6.4375, 6.57812, + (2,49): 6.71094, 6.85156, 6.98438, 7.125, 7.25781, 7.39844, 7.53125, + (2,56): 7.67188, 7.8125, 7.94531, 8.08594, 8.21875, 8.35938, 8.49219, + (2,63): 8.63281, + (3,0): 0, 0.274414, 0.547852, 0.822266, 1.09375, 1.36719, 1.64062, + (3,7): 1.92188, 2.19531, 2.46875, 2.74219, 3.01562, 3.28906, 3.5625, + (3,14): 3.83594, 4.10938, 4.38281, 4.65625, 4.92969, 5.20312, 5.47656, + (3,21): 5.75781, 6.03125, 6.30469, 6.57812, 6.85156, 7.125, 7.39844, + (3,28): 7.67188, 7.94531, 8.21875, 8.49219, 8.76562, 9.03906, 9.3125, + (3,35): 9.59375, 9.86719, 10.125, 10.4375, 10.6875, 10.9375, 11.25, + (3,42): 11.5, 11.8125, 12.0625, 12.3125, 12.625, 12.875, 13.125, + (3,49): 13.4375, 13.6875, 14, 14.25, 14.5, 14.8125, 15.0625, 15.375, + (3,57): 15.625, 15.875, 16.1875, 16.4375, 16.6875, 17, 17.25, + (4,0): 0, 0.411133, 0.822266, 1.23438, 1.64062, 2.05469, 2.46875, + (4,7): 2.875, 3.28906, 3.69531, 4.10938, 4.52344, 4.92969, 5.34375, + (4,14): 5.75781, 6.16406, 6.57812, 6.98438, 7.39844, 7.8125, 8.21875, + (4,21): 8.63281, 9.03906, 9.45312, 9.86719, 10.25, 10.6875, 11.125, + (4,28): 11.5, 11.9375, 12.3125, 12.75, 13.125, 13.5625, 14, 14.375, + (4,36): 14.8125, 15.1875, 15.625, 16, 16.4375, 16.875, 17.25, 17.6875, + (4,44): 18.0625, 18.5, 18.875, 19.3125, 19.75, 20.125, 20.5625, + (4,51): 20.9375, 21.375, 21.8125, 22.1875, 22.625, 23, 23.4375, + (4,58): 23.8125, 24.25, 24.6875, 25.0625, 25.5, 25.875, + (5,0): 0, 0.547852, 1.09375, 1.64062, 2.19531, 2.74219, 3.28906, + (5,7): 3.83594, 4.38281, 4.92969, 5.47656, 6.03125, 6.57812, 7.125, + (5,14): 7.67188, 8.21875, 8.76562, 9.3125, 9.86719, 10.4375, 10.9375, + (5,21): 11.5, 12.0625, 12.625, 13.125, 13.6875, 14.25, 14.8125, 15.375, + (5,29): 15.875, 16.4375, 17, 17.5625, 18.0625, 18.625, 19.1875, 19.75, + (5,37): 20.25, 20.8125, 21.375, 21.9375, 22.4375, 23, 23.5625, 24.125, + (5,45): 24.6875, 25.1875, 25.75, 26.3125, 26.875, 27.375, 27.9375, + (5,52): 28.5, 29.0625, 29.5625, 30.125, 30.6875, 31.25, 31.8125, + (5,59): 32.3125, 32.875, 33.4375, 34, 34.5, + (6,0): 0, 0.68457, 1.36719, 2.05469, 2.74219, 3.42188, 4.10938, + (6,7): 4.79688, 5.47656, 6.16406, 6.85156, 7.53125, 8.21875, 8.90625, + (6,14): 9.59375, 10.25, 10.9375, 11.625, 12.3125, 13, 13.6875, 14.375, + (6,22): 15.0625, 15.75, 16.4375, 17.125, 17.8125, 18.5, 19.1875, + (6,29): 19.875, 20.5625, 21.25, 21.9375, 22.625, 23.3125, 24, 24.6875, + (6,37): 25.375, 26, 26.6875, 27.375, 28.0625, 28.75, 29.4375, 30.125, + (6,45): 30.8125, 31.5, 32.1875, 32.875, 33.5625, 34.25, 34.9375, + (6,52): 35.625, 36.3125, 37, 37.6875, 38.375, 39.0625, 39.75, 40.4375, + (6,60): 41.125, 41.8125, 42.5, 43.125, + (7,0): 0, 0.822266, 1.64062, 2.46875, 3.28906, 4.10938, 4.92969, + (7,7): 5.75781, 6.57812, 7.39844, 8.21875, 9.03906, 9.86719, 10.6875, + (7,14): 11.5, 12.3125, 13.125, 14, 14.8125, 15.625, 16.4375, 17.25, + (7,22): 18.0625, 18.875, 19.75, 20.5625, 21.375, 22.1875, 23, 23.8125, + (7,30): 24.6875, 25.5, 26.3125, 27.125, 27.9375, 28.75, 29.5625, + (7,37): 30.4375, 31.25, 32.0625, 32.875, 33.6875, 34.5, 35.375, + (7,44): 36.1875, 37, 37.8125, 38.625, 39.4375, 40.25, 41.125, 41.9375, + (7,52): 42.75, 43.5625, 44.375, 45.1875, 46.0625, 46.875, 47.6875, + (7,59): 48.5, 49.3125, 50.125, 50.9375, 51.8125, + (8,0): 0, 0.958984, 1.92188, 2.875, 3.83594, 4.79688, 5.75781, 6.71094, + (8,8): 7.67188, 8.63281, 9.59375, 10.5625, 11.5, 12.4375, 13.4375, + (8,15): 14.375, 15.375, 16.3125, 17.25, 18.25, 19.1875, 20.125, 21.125, + (8,23): 22.0625, 23, 24, 24.9375, 25.875, 26.875, 27.8125, 28.75, + (8,31): 29.75, 30.6875, 31.625, 32.625, 33.5625, 34.5, 35.5, 36.4375, + (8,39): 37.375, 38.375, 39.3125, 40.25, 41.25, 42.1875, 43.125, 44.125, + (8,47): 45.0625, 46.0625, 47, 47.9375, 48.9375, 49.875, 50.8125, + (8,54): 51.8125, 52.75, 53.6875, 54.6875, 55.625, 56.5625, 57.5625, + (8,61): 58.5, 59.4375, 60.4375, + (9,0): 0, 1.09375, 2.19531, 3.28906, 4.38281, 5.47656, 6.57812, + (9,7): 7.67188, 8.76562, 9.86719, 10.9375, 12.0625, 13.125, 14.25, + (9,14): 15.375, 16.4375, 17.5625, 18.625, 19.75, 20.8125, 21.9375, 23, + (9,22): 24.125, 25.1875, 26.3125, 27.375, 28.5, 29.5625, 30.6875, + (9,29): 31.8125, 32.875, 34, 35.0625, 36.1875, 37.25, 38.375, 39.4375, + (9,37): 40.5625, 41.625, 42.75, 43.8125, 44.9375, 46.0625, 47.125, + (9,44): 48.25, 49.3125, 50.4375, 51.5, 52.625, 53.6875, 54.8125, + (9,51): 55.875, 57, 58.0625, 59.1875, 60.25, 61.375, 62.5, 63.5625, + (9,59): 64.6875, 65.75, 66.875, 67.9375, 69.0625, + (10,0): 0, 1.23438, 2.46875, 3.69531, 4.92969, 6.16406, 7.39844, + (10,7): 8.63281, 9.86719, 11.125, 12.3125, 13.5625, 14.8125, 16, 17.25, + (10,15): 18.5, 19.75, 20.9375, 22.1875, 23.4375, 24.6875, 25.875, + (10,22): 27.125, 28.375, 29.5625, 30.8125, 32.0625, 33.3125, 34.5, + (10,29): 35.75, 37, 38.25, 39.4375, 40.6875, 41.9375, 43.125, 44.375, + (10,37): 45.625, 46.875, 48.0625, 49.3125, 50.5625, 51.8125, 53, 54.25, + (10,45): 55.5, 56.6875, 57.9375, 59.1875, 60.4375, 61.625, 62.875, + (10,52): 64.125, 65.375, 66.5625, 67.8125, 69.0625, 70.25, 71.5, 72.75, + (10,60): 74, 75.1875, 76.4375, 77.6875, + (11,0): 0, 1.36719, 2.74219, 4.10938, 5.47656, 6.85156, 8.21875, + (11,7): 9.59375, 10.9375, 12.3125, 13.6875, 15.0625, 16.4375, 17.8125, + (11,14): 19.1875, 20.5625, 21.9375, 23.3125, 24.6875, 26, 27.375, + (11,21): 28.75, 30.125, 31.5, 32.875, 34.25, 35.625, 37, 38.375, 39.75, + (11,30): 41.125, 42.5, 43.8125, 45.1875, 46.5625, 47.9375, 49.3125, + (11,37): 50.6875, 52.0625, 53.4375, 54.8125, 56.1875, 57.5625, 58.9375, + (11,44): 60.25, 61.625, 63, 64.375, 65.75, 67.125, 68.5, 69.875, 71.25, + (11,53): 72.625, 74, 75.375, 76.75, 78.0625, 79.4375, 80.8125, 82.1875, + (11,61): 83.5625, 84.9375, 86.3125, + (12,0): 0, 1.50781, 3.01562, 4.52344, 6.03125, 7.53125, 9.03906, + (12,7): 10.5625, 12.0625, 13.5625, 15.0625, 16.5625, 18.0625, 19.5625, + (12,14): 21.125, 22.625, 24.125, 25.625, 27.125, 28.625, 30.125, + (12,21): 31.625, 33.125, 34.6875, 36.1875, 37.6875, 39.1875, 40.6875, + (12,28): 42.1875, 43.6875, 45.1875, 46.6875, 48.25, 49.75, 51.25, + (12,35): 52.75, 54.25, 55.75, 57.25, 58.75, 60.25, 61.8125, 63.3125, + (12,43): 64.8125, 66.3125, 67.8125, 69.3125, 70.8125, 72.3125, 73.8125, + (12,50): 75.375, 76.875, 78.375, 79.875, 81.375, 82.875, 84.375, + (12,57): 85.875, 87.375, 88.9375, 90.4375, 91.9375, 93.4375, 94.9375, + (13,0): 0, 1.64062, 3.28906, 4.92969, 6.57812, 8.21875, 9.86719, 11.5, + (13,8): 13.125, 14.8125, 16.4375, 18.0625, 19.75, 21.375, 23, 24.6875, + (13,16): 26.3125, 27.9375, 29.5625, 31.25, 32.875, 34.5, 36.1875, + (13,23): 37.8125, 39.4375, 41.125, 42.75, 44.375, 46.0625, 47.6875, + (13,30): 49.3125, 50.9375, 52.625, 54.25, 55.875, 57.5625, 59.1875, + (13,37): 60.8125, 62.5, 64.125, 65.75, 67.375, 69.0625, 70.6875, + (13,44): 72.3125, 74, 75.625, 77.25, 78.9375, 80.5625, 82.1875, 83.875, + (13,52): 85.5, 87.125, 88.75, 90.4375, 92.0625, 93.6875, 95.375, 97, + (13,60): 98.625, 100, 102, 104, + (14,0): 0, 1.78125, 3.5625, 5.34375, 7.125, 8.90625, 10.6875, 12.4375, + (14,8): 14.25, 16, 17.8125, 19.5625, 21.375, 23.125, 24.9375, 26.6875, + (14,16): 28.5, 30.25, 32.0625, 33.8125, 35.625, 37.375, 39.1875, + (14,23): 40.9375, 42.75, 44.5, 46.3125, 48.0625, 49.875, 51.625, + (14,30): 53.4375, 55.1875, 57, 58.75, 60.5625, 62.3125, 64.125, 65.875, + (14,38): 67.6875, 69.4375, 71.25, 73, 74.8125, 76.5625, 78.375, 80.125, + (14,46): 81.9375, 83.6875, 85.5, 87.25, 89.0625, 90.8125, 92.625, + (14,53): 94.375, 96.1875, 97.9375, 99.75, 102, 103, 105, 107, 109, 110, + (14,63): 112, + (15,0): 0, 1.92188, 3.83594, 5.75781, 7.67188, 9.59375, 11.5, 13.4375, + (15,8): 15.375, 17.25, 19.1875, 21.125, 23, 24.9375, 26.875, 28.75, + (15,16): 30.6875, 32.625, 34.5, 36.4375, 38.375, 40.25, 42.1875, + (15,23): 44.125, 46.0625, 47.9375, 49.875, 51.8125, 53.6875, 55.625, + (15,30): 57.5625, 59.4375, 61.375, 63.3125, 65.1875, 67.125, 69.0625, + (15,37): 70.9375, 72.875, 74.8125, 76.75, 78.625, 80.5625, 82.5, + (15,44): 84.375, 86.3125, 88.25, 90.125, 92.0625, 94, 95.875, 97.8125, + (15,52): 99.75, 102, 104, 105, 107, 109, 111, 113, 115, 117, 119, 121, + (16,0): 0, 2.05469, 4.10938, 6.16406, 8.21875, 10.25, 12.3125, 14.375, + (16,8): 16.4375, 18.5, 20.5625, 22.625, 24.6875, 26.6875, 28.75, + (16,15): 30.8125, 32.875, 34.9375, 37, 39.0625, 41.125, 43.125, + (16,22): 45.1875, 47.25, 49.3125, 51.375, 53.4375, 55.5, 57.5625, + (16,29): 59.625, 61.625, 63.6875, 65.75, 67.8125, 69.875, 71.9375, 74, + (16,37): 76.0625, 78.0625, 80.125, 82.1875, 84.25, 86.3125, 88.375, + (16,44): 90.4375, 92.5, 94.5, 96.5625, 98.625, 101, 103, 105, 107, 109, + (16,54): 111, 113, 115, 117, 119, 121, 123, 125, 127, 129, + (17,0): 0, 2.19531, 4.38281, 6.57812, 8.76562, 10.9375, 13.125, 15.375, + (17,8): 17.5625, 19.75, 21.9375, 24.125, 26.3125, 28.5, 30.6875, + (17,15): 32.875, 35.0625, 37.25, 39.4375, 41.625, 43.8125, 46.0625, + (17,22): 48.25, 50.4375, 52.625, 54.8125, 57, 59.1875, 61.375, 63.5625, + (17,30): 65.75, 67.9375, 70.125, 72.3125, 74.5, 76.75, 78.9375, 81.125, + (17,38): 83.3125, 85.5, 87.6875, 89.875, 92.0625, 94.25, 96.4375, + (17,45): 98.625, 101, 103, 105, 107, 110, 112, 114, 116, 118, 121, 123, + (17,57): 125, 127, 129, 132, 134, 136, 138, + (18,0): 0, 2.32812, 4.65625, 6.98438, 9.3125, 11.625, 14, 16.3125, + (18,8): 18.625, 20.9375, 23.3125, 25.625, 27.9375, 30.25, 32.625, + (18,15): 34.9375, 37.25, 39.5625, 41.9375, 44.25, 46.5625, 48.9375, + (18,22): 51.25, 53.5625, 55.875, 58.25, 60.5625, 62.875, 65.1875, + (18,29): 67.5625, 69.875, 72.1875, 74.5, 76.875, 79.1875, 81.5, 83.875, + (18,37): 86.1875, 88.5, 90.8125, 93.1875, 95.5, 97.8125, 100, 102, 105, + (18,46): 107, 109, 112, 114, 116, 119, 121, 123, 126, 128, 130, 133, + (18,58): 135, 137, 140, 142, 144, 147, + (19,0): 0, 2.46875, 4.92969, 7.39844, 9.86719, 12.3125, 14.8125, 17.25, + (19,8): 19.75, 22.1875, 24.6875, 27.125, 29.5625, 32.0625, 34.5, 37, + (19,16): 39.4375, 41.9375, 44.375, 46.875, 49.3125, 51.8125, 54.25, + (19,23): 56.6875, 59.1875, 61.625, 64.125, 66.5625, 69.0625, 71.5, 74, + (19,31): 76.4375, 78.9375, 81.375, 83.875, 86.3125, 88.75, 91.25, + (19,38): 93.6875, 96.1875, 98.625, 101, 104, 106, 109, 111, 113, 116, + (19,48): 118, 121, 123, 126, 128, 131, 133, 136, 138, 141, 143, 145, + (19,60): 148, 150, 153, 155, + (20,0): 0, 2.60156, 5.20312, 7.8125, 10.4375, 13, 15.625, 18.25, + (20,8): 20.8125, 23.4375, 26, 28.625, 31.25, 33.8125, 36.4375, 39.0625, + (20,16): 41.625, 44.25, 46.875, 49.4375, 52.0625, 54.6875, 57.25, + (20,23): 59.875, 62.5, 65.0625, 67.6875, 70.25, 72.875, 75.5, 78.0625, + (20,31): 80.6875, 83.3125, 85.875, 88.5, 91.125, 93.6875, 96.3125, + (20,38): 98.9375, 102, 104, 107, 109, 112, 115, 117, 120, 122, 125, + (20,49): 128, 130, 133, 135, 138, 141, 143, 146, 148, 151, 154, 156, + (20,61): 159, 161, 164, + (21,0): 0, 2.74219, 5.47656, 8.21875, 10.9375, 13.6875, 16.4375, + (21,7): 19.1875, 21.9375, 24.6875, 27.375, 30.125, 32.875, 35.625, + (21,14): 38.375, 41.125, 43.8125, 46.5625, 49.3125, 52.0625, 54.8125, + (21,21): 57.5625, 60.25, 63, 65.75, 68.5, 71.25, 74, 76.75, 79.4375, + (21,30): 82.1875, 84.9375, 87.6875, 90.4375, 93.1875, 95.875, 98.625, + (21,37): 101, 104, 107, 110, 112, 115, 118, 121, 123, 126, 129, 132, + (21,49): 134, 137, 140, 142, 145, 148, 151, 153, 156, 159, 162, 164, + (21,61): 167, 170, 173, + (22,0): 0, 2.875, 5.75781, 8.63281, 11.5, 14.375, 17.25, 20.125, 23, + (22,9): 25.875, 28.75, 31.625, 34.5, 37.375, 40.25, 43.125, 46.0625, + (22,17): 48.9375, 51.8125, 54.6875, 57.5625, 60.4375, 63.3125, 66.1875, + (22,24): 69.0625, 71.9375, 74.8125, 77.6875, 80.5625, 83.4375, 86.3125, + (22,31): 89.1875, 92.0625, 94.9375, 97.8125, 101, 104, 106, 109, 112, + (22,40): 115, 118, 121, 124, 127, 129, 132, 135, 138, 141, 144, 147, + (22,52): 150, 152, 155, 158, 161, 164, 167, 170, 173, 175, 178, 181, + (23,0): 0, 3.01562, 6.03125, 9.03906, 12.0625, 15.0625, 18.0625, + (23,7): 21.125, 24.125, 27.125, 30.125, 33.125, 36.1875, 39.1875, + (23,14): 42.1875, 45.1875, 48.25, 51.25, 54.25, 57.25, 60.25, 63.3125, + (23,22): 66.3125, 69.3125, 72.3125, 75.375, 78.375, 81.375, 84.375, + (23,29): 87.375, 90.4375, 93.4375, 96.4375, 99.4375, 102, 105, 109, + (23,37): 112, 115, 118, 121, 124, 127, 130, 133, 136, 139, 142, 145, + (23,49): 148, 151, 154, 157, 160, 163, 166, 169, 172, 175, 178, 181, + (23,61): 184, 187, 190, + (24,0): 0, 3.14844, 6.30469, 9.45312, 12.625, 15.75, 18.875, 22.0625, + (24,8): 25.1875, 28.375, 31.5, 34.6875, 37.8125, 40.9375, 44.125, + (24,15): 47.25, 50.4375, 53.5625, 56.6875, 59.875, 63, 66.1875, + (24,22): 69.3125, 72.5, 75.625, 78.75, 81.9375, 85.0625, 88.25, 91.375, + (24,30): 94.5, 97.6875, 101, 104, 107, 110, 113, 117, 120, 123, 126, + (24,41): 129, 132, 135, 139, 142, 145, 148, 151, 154, 158, 161, 164, + (24,53): 167, 170, 173, 176, 180, 183, 186, 189, 192, 195, 199, + (25,0): 0, 3.28906, 6.57812, 9.86719, 13.125, 16.4375, 19.75, 23, + (25,8): 26.3125, 29.5625, 32.875, 36.1875, 39.4375, 42.75, 46.0625, + (25,15): 49.3125, 52.625, 55.875, 59.1875, 62.5, 65.75, 69.0625, + (25,22): 72.3125, 75.625, 78.9375, 82.1875, 85.5, 88.75, 92.0625, + (25,29): 95.375, 98.625, 102, 105, 109, 112, 115, 118, 122, 125, 128, + (25,40): 132, 135, 138, 141, 145, 148, 151, 155, 158, 161, 164, 168, + (25,52): 171, 174, 178, 181, 184, 187, 191, 194, 197, 201, 204, 207, + (26,0): 0, 3.42188, 6.85156, 10.25, 13.6875, 17.125, 20.5625, 24, + (26,8): 27.375, 30.8125, 34.25, 37.6875, 41.125, 44.5, 47.9375, 51.375, + (26,16): 54.8125, 58.25, 61.625, 65.0625, 68.5, 71.9375, 75.375, 78.75, + (26,24): 82.1875, 85.625, 89.0625, 92.5, 95.875, 99.3125, 103, 106, + (26,32): 110, 113, 116, 120, 123, 127, 130, 134, 137, 140, 144, 147, + (26,44): 151, 154, 158, 161, 164, 168, 171, 175, 178, 182, 185, 188, + (26,56): 192, 195, 199, 202, 205, 209, 212, 216, + (27,0): 0, 3.5625, 7.125, 10.6875, 14.25, 17.8125, 21.375, 24.9375, + (27,8): 28.5, 32.0625, 35.625, 39.1875, 42.75, 46.3125, 49.875, + (27,15): 53.4375, 57, 60.5625, 64.125, 67.6875, 71.25, 74.8125, 78.375, + (27,23): 81.9375, 85.5, 89.0625, 92.625, 96.1875, 99.75, 103, 107, 110, + (27,32): 114, 118, 121, 125, 128, 132, 135, 139, 142, 146, 150, 153, + (27,44): 157, 160, 164, 167, 171, 175, 178, 182, 185, 189, 192, 196, + (27,56): 199, 203, 207, 210, 214, 217, 221, 224, + (28,0): 0, 3.69531, 7.39844, 11.125, 14.8125, 18.5, 22.1875, 25.875, + (28,8): 29.5625, 33.3125, 37, 40.6875, 44.375, 48.0625, 51.8125, 55.5, + (28,16): 59.1875, 62.875, 66.5625, 70.25, 74, 77.6875, 81.375, 85.0625, + (28,24): 88.75, 92.5, 96.1875, 99.875, 104, 107, 111, 115, 118, 122, + (28,34): 126, 129, 133, 137, 141, 144, 148, 152, 155, 159, 163, 166, + (28,46): 170, 174, 178, 181, 185, 189, 192, 196, 200, 203, 207, 211, + (28,58): 215, 218, 222, 226, 229, 233, + (29,0): 0, 3.83594, 7.67188, 11.5, 15.375, 19.1875, 23, 26.875, + (29,8): 30.6875, 34.5, 38.375, 42.1875, 46.0625, 49.875, 53.6875, + (29,15): 57.5625, 61.375, 65.1875, 69.0625, 72.875, 76.75, 80.5625, + (29,22): 84.375, 88.25, 92.0625, 95.875, 99.75, 104, 107, 111, 115, + (29,31): 119, 123, 127, 130, 134, 138, 142, 146, 150, 153, 157, 161, + (29,43): 165, 169, 173, 176, 180, 184, 188, 192, 196, 199, 203, 207, + (29,55): 211, 215, 219, 222, 226, 230, 234, 238, 242, + (30,0): 0, 3.97656, 7.94531, 11.9375, 15.875, 19.875, 23.8125, 27.8125, + (30,8): 31.8125, 35.75, 39.75, 43.6875, 47.6875, 51.625, 55.625, + (30,15): 59.625, 63.5625, 67.5625, 71.5, 75.5, 79.4375, 83.4375, + (30,22): 87.375, 91.375, 95.375, 99.3125, 103, 107, 111, 115, 119, 123, + (30,32): 127, 131, 135, 139, 143, 147, 151, 155, 159, 163, 167, 171, + (30,44): 175, 179, 183, 187, 191, 195, 199, 203, 207, 211, 215, 219, + (30,56): 222, 226, 230, 234, 238, 242, 246, 250, + (31,0): 0, 4.10938, 8.21875, 12.3125, 16.4375, 20.5625, 24.6875, 28.75, + (31,8): 32.875, 37, 41.125, 45.1875, 49.3125, 53.4375, 57.5625, 61.625, + (31,16): 65.75, 69.875, 74, 78.0625, 82.1875, 86.3125, 90.4375, 94.5, + (31,24): 98.625, 103, 107, 111, 115, 119, 123, 127, 132, 136, 140, 144, + (31,36): 148, 152, 156, 160, 164, 169, 173, 177, 181, 185, 189, 193, + (31,48): 197, 201, 205, 210, 214, 218, 222, 226, 230, 234, 238, 242, + (31,60): 247, 251, 255, 259 + } + } +} +} diff --git a/BITROUND/example/testfiles/h5repack_floats.h5-ud_convert.ddl b/BITROUND/example/testfiles/h5repack_floats.h5-ud_convert.ddl new file mode 100644 index 000000000..070455a43 --- /dev/null +++ b/BITROUND/example/testfiles/h5repack_floats.h5-ud_convert.ddl @@ -0,0 +1,26 @@ +HDF5 "out-ud_convert.h5repack_floats.h5" { +GROUP "/" { + DATASET "data" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + STORAGE_LAYOUT { + CHUNKED ( 4, 8 ) + SIZE 8192 (1.000:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32023 + COMMENT Granular BitRound filter + PARAMS { 3 4 0 0 0 } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } +} +} diff --git a/BITROUND/example/testfiles/h5repack_layout.h5 b/BITROUND/example/testfiles/h5repack_layout.h5 deleted file mode 100644 index 2cc44b140..000000000 Binary files a/BITROUND/example/testfiles/h5repack_layout.h5 and /dev/null differ diff --git a/BITROUND/example/testfiles/h5repack_layout.h5-ud_convert.ddl b/BITROUND/example/testfiles/h5repack_layout.h5-ud_convert.ddl deleted file mode 100644 index dd4ae1308..000000000 --- a/BITROUND/example/testfiles/h5repack_layout.h5-ud_convert.ddl +++ /dev/null @@ -1,130 +0,0 @@ -HDF5 "out-ud_convert.h5repack_layout.h5" { -GROUP "/" { - DATASET "dset1" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset2" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset3" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset4" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_chunk" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, H5S_UNLIMITED ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_compact" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_contiguous" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 3840 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } -} -} diff --git a/BITROUND/example/testfiles/ud_convert.h5repack_layout.h5.tst b/BITROUND/example/testfiles/ud_convert.h5repack_floats.h5.tst similarity index 53% rename from BITROUND/example/testfiles/ud_convert.h5repack_layout.h5.tst rename to BITROUND/example/testfiles/ud_convert.h5repack_floats.h5.tst index 2674f6d31..3a41f88bd 100644 --- a/BITROUND/example/testfiles/ud_convert.h5repack_layout.h5.tst +++ b/BITROUND/example/testfiles/ud_convert.h5repack_floats.h5.tst @@ -7,10 +7,4 @@ Making new file ... Type Filter (Compression) Name ----------------------------------------- group / - dset UD (0.833:1) /dset1 - dset UD (0.833:1) /dset2 - dset UD (0.833:1) /dset3 - dset UD (0.833:1) /dset4 - dset UD (0.833:1) /dset_chunk - dset UD (0.833:1) /dset_compact - dset UD (0.833:1) /dset_contiguous + dset UD (1.000:1) /data diff --git a/BITROUND/src/H5Zgranularbr.c b/BITROUND/src/H5Zgranularbr.c index b887584f7..8e468da81 100644 --- a/BITROUND/src/H5Zgranularbr.c +++ b/BITROUND/src/H5Zgranularbr.c @@ -445,6 +445,115 @@ ccr_set_local_granularbr(hid_t dcpl, hid_t type, hid_t space) return 1; } +/* Cross-platform-deterministic table for floor(log10(|val|)) + 1. + * + * The original encoding called log10()/floor() per value to compute + * dgt_nbr = floor(log10(|val|)) + 1 + * which is "the number of decimal digits in the integer part of |val|". + * The C standard does not require log10() to be correctly rounded (only + * approximately, typically within ~1 ULP), and feeding its result into + * floor() can flip the result by +/-1 when the true value is near an + * integer boundary. Different libm implementations (e.g. macOS BSD libm + * vs Linux glibc) therefore produce different stored bytes for inputs + * that fall near power-of-10 boundaries -- a serious reproducibility + * issue for scientific data. + * + * Replacing the libm chain with a direct binary search of a power-of-10 + * table makes the computation bit-deterministic on any IEEE 754 platform + * (the only floating-point operation involved is comparison) and removes + * the libm log10() dependency from the encoding inner loop. + * + * Range: covers all double-precision normal-range powers of 10. Inputs + * smaller than 1e-308 fall back to the table's lower bound, which is + * well outside the range of values where Granular BitRound is sensibly + * applied. + * + */ +#define CCR_GBR_POW10_KMIN (-308) +#define CCR_GBR_POW10_KMAX (308) +#define CCR_GBR_POW10_OFFSET (-CCR_GBR_POW10_KMIN) +#define CCR_GBR_POW10_SIZE (CCR_GBR_POW10_KMAX - CCR_GBR_POW10_KMIN + 1) + +static const double ccr_gbr_pow10[CCR_GBR_POW10_SIZE] = { + 1e-308, 1e-307, 1e-306, 1e-305, 1e-304, 1e-303, 1e-302, 1e-301, 1e-300, 1e-299, 1e-298, 1e-297, 1e-296, + 1e-295, 1e-294, 1e-293, 1e-292, 1e-291, 1e-290, 1e-289, 1e-288, 1e-287, 1e-286, 1e-285, 1e-284, 1e-283, + 1e-282, 1e-281, 1e-280, 1e-279, 1e-278, 1e-277, 1e-276, 1e-275, 1e-274, 1e-273, 1e-272, 1e-271, 1e-270, + 1e-269, 1e-268, 1e-267, 1e-266, 1e-265, 1e-264, 1e-263, 1e-262, 1e-261, 1e-260, 1e-259, 1e-258, 1e-257, + 1e-256, 1e-255, 1e-254, 1e-253, 1e-252, 1e-251, 1e-250, 1e-249, 1e-248, 1e-247, 1e-246, 1e-245, 1e-244, + 1e-243, 1e-242, 1e-241, 1e-240, 1e-239, 1e-238, 1e-237, 1e-236, 1e-235, 1e-234, 1e-233, 1e-232, 1e-231, + 1e-230, 1e-229, 1e-228, 1e-227, 1e-226, 1e-225, 1e-224, 1e-223, 1e-222, 1e-221, 1e-220, 1e-219, 1e-218, + 1e-217, 1e-216, 1e-215, 1e-214, 1e-213, 1e-212, 1e-211, 1e-210, 1e-209, 1e-208, 1e-207, 1e-206, 1e-205, + 1e-204, 1e-203, 1e-202, 1e-201, 1e-200, 1e-199, 1e-198, 1e-197, 1e-196, 1e-195, 1e-194, 1e-193, 1e-192, + 1e-191, 1e-190, 1e-189, 1e-188, 1e-187, 1e-186, 1e-185, 1e-184, 1e-183, 1e-182, 1e-181, 1e-180, 1e-179, + 1e-178, 1e-177, 1e-176, 1e-175, 1e-174, 1e-173, 1e-172, 1e-171, 1e-170, 1e-169, 1e-168, 1e-167, 1e-166, + 1e-165, 1e-164, 1e-163, 1e-162, 1e-161, 1e-160, 1e-159, 1e-158, 1e-157, 1e-156, 1e-155, 1e-154, 1e-153, + 1e-152, 1e-151, 1e-150, 1e-149, 1e-148, 1e-147, 1e-146, 1e-145, 1e-144, 1e-143, 1e-142, 1e-141, 1e-140, + 1e-139, 1e-138, 1e-137, 1e-136, 1e-135, 1e-134, 1e-133, 1e-132, 1e-131, 1e-130, 1e-129, 1e-128, 1e-127, + 1e-126, 1e-125, 1e-124, 1e-123, 1e-122, 1e-121, 1e-120, 1e-119, 1e-118, 1e-117, 1e-116, 1e-115, 1e-114, + 1e-113, 1e-112, 1e-111, 1e-110, 1e-109, 1e-108, 1e-107, 1e-106, 1e-105, 1e-104, 1e-103, 1e-102, 1e-101, + 1e-100, 1e-99, 1e-98, 1e-97, 1e-96, 1e-95, 1e-94, 1e-93, 1e-92, 1e-91, 1e-90, 1e-89, 1e-88, + 1e-87, 1e-86, 1e-85, 1e-84, 1e-83, 1e-82, 1e-81, 1e-80, 1e-79, 1e-78, 1e-77, 1e-76, 1e-75, + 1e-74, 1e-73, 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, 1e-65, 1e-64, 1e-63, 1e-62, + 1e-61, 1e-60, 1e-59, 1e-58, 1e-57, 1e-56, 1e-55, 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, + 1e-48, 1e-47, 1e-46, 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, 1e-39, 1e-38, 1e-37, 1e-36, + 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, 1e-29, 1e-28, 1e-27, 1e-26, 1e-25, 1e-24, 1e-23, + 1e-22, 1e-21, 1e-20, 1e-19, 1e-18, 1e-17, 1e-16, 1e-15, 1e-14, 1e-13, 1e-12, 1e-11, 1e-10, + 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2, 1e3, + 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, + 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, 1e23, 1e24, 1e25, 1e26, 1e27, 1e28, 1e29, + 1e30, 1e31, 1e32, 1e33, 1e34, 1e35, 1e36, 1e37, 1e38, 1e39, 1e40, 1e41, 1e42, + 1e43, 1e44, 1e45, 1e46, 1e47, 1e48, 1e49, 1e50, 1e51, 1e52, 1e53, 1e54, 1e55, + 1e56, 1e57, 1e58, 1e59, 1e60, 1e61, 1e62, 1e63, 1e64, 1e65, 1e66, 1e67, 1e68, + 1e69, 1e70, 1e71, 1e72, 1e73, 1e74, 1e75, 1e76, 1e77, 1e78, 1e79, 1e80, 1e81, + 1e82, 1e83, 1e84, 1e85, 1e86, 1e87, 1e88, 1e89, 1e90, 1e91, 1e92, 1e93, 1e94, + 1e95, 1e96, 1e97, 1e98, 1e99, 1e100, 1e101, 1e102, 1e103, 1e104, 1e105, 1e106, 1e107, + 1e108, 1e109, 1e110, 1e111, 1e112, 1e113, 1e114, 1e115, 1e116, 1e117, 1e118, 1e119, 1e120, + 1e121, 1e122, 1e123, 1e124, 1e125, 1e126, 1e127, 1e128, 1e129, 1e130, 1e131, 1e132, 1e133, + 1e134, 1e135, 1e136, 1e137, 1e138, 1e139, 1e140, 1e141, 1e142, 1e143, 1e144, 1e145, 1e146, + 1e147, 1e148, 1e149, 1e150, 1e151, 1e152, 1e153, 1e154, 1e155, 1e156, 1e157, 1e158, 1e159, + 1e160, 1e161, 1e162, 1e163, 1e164, 1e165, 1e166, 1e167, 1e168, 1e169, 1e170, 1e171, 1e172, + 1e173, 1e174, 1e175, 1e176, 1e177, 1e178, 1e179, 1e180, 1e181, 1e182, 1e183, 1e184, 1e185, + 1e186, 1e187, 1e188, 1e189, 1e190, 1e191, 1e192, 1e193, 1e194, 1e195, 1e196, 1e197, 1e198, + 1e199, 1e200, 1e201, 1e202, 1e203, 1e204, 1e205, 1e206, 1e207, 1e208, 1e209, 1e210, 1e211, + 1e212, 1e213, 1e214, 1e215, 1e216, 1e217, 1e218, 1e219, 1e220, 1e221, 1e222, 1e223, 1e224, + 1e225, 1e226, 1e227, 1e228, 1e229, 1e230, 1e231, 1e232, 1e233, 1e234, 1e235, 1e236, 1e237, + 1e238, 1e239, 1e240, 1e241, 1e242, 1e243, 1e244, 1e245, 1e246, 1e247, 1e248, 1e249, 1e250, + 1e251, 1e252, 1e253, 1e254, 1e255, 1e256, 1e257, 1e258, 1e259, 1e260, 1e261, 1e262, 1e263, + 1e264, 1e265, 1e266, 1e267, 1e268, 1e269, 1e270, 1e271, 1e272, 1e273, 1e274, 1e275, 1e276, + 1e277, 1e278, 1e279, 1e280, 1e281, 1e282, 1e283, 1e284, 1e285, 1e286, 1e287, 1e288, 1e289, + 1e290, 1e291, 1e292, 1e293, 1e294, 1e295, 1e296, 1e297, 1e298, 1e299, 1e300, 1e301, 1e302, + 1e303, 1e304, 1e305, 1e306, 1e307, 1e308, +}; + +/* Returns floor(log10(val_fabs)) + 1, the number of decimal digits in the + * integer part of val_fabs. Caller must ensure val_fabs > 0. Implemented as + * a binary search of ccr_gbr_pow10[] -- only floating-point comparison is + * used, which is bit-deterministic across IEEE 754 platforms. */ +static int +ccr_gbr_dgt_nbr(double val_fabs) +{ + int lo, hi; + + /* Below table range: extremely small (subnormal-scale) inputs. + * Return KMIN as a deterministic floor; not exact but well outside + * the range where Granular BitRound is sensibly applied. */ + if (val_fabs < ccr_gbr_pow10[0]) + return CCR_GBR_POW10_KMIN; + + /* Binary search for largest index i such that ccr_gbr_pow10[i] <= val_fabs. + * dgt_nbr = (i - OFFSET) + 1. */ + lo = 0; + hi = CCR_GBR_POW10_SIZE - 1; + while (lo < hi) { + int mid = (lo + hi + 1) / 2; + if (ccr_gbr_pow10[mid] <= val_fabs) + lo = mid; + else + hi = mid - 1; + } + return lo - CCR_GBR_POW10_OFFSET + 1; +} + /** * \brief Granular BitRound buffer of float values * @@ -472,18 +581,17 @@ ccr_gbr(const int nsd, const int type, const size_t sz, const int has_mss_val, p #define M_LN2 0.69314718055994530942 /* log_e 2 */ #endif /* M_LN2 */ const double bit_per_dgt = M_LN10 / M_LN2; /* 3.32 [frc] Bits per decimal digit of precision = log2(10) */ - const double dgt_per_bit = - M_LN2 / M_LN10; /* 0.301 [frc] Decimal digits per bit of precision = log10(2) */ + /* dgt_per_bit (M_LN2/M_LN10) was used by the old libm-based dgt_nbr formula; + * the table-based ccr_gbr_dgt_nbr() no longer needs it. */ const int bit_xpl_nbr_sgn_flt = 23; /* [nbr] Bits 0-22 of SP significands are explicit. Bit 23 is implicitly 1. */ const int bit_xpl_nbr_sgn_dbl = 52; /* [nbr] Bits 0-51 of DP significands are explicit. Bit 52 is implicitly 1. */ - double mnt; /* [frc] Mantissa, 0.5 <= mnt < 1.0 */ - double mnt_fabs; /* [frc] fabs(mantissa) */ - double mnt_log10_fabs; /* [frc] log10(fabs(mantissa))) */ - double val; /* [frc] Copy of input value to avoid indirection */ + double mnt; /* [frc] Mantissa, 0.5 <= mnt < 1.0 */ + double mnt_fabs; /* [frc] fabs(mantissa) */ + double val; /* [frc] Copy of input value to avoid indirection */ double prc_bnr_xct; /* [nbr] Binary digits of precision, exact */ double mss_val_cmp_dbl; /* Missing value for comparison to double precision values */ @@ -517,10 +625,14 @@ ccr_gbr(const int nsd, const int type, const size_t sz, const int has_mss_val, p if (nsd > 16) PUSH_ERR("ccr_bgr", H5E_CALLBACK, "assert(nsd <= 16) failed\n"); - if (type == NC_FLOAT && prc_bnr_xpl_rqr >= bit_xpl_nbr_sgn_flt) - return; - if (type == NC_DOUBLE && prc_bnr_xpl_rqr >= bit_xpl_nbr_sgn_dbl) - return; + /* Note: an early-return check that read prc_bnr_xpl_rqr was previously + * here. It was likely a copy-paste from H5Zbitgroom.c's ccr_bgr(), where + * prc_bnr_xpl_rqr is computed once per call from NSD before the loop. In + * Granular BitRound, prc_bnr_xpl_rqr is computed per value inside the inner + * loop, so at this point in execution it is uninitialized -- the check was + * reading garbage stack memory. Removed because no meaningful "early skip" + * condition can be derived from a per-value variable that has not been + * computed yet. */ switch (type) { case NC_FLOAT: @@ -533,17 +645,22 @@ ccr_gbr(const int nsd, const int type, const size_t sz, const int has_mss_val, p u32_ptr = op1.ui32p; for (idx = 0L; idx < sz; idx++) { if ((val = op1.fp[idx]) != mss_val_cmp_flt && u32_ptr[idx] != 0U) { - mnt = frexp(val, &xpn_bs2); /* DGG19 p. 4102 (8) */ - mnt_fabs = fabs(mnt); - mnt_log10_fabs = log10(mnt_fabs); - /* 20211003 Continuous determination of dgt_nbr improves CR by ~10% */ - dgt_nbr = - (int)floor(xpn_bs2 * dgt_per_bit + mnt_log10_fabs) + 1; /* DGG19 p. 4102 (8.67) */ - qnt_pwr = (int)floor(bit_per_dgt * (dgt_nbr - nsd)); /* DGG19 p. 4101 (7) */ - prc_bnr_xpl_rqr = mnt_fabs == 0.0 + mnt = frexp(val, &xpn_bs2); /* DGG19 p. 4102 (8) */ + mnt_fabs = fabs(mnt); + /* dgt_nbr = floor(log10(|val|)) + 1, computed via deterministic + * table lookup. See ccr_gbr_dgt_nbr() comment block above for + * why the libm log10()/floor() chain was replaced. */ + dgt_nbr = ccr_gbr_dgt_nbr(fabs(val)); + qnt_pwr = (int)floor(bit_per_dgt * (dgt_nbr - nsd)); /* DGG19 p. 4101 (7) */ + /* prc_bnr_xpl_rqr: floor(xpn_bs2 - log2(mnt_fabs)) is xpn_bs2 + * for mnt_fabs in (0.5, 1.0), and xpn_bs2 + 1 when mnt_fabs is + * exactly 0.5 (val is an exact power of 2). This integer-only + * formulation is bit-deterministic across platforms; the + * original libm-based form is not. */ + prc_bnr_xpl_rqr = (mnt_fabs == 0.0) ? 0 - : abs((int)floor(xpn_bs2 - bit_per_dgt * mnt_log10_fabs) - - qnt_pwr); /* Protect against mnt = -0.0 */ + : (unsigned short)abs(((mnt_fabs == 0.5) ? xpn_bs2 + 1 : xpn_bs2) - + qnt_pwr); /* Protect against mnt = -0.0 */ prc_bnr_xpl_rqr--; /* 20211003 Reduce formula result by 1 bit: Passes all tests, improves CR by ~10% */ @@ -574,17 +691,22 @@ ccr_gbr(const int nsd, const int type, const size_t sz, const int has_mss_val, p for (idx = 0L; idx < sz; idx++) { if ((val = op1.dp[idx]) != mss_val_cmp_dbl && u64_ptr[idx] != 0U) { - mnt = frexp(val, &xpn_bs2); /* DGG19 p. 4102 (8) */ - mnt_fabs = fabs(mnt); - mnt_log10_fabs = log10(mnt_fabs); - /* 20211003 Continuous determination of dgt_nbr improves CR by ~10% */ - dgt_nbr = - (int)floor(xpn_bs2 * dgt_per_bit + mnt_log10_fabs) + 1; /* DGG19 p. 4102 (8.67) */ - qnt_pwr = (int)floor(bit_per_dgt * (dgt_nbr - nsd)); /* DGG19 p. 4101 (7) */ - prc_bnr_xpl_rqr = mnt_fabs == 0.0 + mnt = frexp(val, &xpn_bs2); /* DGG19 p. 4102 (8) */ + mnt_fabs = fabs(mnt); + /* dgt_nbr = floor(log10(|val|)) + 1, computed via deterministic + * table lookup. See ccr_gbr_dgt_nbr() comment block above for + * why the libm log10()/floor() chain was replaced. */ + dgt_nbr = ccr_gbr_dgt_nbr(fabs(val)); + qnt_pwr = (int)floor(bit_per_dgt * (dgt_nbr - nsd)); /* DGG19 p. 4101 (7) */ + /* prc_bnr_xpl_rqr: floor(xpn_bs2 - log2(mnt_fabs)) is xpn_bs2 + * for mnt_fabs in (0.5, 1.0), and xpn_bs2 + 1 when mnt_fabs is + * exactly 0.5 (val is an exact power of 2). This integer-only + * formulation is bit-deterministic across platforms; the + * original libm-based form is not. */ + prc_bnr_xpl_rqr = (mnt_fabs == 0.0) ? 0 - : abs((int)floor(xpn_bs2 - bit_per_dgt * mnt_log10_fabs) - - qnt_pwr); /* Protect against mnt = -0.0 */ + : (unsigned short)abs(((mnt_fabs == 0.5) ? xpn_bs2 + 1 : xpn_bs2) - + qnt_pwr); /* Protect against mnt = -0.0 */ prc_bnr_xpl_rqr--; /* 20211003 Reduce formula result by 1 bit: Passes all tests, improves CR by ~10% */ diff --git a/config/cmake/binex/example/CMakeLists.txt b/config/cmake/binex/example/CMakeLists.txt index 26e6f0965..b3e6ca149 100644 --- a/config/cmake/binex/example/CMakeLists.txt +++ b/config/cmake/binex/example/CMakeLists.txt @@ -316,7 +316,6 @@ if (H5PL_BUILD_TESTING) h5repack_layout.h5-ud_pl_lz4_convert.ddl h5repack_layout.h5-ud_pl_blosc_convert.ddl h5repack_layout.h5-ud_pl_bz2_convert.ddl - ud_convert.h5repack_layout.h5.tst ud_bz2_convert.h5repack_layout.h5.tst ud_blosc_convert.h5repack_layout.h5.tst ud_lz4_convert.h5repack_layout.h5.tst diff --git a/config/cmake/binex/example/test/test.sh b/config/cmake/binex/example/test/test.sh index 1d3f71de8..715a617b2 100755 --- a/config/cmake/binex/example/test/test.sh +++ b/config/cmake/binex/example/test/test.sh @@ -89,7 +89,6 @@ $SRC_TESTFILES/h5repack_layout.h5-ud_pl_convert.ddl $SRC_TESTFILES/h5repack_layout.h5-ud_pl_lz4_convert.ddl $SRC_TESTFILES/h5repack_layout.h5-ud_pl_blosc_convert.ddl $SRC_TESTFILES/h5repack_layout.h5-ud_pl_bz2_convert.ddl -$SRC_TESTFILES/ud_convert.h5repack_layout.h5.tst $SRC_TESTFILES/ud_bz2_convert.h5repack_layout.h5.tst $SRC_TESTFILES/ud_blosc_convert.h5repack_layout.h5.tst $SRC_TESTFILES/ud_lz4_convert.h5repack_layout.h5.tst @@ -430,7 +429,5 @@ DUMPTEST h5repack_layout.h5-ud_pl_lz4_convert.ddl --enable-error-stack -pH out-u REPACKTEST1 ud_lz4_convert.h5repack_layout.h5 ud_mafisc_convert.h5repack_layout.h5 --enable-error-stack -v -f /dset4:UD=32013,1,0,0 -l CHUNK=10x10 DUMPTEST h5repack_layout.h5-ud_pl_convert.ddl --enable-error-stack -pH out-ud_mafisc_convert.h5repack_layout.h5 -#rm -f ud_convert.h5repack_layout.h5 - echo "$nerrors tests failed in example" exit $nerrors diff --git a/config/cmake/binex/example/test/testCM.sh b/config/cmake/binex/example/test/testCM.sh index 94a6f6647..d7a7fd552 100755 --- a/config/cmake/binex/example/test/testCM.sh +++ b/config/cmake/binex/example/test/testCM.sh @@ -87,7 +87,6 @@ $SRC_TESTFILES/h5repack_layout.h5-ud_pl_convert.ddl $SRC_TESTFILES/h5repack_layout.h5-ud_pl_lz4_convert.ddl $SRC_TESTFILES/h5repack_layout.h5-ud_pl_blosc_convert.ddl $SRC_TESTFILES/h5repack_layout.h5-ud_pl_bz2_convert.ddl -$SRC_TESTFILES/ud_convert.h5repack_layout.h5.tst $SRC_TESTFILES/ud_bz2_convert.h5repack_layout.h5.tst $SRC_TESTFILES/ud_blosc_convert.h5repack_layout.h5.tst $SRC_TESTFILES/ud_lz4_convert.h5repack_layout.h5.tst @@ -428,7 +427,5 @@ DUMPTEST h5repack_layout.h5-ud_pl_lz4_convert.ddl --enable-error-stack -pH out-u REPACKTEST1 ud_lz4_convert.h5repack_layout.h5 ud_mafisc_convert.h5repack_layout.h5 --enable-error-stack -v -f /dset4:UD=32013,1,0,0 -l CHUNK=10x10 DUMPTEST h5repack_layout.h5-ud_pl_convert.ddl --enable-error-stack -pH out-ud_mafisc_convert.h5repack_layout.h5 -#rm -f ud_convert.h5repack_layout.h5 - echo "$nerrors tests failed in example" exit $nerrors diff --git a/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl b/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl deleted file mode 100644 index 6ffa71dcc..000000000 --- a/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl +++ /dev/null @@ -1,158 +0,0 @@ -HDF5 "out-ud_convert.h5repack_layout.h5" { -GROUP "/" { - DATASET "dset1" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 40, 20 ) - SIZE 804 (3.980:1 COMPRESSION) - } - FILTERS { - USER_DEFINED_FILTER { - FILTER_ID 307 - COMMENT HDF5 bzip2 filter; see https://github.com/HDFGroup/hdf5_plugins/blob/master/docs/RegisteredFilterPlugins.md - PARAMS { 9 } - } - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset2" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 408 (7.843:1 COMPRESSION) - } - FILTERS { - USER_DEFINED_FILTER { - FILTER_ID 32013 - COMMENT H5Z-ZFP-1.1.1 (ZFP-1.0.1) github.com/LLNL/H5Z-ZFP - PARAMS { XXXX } - } - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset3" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 408 (7.843:1 COMPRESSION) - } - FILTERS { - USER_DEFINED_FILTER { - FILTER_ID 32013 - COMMENT H5Z-ZFP-1.1.1 (ZFP-1.0.1) github.com/LLNL/H5Z-ZFP - PARAMS { XXXX } - } - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset4" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 408 (7.843:1 COMPRESSION) - } - FILTERS { - USER_DEFINED_FILTER { - FILTER_ID 32013 - COMMENT H5Z-ZFP-1.1.1 (ZFP-1.0.1) github.com/LLNL/H5Z-ZFP - PARAMS { XXXX } - } - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_chunk" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, H5S_UNLIMITED ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 408 (7.843:1 COMPRESSION) - } - FILTERS { - USER_DEFINED_FILTER { - FILTER_ID 32013 - COMMENT H5Z-ZFP-1.1.1 (ZFP-1.0.1) github.com/LLNL/H5Z-ZFP - PARAMS { XXXX } - } - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_compact" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 408 (7.843:1 COMPRESSION) - } - FILTERS { - USER_DEFINED_FILTER { - FILTER_ID 32013 - COMMENT H5Z-ZFP-1.1.1 (ZFP-1.0.1) github.com/LLNL/H5Z-ZFP - PARAMS { XXXX } - } - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_contiguous" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } - STORAGE_LAYOUT { - CHUNKED ( 4, 8 ) - SIZE 408 (7.843:1 COMPRESSION) - } - FILTERS { - USER_DEFINED_FILTER { - FILTER_ID 32013 - COMMENT H5Z-ZFP-1.1.1 (ZFP-1.0.1) github.com/LLNL/H5Z-ZFP - PARAMS { XXXX } - } - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE H5D_FILL_VALUE_DEFAULT - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } -} -} diff --git a/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst b/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst deleted file mode 100644 index 32a04e120..000000000 --- a/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst +++ /dev/null @@ -1,16 +0,0 @@ -All objects to modify layout are... - Apply chunked layout to allwith dimension [ 4 8 ] -All objects to apply filter are... - User Defined 32013 -Making new file ... ------------------------------------------ - Type Filter (Compression) Name ------------------------------------------ - group / - dset UD (7.843:1) /dset1 - dset UD (7.843:1) /dset2 - dset UD (7.843:1) /dset3 - dset UD (7.843:1) /dset4 - dset UD (7.843:1) /dset_chunk - dset UD (7.843:1) /dset_compact - dset UD (7.843:1) /dset_contiguous