diff --git a/ZSTD/config/cmake/runTest.cmake b/ZSTD/config/cmake/runTest.cmake index edf46a867..5409772ec 100644 --- a/ZSTD/config/cmake/runTest.cmake +++ b/ZSTD/config/cmake/runTest.cmake @@ -362,12 +362,14 @@ if (TEST_GREP_COMPARE) message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did not contain ${TEST_REFERENCE}") endif () - string (REGEX MATCH "${TEST_FILTER}" TEST_MATCH ${TEST_STREAM}) - if (TEST_EXPECT) - # TEST_EXPECT (1) interprets TEST_FILTER as; NOT to match - string (LENGTH "${TEST_MATCH}" TEST_GREP_RESULT) - if (TEST_GREP_RESULT) - message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did contain ${TEST_FILTER}") + if (TEST_FILTER) + string (REGEX MATCH "${TEST_FILTER}" TEST_MATCH ${TEST_STREAM}) + if (TEST_EXPECT) + # TEST_EXPECT (1) interprets TEST_FILTER as; NOT to match + string (LENGTH "${TEST_MATCH}" TEST_GREP_RESULT) + if (TEST_GREP_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did contain ${TEST_FILTER}") + endif () endif () endif () endif () diff --git a/ZSTD/example/CMakeLists.txt b/ZSTD/example/CMakeLists.txt index 44e946325..378040234 100644 --- a/ZSTD/example/CMakeLists.txt +++ b/ZSTD/example/CMakeLists.txt @@ -120,6 +120,33 @@ if (H5PL_BUILD_TESTING) endif () endmacro () + # Test that reading a .h5 with corrupt zstd data properly throws an error + # via the zstd filter + macro (ADD_H5_DUMPFAIL_TEST testname errmsg) + if (HDF5_BUILD_TOOLS OR HDF5_PROVIDES_TOOLS) + add_test ( + NAME H5DUMP-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5PL_HDF5_DUMP_EXECUTABLE}" + -D "TEST_ARGS:STRING=--enable-error-stack;-d;/DS1;${testname}.h5" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_EXPECT=1" + -D "ERROR_APPEND=1" + -D "TEST_GREP_COMPARE=1" + -D "TEST_OUTPUT=${testname}.out" + -D "TEST_REFERENCE=${errmsg}" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -D "TEST_ENV_VAR=HDF5_PLUGIN_PATH" + -D "TEST_ENV_VALUE=${CMAKE_BINARY_DIR}/plugins" + -P "${H5ZSTD_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5DUMP-${testname} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS ${last_test}) + set (last_test "H5DUMP-${testname}") + endif () + endmacro () + macro (ADD_H5_UD_TEST testname resultcode resultfile) # Remove any output file left over from previous test run add_test ( @@ -183,6 +210,8 @@ if (H5PL_BUILD_TESTING) # Copy all the HDF5 files from the source directory into the test directory # -------------------------------------------------------------------- set (LIST_HDF5_TEST_FILES + h5ex_d_zstd_corrupt.h5 + h5ex_d_zstd_streaming.h5 # h5repack_layout.h5 ) set (LIST_OTHER_TEST_FILES @@ -199,6 +228,9 @@ if (H5PL_BUILD_TESTING) ADD_H5_TEST (h5ex_d_zstd) + ADD_H5_DUMPFAIL_TEST (h5ex_d_zstd_corrupt "Input is not a valid zstd frame") + ADD_H5_DUMPFAIL_TEST (h5ex_d_zstd_streaming "zstd frame missing decompressed size") + # if (NOT DISABLE_H5ZSTD_ENCODER) # #UD ZSTD # ADD_H5_UD_TEST (ud_convert 0 h5repack_layout.h5 --enable-error-stack -v -f UD=32015,0,0) diff --git a/ZSTD/example/testfiles/h5ex_d_zstd_corrupt.h5 b/ZSTD/example/testfiles/h5ex_d_zstd_corrupt.h5 new file mode 100644 index 000000000..da9024b79 Binary files /dev/null and b/ZSTD/example/testfiles/h5ex_d_zstd_corrupt.h5 differ diff --git a/ZSTD/example/testfiles/h5ex_d_zstd_streaming.h5 b/ZSTD/example/testfiles/h5ex_d_zstd_streaming.h5 new file mode 100644 index 000000000..08cdafede Binary files /dev/null and b/ZSTD/example/testfiles/h5ex_d_zstd_streaming.h5 differ diff --git a/ZSTD/src/H5Zzstd.c b/ZSTD/src/H5Zzstd.c index b96fa84c6..4c4e11d69 100644 --- a/ZSTD/src/H5Zzstd.c +++ b/ZSTD/src/H5Zzstd.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -24,6 +25,8 @@ static size_t H5Z_filter_zstd(unsigned int flags, size_t cd_nelmts, const unsign #define PUSH_ERR(func, minor, str) \ H5Epush(H5E_DEFAULT, __FILE__, func, __LINE__, H5E_ERR_CLS, H5E_PLINE, minor, str) +#define PUSH_ERR2(func, minor, str, arg) \ + H5Epush(H5E_DEFAULT, __FILE__, func, __LINE__, H5E_ERR_CLS, H5E_PLINE, minor, str, arg) const H5Z_class2_t H5Z_ZSTD[1] = {{ H5Z_CLASS_T_VERS, /* H5Z_class_t version */ @@ -65,14 +68,42 @@ H5Z_filter_zstd(unsigned int flags, size_t cd_nelmts, const unsigned int cd_valu if (flags & H5Z_FLAG_REVERSE) { /* We're decompressing */ - size_t decompSize = ZSTD_getFrameContentSize(*buf, origSize); - if (NULL == (outbuf = malloc(decompSize))) + unsigned long long contentSize = ZSTD_getFrameContentSize(*buf, origSize); + if (contentSize == ZSTD_CONTENTSIZE_ERROR) { + PUSH_ERR("H5Z_filter_zstd", H5E_CALLBACK, "Input is not a valid zstd frame"); goto error; + } + if (contentSize == ZSTD_CONTENTSIZE_UNKNOWN) { + PUSH_ERR("H5Z_filter_zstd", H5E_CALLBACK, + "zstd frame missing decompressed size; data was likely compressed " + "with the zstd streaming API, which is not supported"); + goto error; + } + if (contentSize == 0) { + PUSH_ERR("H5Z_filter_zstd", H5E_CALLBACK, "zstd frame has zero decompressed size"); + goto error; + } + if (contentSize > SIZE_MAX) { + PUSH_ERR("H5Z_filter_zstd", H5E_CALLBACK, + "zstd frame decompressed size exceeds addressable memory"); + goto error; + } + size_t decompCapacity = (size_t)contentSize; - decompSize = ZSTD_decompress(outbuf, decompSize, inbuf, origSize); + if (NULL == (outbuf = malloc(decompCapacity))) { + PUSH_ERR("H5Z_filter_zstd", H5E_CALLBACK, "Can't allocate zstd decompression buffer"); + goto error; + } + + size_t decompSize = ZSTD_decompress(outbuf, decompCapacity, inbuf, origSize); + if (ZSTD_isError(decompSize)) { + PUSH_ERR2("H5Z_filter_zstd", H5E_CALLBACK, "zstd decompression failed: %s", + ZSTD_getErrorName(decompSize)); + goto error; + } #ifdef ZSTD_DEBUG - fprintf(stderr, " decompressing nbytes: %ld\n", decompSize); + fprintf(stderr, " decompressing nbytes: %zu\n", decompSize); #endif buf_size_out = decompSize; @@ -100,13 +131,20 @@ H5Z_filter_zstd(unsigned int flags, size_t cd_nelmts, const unsigned int cd_valu aggression = ZSTD_maxCLevel(); size_t compSize = ZSTD_compressBound(origSize); - if (NULL == (outbuf = malloc(compSize))) + if (NULL == (outbuf = malloc(compSize))) { + PUSH_ERR("H5Z_filter_zstd", H5E_CALLBACK, "Can't allocate zstd compression buffer"); goto error; + } compSize = ZSTD_compress(outbuf, compSize, inbuf, origSize, aggression); + if (ZSTD_isError(compSize)) { + PUSH_ERR2("H5Z_filter_zstd", H5E_CALLBACK, "zstd compression failed: %s", + ZSTD_getErrorName(compSize)); + goto error; + } #ifdef ZSTD_DEBUG - fprintf(stderr, " compressing nbytes: %ld\n", compSize); + fprintf(stderr, " compressing nbytes: %zu\n", compSize); #endif buf_size_out = compSize;