diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c06f2e1bb98..0f004561744 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2519,12 +2519,18 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, (unsigned)pledgedSrcSize); if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { - return ZSTD_resetCCtx_byAttachingCDict( - cctx, cdict, *params, pledgedSrcSize, zbuff); + FORWARD_IF_ERROR(ZSTD_resetCCtx_byAttachingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff), ""); } else { - return ZSTD_resetCCtx_byCopyingCDict( - cctx, cdict, *params, pledgedSrcSize, zbuff); + FORWARD_IF_ERROR(ZSTD_resetCCtx_byCopyingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff), ""); } + + /* A byRef CDict's window still points into the caller's buffer, which may be + * adjacent to the input : same non-determinism as refPrefix. No-op when attached. */ + cctx->blockState.matchState.forceNonContiguous = params->deterministicRefPrefix; + + return 0; } /*! ZSTD_copyCCtx_internal() : diff --git a/lib/zstd.h b/lib/zstd.h index 97fef316fd6..c9af9059e06 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -2278,6 +2278,9 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo * contiguous, and is free if they weren't contiguous. We don't expect that * intentionally making the dictionary and data contiguous will be worth the * cost to memcpy() the data. + * + * This covers ZSTD_CCtx_refPrefix(), ZSTD_CCtx_loadDictionary_byReference(), + * and CDicts created with ZSTD_dlm_byRef. */ #define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 7b7c9d66631..1c962c29b46 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1516,6 +1516,56 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : testing CDict by reference for determinism : ", testNb++); + { /* Same contract as the test above, for a CDict referencing the caller's + * buffer. ZSTD_dictForceCopy selects the affected path. */ + size_t const testSize = 128 KB; + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + char* const dict = (char*)malloc(2 * testSize); + int level; + + if (cctx == NULL || dict == NULL) { + DISPLAY("Not enough memory, aborting\n"); + testResult = 1; + goto _end; + } + RDG_genBuffer(dict, testSize, 0.5, 0.5, seed); + RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed); + memcpy(dict + testSize, CNBuffer, testSize); + + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceCopy)); + for (level = 1; level <= 5; ++level) { + ZSTD_CDict* const cdict = ZSTD_createCDict_byReference(dict, testSize, level); + size_t cSize0; + XXH64_hash_t compressedChecksum0; + + if (cdict == NULL) { + DISPLAY("Not enough memory, aborting\n"); + testResult = 1; + goto _end; + } + + CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize); + CHECK_Z(cSize); + cSize0 = cSize; + compressedChecksum0 = XXH64(compressedBuffer, cSize, 0); + + CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize); + CHECK_Z(cSize); + ZSTD_freeCDict(cdict); + + if (cSize != cSize0) goto _output_error; + if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error; + } + + ZSTD_freeCCtx(cctx); + free(dict); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++); { ZSTD_CCtx* cctx = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx();