diff --git a/CMakeLists.txt b/CMakeLists.txt index 41fc9a1..f140aee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,6 +193,33 @@ if (Libpsl_FOUND) target_compile_definitions(boost_burl PRIVATE BOOST_BURL_HAS_LIBPSL) endif () +#------------------------------------------------- +# +# Compression backends (optional) +# +#------------------------------------------------- + +# zlib: deflate and gzip content codings +find_package(ZLIB) +if (ZLIB_FOUND) + target_link_libraries(boost_burl PRIVATE ZLIB::ZLIB) + target_compile_definitions(boost_burl PUBLIC BOOST_BURL_HAS_ZLIB) +endif () + +# Brotli: br content coding (decoding only) +find_package(Brotli) +if (Brotli_FOUND) + target_link_libraries(boost_burl PRIVATE Brotli::common Brotli::decoder) + target_compile_definitions(boost_burl PUBLIC BOOST_BURL_HAS_BROTLI) +endif () + +# Zstandard: zstd content coding +find_package(Zstd) +if (Zstd_FOUND) + target_link_libraries(boost_burl PRIVATE Zstd::Zstd) + target_compile_definitions(boost_burl PUBLIC BOOST_BURL_HAS_ZSTD) +endif () + #------------------------------------------------- # # Tests diff --git a/build/Jamfile b/build/Jamfile index d5b7d4a..2a3e016 100644 --- a/build/Jamfile +++ b/build/Jamfile @@ -12,6 +12,7 @@ import config : requires ; using openssl ; using zlib ; +using brotli ; using zstd ; constant c11-requires : @@ -40,8 +41,9 @@ lib boost_burl /boost//capy /boost//corosio [ ac.check-library /boost/corosio//boost_corosio_openssl : /boost/corosio//boost_corosio_openssl : ] - [ ac.check-library /zlib//zlib : /zlib//zlib : ] - [ ac.check-library /zstd//zstd : /zstd//zstd : ] + [ ac.check-library /zlib//zlib : /zlib//zlib BOOST_BURL_HAS_ZLIB : ] + [ ac.check-library /brotli//brotlidec : /brotli//brotlicommon /brotli//brotlidec BOOST_BURL_HAS_BROTLI : ] + [ ac.check-library /zstd//zstd : /zstd//zstd BOOST_BURL_HAS_ZSTD : ] /boost//http /boost//json /boost//url @@ -51,8 +53,9 @@ lib boost_burl /boost//capy /boost//corosio [ ac.check-library /boost/corosio//boost_corosio_openssl : /boost/corosio//boost_corosio_openssl : ] - [ ac.check-library /zlib//zlib : /zlib//zlib : ] - [ ac.check-library /zstd//zstd : /zstd//zstd : ] + [ ac.check-library /zlib//zlib : /zlib//zlib BOOST_BURL_HAS_ZLIB : ] + [ ac.check-library /brotli//brotlidec : /brotli//brotlicommon /brotli//brotlidec BOOST_BURL_HAS_BROTLI : ] + [ ac.check-library /zstd//zstd : /zstd//zstd BOOST_BURL_HAS_ZSTD : ] /boost//http /boost//json /boost//url diff --git a/cmake/FindBrotli.cmake b/cmake/FindBrotli.cmake new file mode 100644 index 0000000..cb6f884 --- /dev/null +++ b/cmake/FindBrotli.cmake @@ -0,0 +1,50 @@ +# +# Copyright (c) 2026 Mohammad Nejati +# +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# +# Official repository: https://github.com/cppalliance/burl +# + +# Provides imported targets: +# Brotli::common +# Brotli::decoder +# Brotli::encoder + +find_path(Brotli_INCLUDE_DIR NAMES "brotli/decode.h") +find_library(Brotli_COMMON_LIBRARY NAMES "brotlicommon") +find_library(Brotli_DEC_LIBRARY NAMES "brotlidec") +find_library(Brotli_ENC_LIBRARY NAMES "brotlienc") + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Brotli + REQUIRED_VARS + Brotli_INCLUDE_DIR + Brotli_COMMON_LIBRARY + Brotli_DEC_LIBRARY + Brotli_ENC_LIBRARY +) + +if(Brotli_FOUND) + add_library(Brotli::common UNKNOWN IMPORTED) + set_target_properties(Brotli::common PROPERTIES + IMPORTED_LOCATION "${Brotli_COMMON_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${Brotli_INCLUDE_DIR}") + + add_library(Brotli::decoder UNKNOWN IMPORTED) + set_target_properties(Brotli::decoder PROPERTIES + IMPORTED_LOCATION "${Brotli_DEC_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${Brotli_INCLUDE_DIR}") + + add_library(Brotli::encoder UNKNOWN IMPORTED) + set_target_properties(Brotli::encoder PROPERTIES + IMPORTED_LOCATION "${Brotli_ENC_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${Brotli_INCLUDE_DIR}") +endif() + +mark_as_advanced( + Brotli_INCLUDE_DIR + Brotli_COMMON_LIBRARY + Brotli_DEC_LIBRARY + Brotli_ENC_LIBRARY) diff --git a/cmake/FindZstd.cmake b/cmake/FindZstd.cmake new file mode 100644 index 0000000..740ea43 --- /dev/null +++ b/cmake/FindZstd.cmake @@ -0,0 +1,33 @@ +# +# Copyright (c) 2026 Mohammad Nejati +# +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# +# Official repository: https://github.com/cppalliance/burl +# + +# Provides imported targets: +# Zstd::Zstd + +find_path(Zstd_INCLUDE_DIR NAMES "zstd.h") +find_library(Zstd_LIBRARY NAMES zstd libzstd zstd_static) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Zstd + REQUIRED_VARS + Zstd_INCLUDE_DIR + Zstd_LIBRARY +) + +if(Zstd_FOUND) + add_library(Zstd::Zstd UNKNOWN IMPORTED) + set_target_properties(Zstd::Zstd PROPERTIES + IMPORTED_LOCATION "${Zstd_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${Zstd_INCLUDE_DIR}" + ) +endif() + +mark_as_advanced( + Zstd_INCLUDE_DIR + Zstd_LIBRARY) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 44d6e90..740604d 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -10,14 +10,6 @@ add_executable(burl_example_usage usage.cpp) target_link_libraries(burl_example_usage PRIVATE Boost::burl Boost::hash2) -if (TARGET Boost::http_zlib) - target_link_libraries(burl_example_usage PRIVATE Boost::http_zlib) -endif() - -if (TARGET Boost::http_brotli) - target_link_libraries(burl_example_usage PRIVATE Boost::http_brotli) -endif() - find_package(nlohmann_json QUIET) if (nlohmann_json_FOUND) add_executable(burl_example_nlohmann_json nlohmann_json.cpp) diff --git a/example/usage.cpp b/example/usage.cpp index 86e924a..df6ea9c 100644 --- a/example/usage.cpp +++ b/example/usage.cpp @@ -11,8 +11,6 @@ #include #include #include -#include -#include #include #include @@ -569,13 +567,6 @@ main(int argc, char* argv[]) corosio::io_context ioc; corosio::tls_context tls_ctx; -#ifdef BOOST_HTTP_HAS_BROTLI - http::brotli::install_decode_service(capy::get_system_context()); -#endif -#ifdef BOOST_HTTP_HAS_ZLIB - http::zlib::install_inflate_service(capy::get_system_context()); -#endif - capy::run_async( ioc.get_executor(), [] {}, diff --git a/include/boost/burl/client.hpp b/include/boost/burl/client.hpp index 189b746..b0d7d70 100644 --- a/include/boost/burl/client.hpp +++ b/include/boost/burl/client.hpp @@ -140,11 +140,11 @@ class client When enabled, `br` is included in the `Accept-Encoding` header and response bodies are decoded transparently. - Effective only when the Brotli decode - service is installed in the system - context. Not applied when the request - carries an explicit `Accept-Encoding` - header. + Effective only when the library was built + with Brotli support + (`BOOST_BURL_HAS_BROTLI`). Not applied + when the request carries an explicit + `Accept-Encoding` header. */ bool brotli = true; @@ -154,9 +154,9 @@ class client the `Accept-Encoding` header and response bodies are decoded transparently. Effective only when the - zlib inflate service is installed in the - system context. Not applied when the - request carries an explicit + library was built with zlib support + (`BOOST_BURL_HAS_ZLIB`). Not applied when + the request carries an explicit `Accept-Encoding` header. */ bool deflate = true; @@ -166,14 +166,27 @@ class client When enabled, `gzip` is included in the `Accept-Encoding` header and response bodies are decoded transparently. - Effective only when the zlib inflate - service is installed in the system - context. Not applied when the request - carries an explicit `Accept-Encoding` - header. + Effective only when the library was built + with zlib support + (`BOOST_BURL_HAS_ZLIB`). Not applied when + the request carries an explicit + `Accept-Encoding` header. */ bool gzip = true; + /** Advertise and decode the zstd content coding. + + When enabled, `zstd` is included in the + `Accept-Encoding` header and response + bodies are decoded transparently. + Effective only when the library was built + with zstd support + (`BOOST_BURL_HAS_ZSTD`). Not applied when + the request carries an explicit + `Accept-Encoding` header. + */ + bool zstd = true; + /** Maximum allowed size of a response body. Reading a body which exceeds the limit diff --git a/include/boost/burl/detail/parser.hpp b/include/boost/burl/detail/parser.hpp index e1e657e..a246b5b 100644 --- a/include/boost/burl/detail/parser.hpp +++ b/include/boost/burl/detail/parser.hpp @@ -165,8 +165,7 @@ class parser capy::io_task do_read_some( - std::span buffers, - bool skip_out = false); + std::span buffers); struct header_deleter { diff --git a/include/boost/burl/error.hpp b/include/boost/burl/error.hpp index fa5d275..7c8eb57 100644 --- a/include/boost/burl/error.hpp +++ b/include/boost/burl/error.hpp @@ -72,6 +72,14 @@ enum class error request. */ body_size_mismatch, + + /** The response body could not be decoded. + + Decoding the response body according to its + `Content-Encoding` failed because the compressed + data was corrupt or invalid. + */ + decode_error, }; /** Error conditions corresponding to sets of error codes. diff --git a/src/client.cpp b/src/client.cpp index 9f8dcf2..de37bb0 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -20,16 +20,13 @@ #include #include -#include #include #include #include -#include #include #include #include #include -#include #include #include @@ -66,6 +63,9 @@ set_accept_encoding( if(cfg.gzip) accept("gzip"); + if(cfg.zstd) + accept("zstd"); + if(!accept_encoding.empty()) headers.set(http::field::accept_encoding, accept_encoding); } @@ -96,12 +96,16 @@ client::client( std::make_shared( exec, std::move(tls_ctx), cfg)) { - // Disable codings whose decoder service is unavailable. - auto const& ctx = capy::get_system_context(); - if(!ctx.has_service()) - config_.brotli = false; - if(!ctx.has_service()) - config_.deflate = config_.gzip = false; + // Disable codings whose decoder was not compiled in. +#ifndef BOOST_BURL_HAS_BROTLI + config_.brotli = false; +#endif +#ifndef BOOST_BURL_HAS_ZLIB + config_.deflate = config_.gzip = false; +#endif +#ifndef BOOST_BURL_HAS_ZSTD + config_.zstd = false; +#endif } void diff --git a/src/detail/decoders.cpp b/src/detail/decoders.cpp index 4404d9a..e743d6f 100644 --- a/src/detail/decoders.cpp +++ b/src/detail/decoders.cpp @@ -9,19 +9,27 @@ #include "decoders.hpp" +#include + #include -#include -#include -#include -#include -#include -#include -#include +#include #include #include #include +#ifdef BOOST_BURL_HAS_ZLIB +#include +#endif + +#ifdef BOOST_BURL_HAS_BROTLI +#include +#endif + +#ifdef BOOST_BURL_HAS_ZSTD +#include +#endif + namespace boost { namespace burl @@ -32,27 +40,24 @@ namespace detail namespace { +#ifdef BOOST_BURL_HAS_ZLIB class zlib_decoder final : public parser::decoder { - http::zlib::inflate_service& svc_; - http::zlib::stream strm_ = {}; + z_stream strm_ = {}; public: - zlib_decoder( - http::zlib::inflate_service& svc, - int window_bits) - : svc_(svc) + // window_bits: 15 for zlib/deflate, 15 + 16 for gzip. + explicit + zlib_decoder(int window_bits) { - std::error_code ec = static_cast( - svc_.init2(strm_, window_bits)); - if(ec != http::zlib::error::ok) - throw std::system_error(ec); + if(inflateInit2(&strm_, window_bits) != Z_OK) + throw std::bad_alloc(); } ~zlib_decoder() override { - svc_.inflate_end(strm_); + inflateEnd(&strm_); } result @@ -68,19 +73,15 @@ class zlib_decoder final static_cast(out.data()); strm_.avail_out = saturate(out.size()); - auto const rs = static_cast( - svc_.inflate( - strm_, - eof ? http::zlib::finish - : http::zlib::no_flush)); + auto const rs = ::inflate( + &strm_, eof ? Z_FINISH : Z_NO_FLUSH); auto const ec = [&]() -> std::error_code { - if(rs == http::zlib::error::stream_end) + if(rs == Z_STREAM_END) return capy::error::eof; - if(rs < http::zlib::error::ok && - rs != http::zlib::error::buf_err) - return rs; + if(rs != Z_OK && rs != Z_BUF_ERROR) + return error::decode_error; return {}; }(); @@ -101,18 +102,17 @@ class zlib_decoder final return static_cast(n); } }; +#endif // BOOST_BURL_HAS_ZLIB +#ifdef BOOST_BURL_HAS_BROTLI class brotli_decoder final : public parser::decoder { - http::brotli::decode_service& svc_; - http::brotli::decoder_state* state_; + BrotliDecoderState* state_; public: - brotli_decoder( - http::brotli::decode_service& svc) - : svc_(svc) - , state_(svc.create_instance( + brotli_decoder() + : state_(BrotliDecoderCreateInstance( nullptr, nullptr, nullptr)) { if(!state_) @@ -121,7 +121,7 @@ class brotli_decoder final ~brotli_decoder() override { - svc_.destroy_instance(state_); + BrotliDecoderDestroyInstance(state_); } result @@ -135,7 +135,7 @@ class brotli_decoder final auto* next_out = static_cast(out.data()); auto available_out = out.size(); - auto const rs = svc_.decompress_stream( + auto const rs = BrotliDecoderDecompressStream( state_, &available_in, &next_in, @@ -145,10 +145,10 @@ class brotli_decoder final auto const ec = [&]() -> std::error_code { - if(svc_.is_finished(state_)) + if(BrotliDecoderIsFinished(state_)) return capy::error::eof; - if(rs == http::brotli::decoder_result::error) - return svc_.get_error_code(state_); + if(rs == BROTLI_DECODER_RESULT_ERROR) + return error::decode_error; return {}; }(); @@ -158,33 +158,78 @@ class brotli_decoder final .ec = ec }; } }; +#endif // BOOST_BURL_HAS_BROTLI + +#ifdef BOOST_BURL_HAS_ZSTD +class zstd_decoder final + : public parser::decoder +{ + ZSTD_DStream* strm_; + +public: + zstd_decoder() + : strm_(ZSTD_createDStream()) + { + if(!strm_) + throw std::bad_alloc(); + ZSTD_initDStream(strm_); + } + + ~zstd_decoder() override + { + ZSTD_freeDStream(strm_); + } + + result + process( + capy::mutable_buffer out, + capy::const_buffer in, + bool eof) override + { + ZSTD_inBuffer in_buf{ in.data(), in.size(), 0 }; + ZSTD_outBuffer out_buf{ out.data(), out.size(), 0 }; + + auto const rs = + ZSTD_decompressStream(strm_, &out_buf, &in_buf); + + auto const ec = [&]() -> std::error_code + { + if(rs == 0) + return capy::error::eof; + if(ZSTD_isError(rs)) + return error::decode_error; + return {}; + }(); + + return { + .consumed = in_buf.pos, + .produced = out_buf.pos, + .ec = ec }; + } +}; +#endif // BOOST_BURL_HAS_ZSTD } // namespace std::unique_ptr make_decoder(http::content_coding coding) { - auto& ctx = capy::get_system_context(); switch(coding) { +#ifdef BOOST_BURL_HAS_ZLIB case http::content_coding::deflate: - if(auto* svc = ctx.find_service< - http::zlib::inflate_service>()) - return std::make_unique( - *svc, 15); - break; + return std::make_unique(15); case http::content_coding::gzip: - if(auto* svc = ctx.find_service< - http::zlib::inflate_service>()) - return std::make_unique( - *svc, 15 + 16); - break; + return std::make_unique(15 + 16); +#endif +#ifdef BOOST_BURL_HAS_BROTLI case http::content_coding::br: - if(auto* svc = ctx.find_service< - http::brotli::decode_service>()) - return std::make_unique( - *svc); - break; + return std::make_unique(); +#endif +#ifdef BOOST_BURL_HAS_ZSTD + case http::content_coding::zstd: + return std::make_unique(); +#endif default: break; } diff --git a/src/detail/parser.cpp b/src/detail/parser.cpp index c7d8fdb..8048a40 100644 --- a/src/detail/parser.cpp +++ b/src/detail/parser.cpp @@ -140,11 +140,29 @@ class chained_sequence }; std::error_code -parse_chunk_size( +skip_to_eol(chained_sequence& cs) noexcept +{ + while(!cs.empty()) + { + if(cs.value() == '\r') + { + if(!cs.next()) + break; + if(cs.value() != '\n') + return bad_payload; + cs.next(); + return {}; + } + cs.next(); + } + return need_data; +} + +std::error_code +parse_chunk_header( chained_sequence& cs, std::uint64_t& size) noexcept { - size = 0; for(auto const start = cs.size();;) { if(cs.empty()) @@ -154,7 +172,8 @@ parse_chunk_size( { if(start == cs.size()) return bad_payload; - return {}; + // skip chunk header's exts + return skip_to_eol(cs); } // at least 4 significant bits are free if(size > (std::numeric_limits::max)() >> 4) @@ -164,25 +183,6 @@ parse_chunk_size( } } -std::error_code -skip_to_eol(chained_sequence& cs) noexcept -{ - while(!cs.empty()) - { - if(cs.value() == '\r') - { - if(!cs.next()) - break; - if(cs.value() != '\n') - return bad_payload; - cs.next(); - return {}; - } - cs.next(); - } - return need_data; -} - std::error_code skip_trailer(chained_sequence& cs) noexcept { @@ -487,10 +487,7 @@ walk_chunks( goto invoke; loop: - if(auto ec = parse_chunk_size(cs, size); ec) - return ec; - - if(auto ec = skip_to_eol(cs); ec) + if(auto ec = parse_chunk_header(cs, size); ec) return ec; // final chunk @@ -562,11 +559,8 @@ flatten_chunks() return bail(ec); } - std::uint64_t size; - if(auto ec = parse_chunk_size(cs, size); ec) - return bail(ec); - - if(auto ec = skip_to_eol(cs); ec) + std::uint64_t size = 0; + if(auto ec = parse_chunk_header(cs, size); ec) return bail(ec); if(size == 0) @@ -749,6 +743,7 @@ http::static_response const& parser:: get_response() const { + BOOST_ASSERT(h_); return reinterpret_cast< http::static_response const&>(*h_); } @@ -757,6 +752,7 @@ http::static_request const& parser:: get_request() const { + BOOST_ASSERT(h_); return reinterpret_cast< http::static_request const&>(*h_); } @@ -770,8 +766,9 @@ decode_some( co_return { {}, 0 }; auto slice = capy::buffer_slice(buffers); - auto prod = std::size_t(0); - auto pump = [&](capy::const_buffer in, bool last) + std::size_t prod = 0; + auto decode = + [&](capy::const_buffer in, bool last) -> capy::io_result { if(dec_err_) @@ -786,10 +783,10 @@ decode_some( std::size_t cons = 0; for(;;) { - auto const lim = dec_limit_rem(); auto const out = capy::front(slice.data()); if(out.size() == 0) return { {}, cons }; + auto const lim = dec_limit_rem(); if(lim == 0) return { body_too_large, cons }; auto const r = dec_->process( @@ -807,8 +804,7 @@ decode_some( } if(r.produced == 0 && r.consumed == 0) { - // TODO: dedicated error code - dec_err_ = bad_payload; + dec_err_ = error::decode_error; return { {}, cons }; } if(in.size() == 0) @@ -827,7 +823,7 @@ decode_some( { for(;;) { - auto ec = walk_chunks(pump); + auto ec = walk_chunks(decode); if(prod != 0) co_return { {}, prod }; if(ec != need_more_input) @@ -849,7 +845,7 @@ decode_some( co_return { fec, 0 }; continue; } - auto [ec, cons] = pump(in, got_body_ && in.size() == rem); + auto [ec, cons] = decode(in, got_body_ && in.size() == rem); in_.consume(cons); if(prod != 0) co_return { {}, prod }; @@ -863,21 +859,21 @@ decode_some( capy::io_task parser:: do_read_some( - std::span buffers, - bool skip_out) + std::span buffers) { if(auto [ec] = co_await read_header(); ec) co_return { ec, 0 }; - if(!out_.empty() && !skip_out) - { - auto const n = capy::buffer_copy(buffers, out_.data()); - out_.consume(n); - co_return { {}, n }; - } - if(dec_) + { + if(!out_.empty()) + { + auto const n = capy::buffer_copy(buffers, out_.data()); + out_.consume(n); + co_return { {}, n }; + } co_return co_await decode_some(buffers); + } auto copy = [&](std::size_t at_most) { @@ -985,11 +981,10 @@ pull(std::span dest) if(auto [ec] = co_await read_header(); ec) co_return { ec, {} }; - if(!out_.empty()) - co_return { {}, collect(dest, out_.data()) }; - if(dec_) { + if(!out_.empty()) + co_return { {}, collect(dest, out_.data()) }; auto [ec, n] = co_await decode_some(out_.prepare()); out_.commit(n); if(ec && n == 0) @@ -1077,7 +1072,7 @@ void parser:: consume(std::size_t n) noexcept { - if(dec_ || !out_.empty()) + if(dec_) return out_.consume(n); switch(payload_) diff --git a/src/error.cpp b/src/error.cpp index f42b947..ccad005 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -52,6 +52,8 @@ error_category::message(int ev) const return "unsupported proxy protocol version"; case error::body_size_mismatch: return "request body size did not match content length"; + case error::decode_error: + return "response body could not be decoded"; default: return "unknown error"; } diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 7fcb4e3..ee418b6 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -22,8 +22,10 @@ add_executable(boost_burl_tests ${PFILES}) target_link_libraries(boost_burl_tests PRIVATE boost_capy_test_suite_main Boost::burl - $ - $) + $ + $ + $ + $) target_include_directories(boost_burl_tests PRIVATE . ../../) diff --git a/test/unit/Jamfile b/test/unit/Jamfile index 5bd0957..5b3ee72 100644 --- a/test/unit/Jamfile +++ b/test/unit/Jamfile @@ -8,11 +8,15 @@ # import testing ; +import ac ; + +using brotli ; project : requirements $(c20-requires) /boost/burl//boost_burl + [ ac.check-library /brotli//brotlienc : /brotli//brotlienc : ] ../../../capy/extra/test_suite/test_main.cpp ../../../capy/extra/test_suite/test_suite.cpp . diff --git a/test/unit/client.cpp b/test/unit/client.cpp index ece35c7..a2de04b 100644 --- a/test/unit/client.cpp +++ b/test/unit/client.cpp @@ -17,8 +17,6 @@ #include #include #include -#include -#include #include #include #include @@ -695,15 +693,7 @@ class client_test void testGzipDecode() { -#ifdef BOOST_HTTP_HAS_ZLIB - if(!capy::get_system_context() - .has_service()) - http::zlib::install_inflate_service( - capy::get_system_context()); -#else - return; -#endif - +#ifdef BOOST_BURL_HAS_ZLIB // gzip("hello world"), mtime=0. static char const gz[] = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xcb\x48" @@ -742,20 +732,13 @@ class client_test BOOST_TEST( net.written(0).find("Accept-Encoding: deflate, gzip\r\n") != std::string::npos); +#endif } void testBrotliDecode() { -#ifdef BOOST_HTTP_HAS_BROTLI - if(!capy::get_system_context() - .has_service()) - http::brotli::install_decode_service( - capy::get_system_context()); -#else - return; -#endif - +#ifdef BOOST_BURL_HAS_BROTLI // brotli("hello world"). static char const br[] = "\x0b\x05\x80\x68\x65\x6c\x6c\x6f\x20\x77\x6f\x72" @@ -792,6 +775,50 @@ class client_test BOOST_TEST( net.written(0).find("Accept-Encoding: br\r\n") != std::string::npos); +#endif + } + + void + testZstdDecode() + { +#ifdef BOOST_BURL_HAS_ZSTD + // zstd("hello world"). + static char const zs[] = + "\x28\xb5\x2f\xfd\x20\x0b\x59\x00\x00\x68\x65\x6c" + "\x6c\x6f\x20\x77\x6f\x72\x6c\x64"; + auto const body = std::string(zs, sizeof(zs) - 1); + + scripted_net net; + net.scripts = { std::string( + "HTTP/1.1 200 OK\r\n" + "Content-Encoding: zstd\r\n" + "Content-Length: " + + std::to_string(body.size()) + + "\r\n\r\n" + body) }; + + capy::test::run_blocking()([&]() -> capy::task<> + { + auto cfg = net.config(); + cfg.zstd = true; + client c( + co_await capy::this_coro::executor, + corosio::tls_context(), + cfg); + + auto [ec, r] = co_await c + .get("http://example.com/z") + .send(); + BOOST_TEST(!ec); + + auto [ec2, text] = co_await r.try_as_view(); + BOOST_TEST(!ec2); + BOOST_TEST_EQ(text, "hello world"); + }()); + + BOOST_TEST( + net.written(0).find("Accept-Encoding: zstd\r\n") != + std::string::npos); +#endif } void @@ -1025,6 +1052,7 @@ class client_test testHeadNoBody(); testGzipDecode(); testBrotliDecode(); + testZstdDecode(); testTimeoutHeader(); testTimeoutBody(); testTimeoutOverride(); diff --git a/test/unit/detail/decoders.cpp b/test/unit/detail/decoders.cpp index 0d1ba7a..74015c2 100644 --- a/test/unit/detail/decoders.cpp +++ b/test/unit/detail/decoders.cpp @@ -10,27 +10,29 @@ // Test that header file is self-contained. #include "src/detail/decoders.hpp" +#include + #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include "test_suite.hpp" +#include #include #include #include +#ifdef BOOST_BURL_HAS_ZLIB +#include +#endif + +#ifdef BOOST_BURL_HAS_BROTLI +#include +#endif + +#ifdef BOOST_BURL_HAS_ZSTD +#include +#endif + namespace boost { namespace burl @@ -99,50 +101,49 @@ class decoders_test return body; } +#ifdef BOOST_BURL_HAS_ZLIB static std::string zlib_compress(std::string_view body, int window_bits) { - auto& svc = *capy::get_system_context() - .find_service(); - - http::zlib::stream st = {}; - if(svc.init2( - st, - http::zlib::default_compression, - http::zlib::deflated, + z_stream st = {}; + if(deflateInit2( + &st, + Z_DEFAULT_COMPRESSION, + Z_DEFLATED, window_bits, 8, - http::zlib::default_strategy) != 0) + Z_DEFAULT_STRATEGY) != Z_OK) return {}; - std::string out(svc.bound( - st, static_cast(body.size())), '\0'); + std::string out( + deflateBound( + &st, static_cast(body.size())), + '\0'); st.next_in = reinterpret_cast( const_cast(body.data())); st.avail_in = static_cast(body.size()); st.next_out = reinterpret_cast(out.data()); st.avail_out = static_cast(out.size()); - auto const rs = svc.deflate(st, http::zlib::finish); + auto const rs = deflate(&st, Z_FINISH); out.resize(out.size() - st.avail_out); - svc.deflate_end(st); - if(rs != static_cast(http::zlib::error::stream_end)) + deflateEnd(&st); + if(rs != Z_STREAM_END) return {}; return out; } +#endif +#ifdef BOOST_BURL_HAS_BROTLI static std::string brotli_compress(std::string_view body) { - auto& svc = *capy::get_system_context() - .find_service(); - - std::string out( - svc.max_compressed_size(body.size()) + 64, '\0'); - std::size_t encoded_size = out.size(); - if(!svc.compress( - http::brotli::default_quality, - http::brotli::default_window, - http::brotli::encoder_mode::generic, + std::size_t encoded_size = + BrotliEncoderMaxCompressedSize(body.size()) + 64; + std::string out(encoded_size, '\0'); + if(!BrotliEncoderCompress( + BROTLI_DEFAULT_QUALITY, + BROTLI_DEFAULT_WINDOW, + BROTLI_MODE_GENERIC, body.size(), reinterpret_cast(body.data()), &encoded_size, @@ -151,6 +152,25 @@ class decoders_test out.resize(encoded_size); return out; } +#endif + +#ifdef BOOST_BURL_HAS_ZSTD + static std::string + zstd_compress(std::string_view body) + { + std::string out(ZSTD_compressBound(body.size()), '\0'); + auto const n = ZSTD_compress( + out.data(), + out.size(), + body.data(), + body.size(), + ZSTD_CLEVEL_DEFAULT); + if(ZSTD_isError(n)) + return {}; + out.resize(n); + return out; + } +#endif public: void @@ -160,12 +180,38 @@ class decoders_test make_decoder(http::content_coding::identity) == nullptr); BOOST_TEST( make_decoder(http::content_coding::unknown) == nullptr); - BOOST_TEST( - make_decoder(http::content_coding::zstd) == nullptr); BOOST_TEST( make_decoder(http::content_coding::compress) == nullptr); + + // Availability of each decoder follows the build configuration. +#ifdef BOOST_BURL_HAS_ZLIB + BOOST_TEST( + make_decoder(http::content_coding::deflate) != nullptr); + BOOST_TEST( + make_decoder(http::content_coding::gzip) != nullptr); +#else + BOOST_TEST( + make_decoder(http::content_coding::deflate) == nullptr); + BOOST_TEST( + make_decoder(http::content_coding::gzip) == nullptr); +#endif +#ifdef BOOST_BURL_HAS_BROTLI + BOOST_TEST( + make_decoder(http::content_coding::br) != nullptr); +#else + BOOST_TEST( + make_decoder(http::content_coding::br) == nullptr); +#endif +#ifdef BOOST_BURL_HAS_ZSTD + BOOST_TEST( + make_decoder(http::content_coding::zstd) != nullptr); +#else + BOOST_TEST( + make_decoder(http::content_coding::zstd) == nullptr); +#endif } +#ifdef BOOST_BURL_HAS_ZLIB void test_zlib_round_trip() { @@ -284,9 +330,11 @@ class decoders_test auto const r = run_decoder( *dec, input, input.size(), 64); BOOST_TEST(!r.finished); - BOOST_TEST(r.ec == http::zlib::error::data_err); + BOOST_TEST(r.ec == error::decode_error); } +#endif // BOOST_BURL_HAS_ZLIB +#ifdef BOOST_BURL_HAS_BROTLI void test_brotli_round_trip() { @@ -360,21 +408,85 @@ class decoders_test auto const r = run_decoder( *dec, input, input.size(), 64); BOOST_TEST(!r.finished); - BOOST_TEST(r.ec); - BOOST_TEST(r.ec != http::error::bad_payload); + BOOST_TEST(r.ec == error::decode_error); } +#endif // BOOST_BURL_HAS_BROTLI +#ifdef BOOST_BURL_HAS_ZSTD void - run() + test_zstd_round_trip() { -#if defined(BOOST_HTTP_HAS_ZLIB) + auto const body = make_body(200); + auto const compressed = zstd_compress(body); + + // single pass + { + auto dec = make_decoder(http::content_coding::zstd); + if(!BOOST_TEST(dec != nullptr)) + return; + auto const r = run_decoder( + *dec, compressed, compressed.size(), 1024); + BOOST_TEST(!r.ec); + BOOST_TEST(r.finished); + BOOST_TEST(r.body == body); + BOOST_TEST_EQ(r.leftover, 0u); + } + + // starved input and output { - auto& ctx = capy::get_system_context(); - if(!ctx.has_service()) - http::zlib::install_inflate_service(ctx); - if(!ctx.has_service()) - http::zlib::install_deflate_service(ctx); + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder(*dec, compressed, 3, 7); + BOOST_TEST(!r.ec); + BOOST_TEST(r.finished); + BOOST_TEST(r.body == body); + BOOST_TEST_EQ(r.leftover, 0u); } + } + + void + test_zstd_large() + { + auto const body = make_body(64 * 1024); + auto const compressed = zstd_compress(body); + + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder(*dec, compressed, 1024, 1024); + BOOST_TEST(!r.ec); + BOOST_TEST(r.finished); + BOOST_TEST(r.body == body); + } + + void + test_zstd_empty_body() + { + auto const compressed = zstd_compress({}); + + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder( + *dec, compressed, compressed.size(), 64); + BOOST_TEST(!r.ec); + BOOST_TEST(r.finished); + BOOST_TEST(r.body.empty()); + } + + void + test_zstd_invalid() + { + std::string_view const input = + "this body was never compressed"; + + auto dec = make_decoder(http::content_coding::zstd); + auto const r = run_decoder( + *dec, input, input.size(), 64); + BOOST_TEST(!r.finished); + BOOST_TEST(r.ec == error::decode_error); + } +#endif // BOOST_BURL_HAS_ZSTD + + void + run() + { +#ifdef BOOST_BURL_HAS_ZLIB test_zlib_round_trip(); test_zlib_large(); test_zlib_empty_body(); @@ -382,18 +494,17 @@ class decoders_test test_zlib_trailing_garbage(); test_zlib_invalid(); #endif -#if defined(BOOST_HTTP_HAS_BROTLI) - { - auto& ctx = capy::get_system_context(); - if(!ctx.has_service()) - http::brotli::install_decode_service(ctx); - if(!ctx.has_service()) - http::brotli::install_encode_service(ctx); - } +#ifdef BOOST_BURL_HAS_BROTLI test_brotli_round_trip(); test_brotli_large(); test_brotli_finished_latch(); test_brotli_invalid(); +#endif +#ifdef BOOST_BURL_HAS_ZSTD + test_zstd_round_trip(); + test_zstd_large(); + test_zstd_empty_body(); + test_zstd_invalid(); #endif test_make_decoder(); } diff --git a/test/unit/detail/parser.cpp b/test/unit/detail/parser.cpp index 3a7ca00..5abfa9d 100644 --- a/test/unit/detail/parser.cpp +++ b/test/unit/detail/parser.cpp @@ -1965,12 +1965,12 @@ class parser_test // the raw side ended and the decoder stalls: this // must be an error, not a hang on the socket auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); - BOOST_TEST(ec == http::error::bad_payload); + BOOST_TEST(ec == error::decode_error); BOOST_TEST_EQ(n, 0); } { auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); - BOOST_TEST(ec == http::error::bad_payload); + BOOST_TEST(ec == error::decode_error); BOOST_TEST_EQ(n, 0); } }()); diff --git a/test/unit/scripted_net.hpp b/test/unit/scripted_net.hpp index 3836ce4..398b037 100644 --- a/test/unit/scripted_net.hpp +++ b/test/unit/scripted_net.hpp @@ -51,6 +51,7 @@ struct scripted_net cfg.brotli = false; cfg.deflate = false; cfg.gzip = false; + cfg.zstd = false; cfg.connect_handler = [this](urls::url_view url) -> capy::io_task {