diff --git a/include/boost/burl/detail/parser.hpp b/include/boost/burl/detail/parser.hpp index 0cbc395..e1e657e 100644 --- a/include/boost/burl/detail/parser.hpp +++ b/include/boost/burl/detail/parser.hpp @@ -156,6 +156,9 @@ class parser capy::const_buffer, bool last)> f, bool dry = false); + std::error_code + flatten_chunks(); + capy::io_task decode_some( std::span buffers); @@ -195,8 +198,8 @@ class parser bool got_header_ = false; bool got_body_ = false; bool mid_chunk_ = false; + bool fin_chunk_ = false; bool eof_ = false; - bool dec_eof_ = false; }; template diff --git a/src/detail/parser.cpp b/src/detail/parser.cpp index 302aaf3..c7d8fdb 100644 --- a/src/detail/parser.cpp +++ b/src/detail/parser.cpp @@ -39,7 +39,6 @@ using http::condition::need_more_input; using header = http::detail::header; using payload = http::payload; - namespace { @@ -59,6 +58,14 @@ class chained_sequence { } + chained_sequence(char const* p, std::size_t n) + : pos_(p) + , end_(p + n) + , begin_b_(end_) + , end_b_(end_) + { + } + char const* next() noexcept { @@ -98,7 +105,7 @@ class chained_sequence } bool - is_empty() const noexcept + empty() const noexcept { return pos_ == end_; } @@ -109,6 +116,12 @@ class chained_sequence return *pos_; } + char const * + pos() const noexcept + { + return pos_; + } + std::size_t size() const noexcept { @@ -126,6 +139,83 @@ class chained_sequence } }; +std::error_code +parse_chunk_size( + chained_sequence& cs, + std::uint64_t& size) noexcept +{ + size = 0; + for(auto const start = cs.size();;) + { + if(cs.empty()) + return need_data; + auto const n = urls::grammar::hexdig_value(cs.value()); + if(n < 0) + { + if(start == cs.size()) + return bad_payload; + return {}; + } + // at least 4 significant bits are free + if(size > (std::numeric_limits::max)() >> 4) + return bad_payload; + size = (size << 4) | static_cast(n); + cs.next(); + } +} + +std::error_code +skip_to_eol(chained_sequence& cs) noexcept +{ + while(!cs.empty()) + { + if(cs.value() == '\r') + { + if(!cs.next()) + break; + if(cs.value() != '\n') + return bad_payload; + cs.next(); + return {}; + } + cs.next(); + } + return need_data; +} + +std::error_code +skip_trailer(chained_sequence& cs) noexcept +{ + for(;;) + { + if(cs.empty()) + return need_data; + if(cs.value() == '\r') + { + if(!cs.next()) + return need_data; + if(cs.value() != '\n') + return bad_payload; + cs.next(); + return {}; + } + // skip to the end of the field + if(auto ec = skip_to_eol(cs); ec) + return ec; + } +} + +std::error_code +skip_crlf(chained_sequence& cs) noexcept +{ + if(cs.size() < 2) + return need_data; + if(cs.value() != '\r' || *cs.next() != '\n') + return bad_payload; + cs.next(); + return {}; +} + std::span collect( std::span dest, @@ -170,6 +260,14 @@ move_leftovers( } while(an); } +auto +prefix( + auto buf, + std::size_t n) noexcept -> decltype(buf) +{ + return { buf.data(), clamp(buf.size(), n) }; +}; + } // namespace parser:: @@ -216,9 +314,10 @@ bool parser:: has_buffered_data() const noexcept { - BOOST_ASSERT(got_body_); switch(payload_) { + case payload::chunked: + return in_.size() > chunk_rem_; case payload::size: return in_.size() > payload_rem(); case payload::to_eof: @@ -276,7 +375,6 @@ parser:: start(bool head) { BOOST_ASSERT(!started_ || got_body_); - BOOST_ASSERT(!dec_err_); if(payload_sized() && got_body_) in_.consume(payload_rem()); @@ -298,13 +396,14 @@ start(bool head) chunk_rem_ = 0; transferred_ = 0; decoded_ = 0; + dec_err_ = {}; payload_ = payload::none; head_ = head; started_ = true; got_header_ = false; got_body_ = false; mid_chunk_ = false; - dec_eof_ = false; + fin_chunk_ = false; } void @@ -330,8 +429,8 @@ reset(capy::any_read_stream stream) noexcept got_header_ = false; got_body_ = false; mid_chunk_ = false; + fin_chunk_ = false; eof_ = false; - dec_eof_ = false; } capy::io_task<> @@ -366,123 +465,145 @@ walk_chunks( capy::const_buffer, bool)> f, bool dry) { - if(got_body_ && chunk_rem_ == 0) - return f({}, true).ec; - chained_sequence cs = in_.data(); - std::uint64_t len = chunk_rem_; + std::uint64_t size = chunk_rem_; - auto skip_to_eol = [&]() -> std::error_code + if(fin_chunk_) { - while(!cs.is_empty()) + // from flatten_chunks + auto const b = prefix( + in_.data()[0], clamp(chunk_rem_)); + auto const [ec, n] = f(b, true); + if(!dry) { - if(cs.value() == '\r') - { - if(!cs.next()) - break; - if(cs.value() != '\n') - return bad_payload; - cs.next(); - return {}; - } - cs.next(); + in_.consume(n); + chunk_rem_ -= n; + transferred_ += n; } - return need_data; - }; + return ec; + } - if(len != 0 || mid_chunk_) + if(mid_chunk_) goto invoke; loop: - // chunk header - for(auto hdr_start = cs.size();;) - { - if(cs.is_empty()) - return need_data; - auto n = urls::grammar::hexdig_value(cs.value()); - if(n < 0) - { - if(hdr_start == cs.size()) - return bad_payload; - break; - } - // at least 4 significant bits are free - if(len > (std::numeric_limits::max)() >> 4) - return bad_payload; - len = (len << 4) | static_cast(n); - cs.next(); - } + if(auto ec = parse_chunk_size(cs, size); ec) + return ec; - // skip chunk exts - if(auto ec = skip_to_eol(); ec) + if(auto ec = skip_to_eol(cs); ec) return ec; // final chunk - if(len == 0) + if(size == 0) { - // skip trailer headers - for(;;) - { - if(cs.is_empty()) - return need_data; - if(cs.value() == '\r') - { - if(!cs.next()) - return need_data; - if(cs.value() != '\n') - return bad_payload; - cs.next(); - break; - } - // skip to the end of the field - if(auto ec = skip_to_eol(); ec) - return ec; - } + if(auto ec = skip_trailer(cs); ec) + return ec; + got_body_ = true; if(!dry) { + fin_chunk_ = true; in_.consume(in_.size() - cs.size()); - got_body_ = true; } return f({}, true).ec; } invoke: - for(const auto& b : cs.prefix(clamp(len))) + for(const auto& b : cs.prefix(clamp(size))) { if(b.size() == 0) break; auto const [ec, n] = f(b, false); cs.advance(n); - len -= n; + size -= n; if(!dry) { in_.consume(in_.size() - cs.size()); - mid_chunk_ = true; - chunk_rem_ = len; + chunk_rem_ = size; + transferred_ += n; + mid_chunk_ = true; } if(ec || n < b.size()) return ec; } - // CRLF - if(cs.size() < 2) - return need_data; - if(cs.value() != '\r' || *cs.next() != '\n') - return bad_payload; - cs.next(); + if(auto ec = skip_crlf(cs); ec) + return ec; goto loop; } +std::error_code +parser:: +flatten_chunks() +{ + if(fin_chunk_) + return {}; + + BOOST_ASSERT(in_.pos == 0); + + std::size_t flat = clamp(chunk_rem_, in_.len); + char const* keep = in_.ptr + flat; + chained_sequence cs(keep, in_.len - flat); + + auto bail = [&](std::error_code ec) + { + auto const tail = static_cast( + in_.ptr + in_.len - keep); + std::memmove(in_.ptr + flat, keep, tail); + in_.len = flat + tail; + return ec; + }; + + for(;;) + { + if(mid_chunk_) + { + if(auto ec = skip_crlf(cs); ec) + return bail(ec); + } + + std::uint64_t size; + if(auto ec = parse_chunk_size(cs, size); ec) + return bail(ec); + + if(auto ec = skip_to_eol(cs); ec) + return bail(ec); + + if(size == 0) + { + if(auto ec = skip_trailer(cs); ec) + return bail(ec); + got_body_ = true; + fin_chunk_ = true; + keep = cs.pos(); + return bail({}); + } + + if(size > in_.cap - flat) + return bail(in_place_overflow); + + chunk_rem_ = flat + size; + mid_chunk_ = true; + + auto const n = clamp(size, cs.size()); + std::memmove(in_.ptr + flat, cs.pos(), n); + flat += n; + cs.advance(n); + keep = cs.pos(); + if(n < size) + return bail(need_data); + } +} + capy::io_task<> parser:: read_header() { + BOOST_ASSERT(started_); + if(got_header_) co_return {}; - BOOST_ASSERT(started_); - for(;;) { system::error_code ec; @@ -532,6 +653,7 @@ void parser:: set_decoder(decoder* dec) noexcept { + BOOST_ASSERT(transferred_ == 0); dec_ = dec; } @@ -568,6 +690,9 @@ read_body() } } + if(transferred_ != 0) + co_return { incomplete, {} }; + switch(payload_) { case payload::error: @@ -577,26 +702,23 @@ read_body() } case payload::chunked: { - if(transferred_ != out_.size()) - co_return { incomplete, {} }; for(;;) { - if(out_.full()) - co_return { in_place_overflow, {} }; - auto [ec, n] = co_await do_read_some(out_.prepare(), true); - out_.commit(n); - if(ec) + if(chunk_rem_ > raw_limit_rem()) + co_return { body_too_large, {} }; + if(fin_chunk_) + co_return { {}, { in_.ptr, clamp(chunk_rem_) } }; + if(auto ec = flatten_chunks(); ec) { - if(ec == capy::cond::eof) - co_return { {}, { out_.ptr, out_.len } }; - co_return { ec, {} }; + if(ec != need_more_input) + co_return { ec, {} }; + if(auto [fec] = co_await refill(); fec) + co_return { fec, {} }; } } } case payload::size: { - if(transferred_ != 0) - co_return { incomplete, {} }; auto const rem = payload_rem(); if(rem > raw_limit_rem()) co_return { body_too_large, {} }; @@ -604,22 +726,20 @@ read_body() { if(got_body_) co_return { {}, { in_.ptr, clamp(in_.len, rem) } }; - if(auto [ec] = co_await refill(); ec) - co_return { ec, {} }; + if(auto [fec] = co_await refill(); fec) + co_return { fec, {} }; } } case payload::to_eof: { - if(transferred_ != 0) - co_return { incomplete, {} }; for(;;) { if(in_.size() > raw_limit_rem()) co_return { body_too_large, {} }; if(got_body_) co_return { {}, { in_.ptr, in_.len } }; - if(auto [ec] = co_await refill(); ec) - co_return { ec, {} }; + if(auto [fec] = co_await refill(); fec) + co_return { fec, {} }; } } } @@ -629,20 +749,16 @@ http::static_response const& parser:: get_response() const { - if(!got_header_) - http::detail::throw_logic_error(); - - return reinterpret_cast(*h_); + return reinterpret_cast< + http::static_response const&>(*h_); } http::static_request const& parser:: get_request() const { - if(!got_header_) - http::detail::throw_logic_error(); - - return reinterpret_cast(*h_); + return reinterpret_cast< + http::static_request const&>(*h_); } capy::io_task @@ -650,59 +766,50 @@ parser:: decode_some( std::span buffers) { - if(dec_err_) - co_return { dec_err_, 0 }; + if(capy::buffer_empty(buffers)) + co_return { {}, 0 }; - std::size_t lim = dec_limit_rem(); auto slice = capy::buffer_slice(buffers); - - auto const prefix = [](auto buf, std::size_t n) - -> decltype(buf) - { - return { buf.data(), clamp(buf.size(), n) }; - }; - - std::size_t prod = 0; - auto const pump = [&](capy::const_buffer in) + auto prod = std::size_t(0); + auto pump = [&](capy::const_buffer in, bool last) -> capy::io_result { - if(dec_eof_) + if(dec_err_) { - if(!got_body_ || in.size() != 0) - return { bad_payload, 0 }; - return { capy::error::eof, 0 }; + if(dec_err_ == capy::cond::eof) + { + if(!last || in.size() != 0) + return { bad_payload, 0 }; + } + return { dec_err_, 0 }; } std::size_t cons = 0; for(;;) { - auto const out = prefix(capy::front(slice.data()), lim); + auto const lim = dec_limit_rem(); + auto const out = capy::front(slice.data()); if(out.size() == 0) return { {}, cons }; if(lim == 0) return { body_too_large, cons }; - auto const res = dec_->process(out, in, got_body_); - in += res.consumed; - cons += res.consumed; - transferred_ += res.consumed; - prod += res.produced; - decoded_ += res.produced; - lim -= res.produced; - slice.remove_prefix(res.produced); - if(res.ec) + auto const r = dec_->process( + prefix(out, lim), in, last); + in += r.consumed; + cons += r.consumed; + transferred_ += r.consumed; + prod += r.produced; + decoded_ += r.produced; + slice.remove_prefix(r.produced); + if(r.ec) { - if(res.ec == capy::cond::eof) - { - dec_eof_ = true; - return { {}, cons }; - } - dec_err_ = res.ec; + dec_err_ = r.ec; return { {}, cons }; } - if(res.produced == 0 && res.consumed == 0) + if(r.produced == 0 && r.consumed == 0) { // TODO: dedicated error code dec_err_ = bad_payload; - return { dec_err_, cons }; + return { {}, cons }; } if(in.size() == 0) return { {}, cons }; @@ -720,22 +827,13 @@ decode_some( { for(;;) { - auto ec = walk_chunks( - [&](capy::const_buffer in, bool last) - -> capy::io_result - { - return pump(in); - }); + auto ec = walk_chunks(pump); if(prod != 0) co_return { {}, prod }; - if(ec == need_more_input) - { - if(auto [fec] = co_await refill(); fec) - co_return { fec, 0 }; - continue; - } - if(ec) + if(ec != need_more_input) co_return { ec, 0 }; + if(auto [fec] = co_await refill(); fec) + co_return { fec, 0 }; } } case payload::size: @@ -743,19 +841,15 @@ decode_some( { for(;;) { - auto const in = prefix( - in_.data()[0], - payload_sized() ? payload_rem() : std::size_t(-1)); - if(in.size() == 0) + auto const rem = payload_sized() ? payload_rem() : in_.size(); + auto const in = prefix(in_.data()[0], rem); + if(in.size() == 0 && !got_body_) { - if(!got_body_) - { - if(auto [ec] = co_await refill(); ec) - co_return { ec, 0 }; - continue; - } + if(auto [fec] = co_await refill(); fec) + co_return { fec, 0 }; + continue; } - auto [ec, cons] = pump(in); + auto [ec, cons] = pump(in, got_body_ && in.size() == rem); in_.consume(cons); if(prod != 0) co_return { {}, prod }; @@ -785,7 +879,7 @@ do_read_some( if(dec_) co_return co_await decode_some(buffers); - auto copy_form_in = [&](std::size_t at_most) + auto copy = [&](std::size_t at_most) { auto const n = capy::buffer_copy( buffers, in_.data(), at_most); @@ -817,7 +911,6 @@ do_read_some( auto const n = capy::buffer_copy( slice.data(), b, take); read += n; - transferred_ += n; slice.remove_prefix(n); if(take < b.size()) return { body_too_large, n }; @@ -832,7 +925,7 @@ do_read_some( co_return { fec, 0 }; continue; } - if(ec) + else if(ec) co_return { ec, 0 }; BOOST_ASSERT(got_body_); co_return { capy::error::eof, 0 }; @@ -841,13 +934,13 @@ do_read_some( case payload::size: { auto const rem = payload_rem(); - auto const lim = raw_limit_rem(); if(rem == 0) co_return { capy::error::eof, 0 }; + auto const lim = raw_limit_rem(); if(lim == 0) co_return { body_too_large, 0 }; if(!in_.empty()) - co_return { {}, copy_form_in(clamp(rem, lim)) }; + co_return { {}, copy(clamp(rem, lim)) }; if(eof_) co_return { incomplete, 0 }; auto [ec, n] = co_await stream_.read_some( @@ -865,13 +958,13 @@ do_read_some( } case payload::to_eof: { + if(eof_) + co_return { capy::error::eof, 0 }; auto const lim = raw_limit_rem(); if(lim == 0) co_return { body_too_large, 0 }; if(!in_.empty()) - co_return { {}, copy_form_in(lim) }; - if(eof_) - co_return { capy::error::eof, 0 }; + co_return { {}, copy(lim) }; auto [ec, n] = co_await stream_.read_some( capy::buffer_slice(buffers, 0, lim).data()); transferred_ += n; @@ -923,7 +1016,7 @@ pull(std::span dest) [&](capy::const_buffer b, bool last) -> capy::io_result { - if(last) + if(last && b.size() == 0) return { capy::error::eof, 0 }; auto const take = clamp(b.size(), lim); if(take == 0 || n == dest.size()) @@ -935,17 +1028,14 @@ pull(std::span dest) true); if(n != 0) co_return { {}, dest.first(n) }; - if(ec == need_more_input) - { - if(auto [fec] = co_await refill(); fec) - co_return { fec, {} }; - } - else + if(ec != need_more_input) { if(ec == capy::error::eof) consume(0); // chunk trailer co_return { ec, {} }; } + if(auto [fec] = co_await refill(); fec) + co_return { fec, {} }; } } case payload::size: @@ -961,8 +1051,8 @@ pull(std::span dest) if(!in_.empty()) co_return { {}, collect( dest, in_.data(), clamp(rem, lim)) }; - if(auto [ec] = co_await refill(); ec) - co_return { ec, {} }; + if(auto [fec] = co_await refill(); fec) + co_return { fec, {} }; } } case payload::to_eof: @@ -976,8 +1066,8 @@ pull(std::span dest) co_return { {}, collect(dest, in_.data(), lim) }; if(eof_) co_return { capy::error::eof, {} }; - if(auto [ec] = co_await refill(); ec) - co_return { ec, {} }; + if(auto [fec] = co_await refill(); fec) + co_return { fec, {} }; } } } @@ -992,32 +1082,21 @@ consume(std::size_t n) noexcept switch(payload_) { - case payload::error: - case payload::none: - { - return; - } case payload::chunked: - { walk_chunks( [&](capy::const_buffer b, bool) -> capy::io_result { - auto const k = n < b.size() ? n : b.size(); - n -= k; - transferred_ += k; - return { {}, k }; + auto const take = clamp(b.size(), n); + n -= take; + return { {}, take }; }); return; - } - case payload::size: - case payload::to_eof: - { + default: in_.consume(n); transferred_ += n; return; } - } } } // namespace detail diff --git a/test/unit/detail/parser.cpp b/test/unit/detail/parser.cpp index 17ee29a..3a7ca00 100644 --- a/test/unit/detail/parser.cpp +++ b/test/unit/detail/parser.cpp @@ -955,6 +955,96 @@ class parser_test }()); } + void + testChunkedReadBodyByteByByte() + { + // one octet per read exercises every resume point of the + // in-place flattening walk: split chunk-size line, split + // extension, split chunk data, split closing CRLF, and + // split trailers + capy::test::read_stream server({}, 1); + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "5;ext=1\r\nhello\r\n" + "6\r\n world\r\n" + "0\r\n" + "X-Trailer: v\r\n" + "\r\n"); + + pr.start(); + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(!ec); + BOOST_TEST(body == "hello world"); + BOOST_TEST(pr.got_body()); + BOOST_TEST(!pr.has_buffered_data()); + }()); + } + + void + testChunkedReadBodySplitMidChunk() + { + // a chunk larger than the transport's read size needs several + // refills; the missing bytes must land in position without + // corrupting the already-flattened prefix + capy::test::read_stream server({}, 4); + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "1a\r\nabcdefghijklmnopqrstuvwxyz\r\n" + "0\r\n\r\n"); + + pr.start(); + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(!ec); + BOOST_TEST(body == "abcdefghijklmnopqrstuvwxyz"); + BOOST_TEST(pr.got_body()); + BOOST_TEST(!pr.has_buffered_data()); + }()); + } + + void + testChunkedReadBodyPipelined() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "5\r\nhello\r\n" + "6\r\n world\r\n" + "0\r\n\r\n" + "NEXT"); + + pr.start(); + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(!ec); + BOOST_TEST(body == "hello world"); + BOOST_TEST(pr.got_body()); + + // the pipelined octets survive the compaction and sit + // right past the body view + BOOST_TEST(pr.has_buffered_data()); + }()); + } + void testChunkedTrailersAndExtensions() { @@ -1278,19 +1368,23 @@ class parser_test void testChunkedReadBodyOverflowThenStream() { + constexpr std::string_view hdr = + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n"; + // dec_buffer smaller than the dechunked body parser::config cfg; - cfg.dec_buffer = 8; + cfg.hdr_limits.max_size = hdr.size(); + cfg.in_buffer = 5; capy::test::read_stream server; capy::any_read_stream stream(&server); test_parser pr(cfg, &stream); capy::test::run_blocking()([&]() -> capy::task<> { + server.provide(hdr); server.provide( - "HTTP/1.1 200 OK\r\n" - "Transfer-Encoding: chunked\r\n" - "\r\n" "5\r\nhello\r\n" "6\r\n world\r\n" "0\r\n\r\n"); @@ -2648,6 +2742,9 @@ class parser_test testChunkedReadSome(); testChunkedPullConsume(); testChunkedReadBody(); + testChunkedReadBodyByteByByte(); + testChunkedReadBodySplitMidChunk(); + testChunkedReadBodyPipelined(); testChunkedTrailersAndExtensions(); testChunkedBadFraming(); testChunkedBadFramingWithData();