Skip to content

Decode PBF blobs by data field #895

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Symmetricity wants to merge 1 commit into
base: master
Choose a base branch
from
+18 −5
Open

Decode PBF blobs by data field #895

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions src/pbf_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,10 @@ protozero::data_view PbfReader::PbfReader::readBlob(int32_t datasize, std::istre
if (input.eof())
throw std::runtime_error("readBlob: unexpected eof");

enum class BlobDataType { None, Raw, Zlib };

int32_t rawSize = -1;
BlobDataType dataType = BlobDataType::None;
protozero::data_view view;
protozero::pbf_message<Schema::Blob> message{&blobStorage[0], blobStorage.size()};
while (message.next()) {
Expand All @@ -77,23 +80,34 @@ protozero::data_view PbfReader::PbfReader::readBlob(int32_t datasize, std::istre
rawSize = message.get_int32();
break;
case Schema::Blob::oneof_data_bytes_raw:
dataType = BlobDataType::Raw;
view = message.get_view();
break;
case Schema::Blob::oneof_data_bytes_zlib_data:
dataType = BlobDataType::Zlib;
view = message.get_view();
break;
case Schema::Blob::oneof_data_bytes_lzma_data:
case Schema::Blob::oneof_data_bytes_lz4_data:
case Schema::Blob::oneof_data_bytes_zstd_data:
throw std::runtime_error("Blob: unsupported compression tag: " + std::to_string(static_cast<uint32_t>(message.tag())));
default:
throw std::runtime_error("Blob: unknown tag: " + std::to_string(static_cast<uint32_t>(message.tag())));
}
}

if (rawSize == -1)
// Data is not compressed, can return it directly.
if (dataType == BlobDataType::None)
throw std::runtime_error("Blob: missing data");

if (dataType == BlobDataType::Raw)
return view;

blobStorage2.resize(rawSize);
if (rawSize != -1)
blobStorage2.resize(rawSize);
else
blobStorage2.clear();
decompress_string(blobStorage2, view.data(), view.size(), false);
return { &blobStorage2[0], blobStorage2.size() };
return { blobStorage2.data(), blobStorage2.size() };
}

PbfReader::HeaderBBox PbfReader::PbfReader::readHeaderBBox(protozero::data_view data) {
Expand Down Expand Up @@ -587,4 +601,3 @@ PbfReader::HeaderBlock PbfReader::PbfReader::readHeaderFromFile(std::istream& in

return header;
}