From 3f2550c6a9b599cc24de9377a6edef45d8d9f1d8 Mon Sep 17 00:00:00 2001 From: xulei25 Date: Mon, 21 Sep 2026 00:54:35 +0800 Subject: [PATCH 1/3] Add FlatBuffers message construction support Add IOBuf-backed messages, allocator-aware builders and stable service method descriptors. Provide a standalone binding generator using the upstream FlatBuffers parser rather than requiring a compiler fork. Keep the feature disabled by default in CMake, Make and Bazel. Cover message ownership, framing, schema verification, method IDs and generated name collisions with regression tests. Build on the SingleIOBuf foundation merged in apache/brpc#3062 and the message-construction work in apache/brpc#3196. Network transport and Channel/Server integration remain separate. --- BUILD.bazel | 27 +- CMakeLists.txt | 16 + MODULE.bazel | 14 + Makefile | 3 + WORKSPACE | 8 + config.h.in | 5 + config_brpc.sh | 19 +- docs/en/flatbuffers.md | 123 +++++ src/brpc/flatbuffers/message.cpp | 304 +++++++++++++ src/brpc/flatbuffers/message.h | 166 +++++++ src/brpc/flatbuffers/service.cpp | 158 +++++++ src/brpc/flatbuffers/service.h | 127 ++++++ test/BUILD.bazel | 42 +- test/CMakeLists.txt | 27 ++ test/Makefile | 22 +- test/brpc_flatbuffers_unittest.cpp | 528 ++++++++++++++++++++++ test/flatbuffers_codegen/CMakeLists.txt | 78 ++++ test/flatbuffers_codegen/acceptance.cmake | 269 +++++++++++ test/flatbuffers_codegen/echo.fbs | 37 ++ test/flatbuffers_codegen/runtime.cpp | 308 +++++++++++++ test/flatbuffers_message.fbs | 26 ++ tools/flatbuffers/CMakeLists.txt | 39 ++ tools/flatbuffers/README.md | 134 ++++++ tools/flatbuffers/brpc_flatc.cpp | 486 ++++++++++++++++++++ 24 files changed, 2959 insertions(+), 7 deletions(-) create mode 100644 docs/en/flatbuffers.md create mode 100644 src/brpc/flatbuffers/message.cpp create mode 100644 src/brpc/flatbuffers/message.h create mode 100644 src/brpc/flatbuffers/service.cpp create mode 100644 src/brpc/flatbuffers/service.h create mode 100644 test/brpc_flatbuffers_unittest.cpp create mode 100644 test/flatbuffers_codegen/CMakeLists.txt create mode 100644 test/flatbuffers_codegen/acceptance.cmake create mode 100644 test/flatbuffers_codegen/echo.fbs create mode 100644 test/flatbuffers_codegen/runtime.cpp create mode 100644 test/flatbuffers_message.fbs create mode 100644 tools/flatbuffers/CMakeLists.txt create mode 100644 tools/flatbuffers/README.md create mode 100644 tools/flatbuffers/brpc_flatc.cpp diff --git a/BUILD.bazel b/BUILD.bazel index db73605e6e..31c75281b6 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -20,6 +20,12 @@ licenses(["notice"]) # Apache v2 exports_files(["LICENSE"]) +config_setting( + name = "brpc_with_flatbuffers", + define_values = {"BRPC_WITH_FLATBUFFERS": "true"}, + visibility = ["//visibility:public"], +) + COPTS = [ "-fno-omit-frame-pointer", ] + select({ @@ -45,6 +51,9 @@ DEFINES = [ }) + select({ "//bazel/config:brpc_with_thrift": ["ENABLE_THRIFT_FRAMED_PROTOCOL=1"], "//conditions:default": [], + }) + select({ + ":brpc_with_flatbuffers": ["BRPC_WITH_FLATBUFFERS=1"], + "//conditions:default": ["BRPC_WITH_FLATBUFFERS=0"], }) + select({ "//bazel/config:brpc_with_thrift_legacy_version": [], "//conditions:default": ["THRIFT_STDCXX=std"], @@ -125,6 +134,14 @@ genrule( "//conditions:default": "0", }) + """ +#ifdef BRPC_WITH_FLATBUFFERS +#undef BRPC_WITH_FLATBUFFERS +#endif +#define BRPC_WITH_FLATBUFFERS """ + select({ + ":brpc_with_flatbuffers": "1", + "//conditions:default": "0", + }) + + """ #ifdef BUTIL_USE_CPU_FREQUENCY #undef BUTIL_USE_CPU_FREQUENCY #endif @@ -539,6 +556,8 @@ brpc_proto_library( visibility = ["//visibility:public"], ) +FLATBUFFERS_SRC_PATTERNS = ["src/brpc/flatbuffers/*.cpp"] + URMA_SRC_PATTERNS = [ "src/brpc/urma/*.cpp", "src/brpc/urma/**/*.cpp", @@ -562,7 +581,7 @@ BRPC_BASE_SRCS = glob( "src/brpc/policy/thrift_protocol.cpp", "src/brpc/event_dispatcher_epoll.cpp", "src/brpc/event_dispatcher_kqueue.cpp", - ] + URMA_SRC_PATTERNS, + ] + URMA_SRC_PATTERNS + FLATBUFFERS_SRC_PATTERNS, ) cc_library( @@ -573,6 +592,9 @@ cc_library( "src/brpc/**/thrift*.cpp", ]), "//conditions:default": [], + }) + select({ + ":brpc_with_flatbuffers": glob(FLATBUFFERS_SRC_PATTERNS), + "//conditions:default": [], }) + select({ "//bazel/config:brpc_with_urma_use_real": URMA_SRCS, "//bazel/config:brpc_with_urma": URMA_SRCS + [ @@ -612,6 +634,9 @@ cc_library( "@org_apache_thrift//:thrift", ], "//conditions:default": [], + }) + select({ + ":brpc_with_flatbuffers": ["@com_github_google_flatbuffers//:runtime_cc"], + "//conditions:default": [], }), ) diff --git a/CMakeLists.txt b/CMakeLists.txt index f9e1aa8fc3..c457231124 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ option(WITH_MESALINK "With MesaLink" OFF) option(WITH_BORINGSSL "With BoringSSL" OFF) option(WITH_DEBUG_SYMBOLS "With debug symbols" ON) option(WITH_THRIFT "With thrift framed protocol supported" OFF) +option(WITH_FLATBUFFERS "With FlatBuffers message support (headers only)" OFF) option(WITH_BTHREAD_TRACER "With bthread tracer supported" OFF) option(WITH_SNAPPY "With snappy" OFF) option(WITH_RDMA "With RDMA" OFF) @@ -82,6 +83,17 @@ if(WITH_GLOG) set(BRPC_WITH_GLOG 1) endif() +set(WITH_FLATBUFFERS_VAL "0") +if(WITH_FLATBUFFERS) + find_path(FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h) + if(NOT FLATBUFFERS_INCLUDE_DIR) + message(FATAL_ERROR + "WITH_FLATBUFFERS requires FlatBuffers headers; set FLATBUFFERS_INCLUDE_DIR.") + endif() + set(WITH_FLATBUFFERS_VAL "1") + list(APPEND BRPC_COMMON_INCLUDE_DIRS ${FLATBUFFERS_INCLUDE_DIR}) +endif() + set(WITH_CPU_FREQUENCY_VAL "0") if(WITH_CPU_FREQUENCY) set(WITH_CPU_FREQUENCY_VAL "1") @@ -177,6 +189,7 @@ endif() list(APPEND BRPC_COMMON_DEFINITIONS BRPC_WITH_GLOG=${WITH_GLOG_VAL} + BRPC_WITH_FLATBUFFERS=${WITH_FLATBUFFERS_VAL} BRPC_WITH_RDMA=${WITH_RDMA_VAL} BRPC_WITH_URMA=${WITH_URMA_VAL} BRPC_WITH_UBRING=${WITH_UBRING_VAL} @@ -670,6 +683,9 @@ file(GLOB_RECURSE BTHREAD_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/b file(GLOB_RECURSE JSON2PB_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/json2pb/*.cpp") file(GLOB_RECURSE BRPC_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/brpc/*.cpp") file(GLOB_RECURSE THRIFT_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/brpc/thrift*.cpp") +if(NOT WITH_FLATBUFFERS) + list(FILTER BRPC_SOURCES EXCLUDE REGEX "/brpc/flatbuffers/.*\\.cpp$") +endif() file(GLOB_RECURSE EXCLUDE_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/brpc/event_dispatcher_*.cpp") # When building with the real liburma, exclude the link-time mock so its urma_* diff --git a/MODULE.bazel b/MODULE.bazel index 97862d36f1..4f72f9b07b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -79,3 +79,17 @@ git_repository( remote = 'https://atomgit.com/openeuler/umdk.git', commit = '564ee727a55523d4351a8fb3c94292b388ebb924', # v26.06.0_CAM ) + +# runtime_cc and flatc do not need FlatBuffers' gRPC module dependency, which +# would otherwise conflict with brpc's BoringSSL version even when disabled. +# Keep the archive and checksum in sync with WORKSPACE. +flatbuffers_http_archive = use_repo_rule( + '@bazel_tools//tools/build_defs/repo:http.bzl', + 'http_archive', +) +flatbuffers_http_archive( + name = 'com_github_google_flatbuffers', + sha256 = 'b9c2df49707c57a48fc0923d52b8c73beb72d675f9d44b2211e4569be40a7421', + strip_prefix = 'flatbuffers-25.2.10', + urls = ['https://github.com/google/flatbuffers/archive/refs/tags/v25.2.10.tar.gz'], +) diff --git a/Makefile b/Makefile index 271b518ae6..08df1f04e9 100644 --- a/Makefile +++ b/Makefile @@ -204,6 +204,9 @@ JSON2PB_SOURCES = $(foreach d,$(JSON2PB_DIRS),$(wildcard $(addprefix $(d)/*,$(SR JSON2PB_OBJS = $(addsuffix .o, $(basename $(JSON2PB_SOURCES))) BRPC_DIRS = src/brpc src/brpc/details src/brpc/builtin src/brpc/policy src/brpc/policy/mysql src/brpc/rdma +ifeq ($(WITH_FLATBUFFERS),1) +BRPC_DIRS += src/brpc/flatbuffers +endif ifeq ($(WITH_URMA),1) BRPC_DIRS += src/brpc/urma endif diff --git a/WORKSPACE b/WORKSPACE index 22fc411b32..9778a704ba 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -150,6 +150,14 @@ http_archive( urls = ["https://github.com/google/crc32c/archive/1.1.2.tar.gz"], ) +# Optional FlatBuffers support uses runtime_cc; keep this version in sync with MODULE.bazel. +http_archive( + name = "com_github_google_flatbuffers", + integrity = "sha256-ucLfSXB8V6SPwJI9UrjHO+ty1nX51EsiEeRWm+QKdCE=", + strip_prefix = "flatbuffers-25.2.10", + urls = ["https://github.com/google/flatbuffers/archive/refs/tags/v25.2.10.tar.gz"], +) + http_archive( name = "com_github_google_glog", # 2021-05-07T23:06:39Z patch_args = ["-p1"], diff --git a/config.h.in b/config.h.in index d8de111be9..ece54504ce 100644 --- a/config.h.in +++ b/config.h.in @@ -21,6 +21,11 @@ #endif #cmakedefine BRPC_WITH_GLOG @WITH_GLOG_VAL@ +#ifdef BRPC_WITH_FLATBUFFERS +#undef BRPC_WITH_FLATBUFFERS +#endif +#define BRPC_WITH_FLATBUFFERS @WITH_FLATBUFFERS_VAL@ + #ifdef BUTIL_USE_CPU_FREQUENCY #undef BUTIL_USE_CPU_FREQUENCY #endif diff --git a/config_brpc.sh b/config_brpc.sh index 0efa4d374d..e940e486f4 100755 --- a/config_brpc.sh +++ b/config_brpc.sh @@ -54,9 +54,10 @@ else LDD=ldd fi -TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-rdma,with-urma,with-urma-mock,without-urma-mock,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"` +TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-flatbuffers,with-rdma,with-urma,with-urma-mock,without-urma-mock,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"` WITH_GLOG=0 WITH_THRIFT=0 +WITH_FLATBUFFERS=0 WITH_RDMA=0 WITH_URMA=0 URMA_MOCK_MODE=auto @@ -91,6 +92,7 @@ while true; do --cxx ) CXX=$2; shift 2 ;; --with-glog ) WITH_GLOG=1; shift 1 ;; --with-thrift) WITH_THRIFT=1; shift 1 ;; + --with-flatbuffers) WITH_FLATBUFFERS=1; shift 1 ;; --with-rdma) WITH_RDMA=1; shift 1 ;; --with-urma) WITH_URMA=1; shift 1 ;; --with-urma-mock) URMA_MOCK_MODE=on; shift 1 ;; @@ -479,6 +481,7 @@ append_to_output "HDRS=$($ECHO $HDRS)" append_to_output "LIBS=$($ECHO $LIBS)" append_to_output "PROTOC=$PROTOC" append_to_output "PROTOBUF_HDR=$PROTOBUF_HDR" +append_to_output "WITH_FLATBUFFERS=$WITH_FLATBUFFERS" append_to_output "CC=$CC" append_to_output "CXX=$CXX" append_to_output "GCC_VERSION=$GCC_VERSION" @@ -486,7 +489,7 @@ append_to_output "STATIC_LINKINGS=$STATIC_LINKINGS" append_to_output "DYNAMIC_LINKINGS=$DYNAMIC_LINKINGS" # CPP means C PreProcessing, not C PlusPlus -CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK -DBUTIL_USE_CPU_FREQUENCY=$WITH_CPU_FREQUENCY" +CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_WITH_FLATBUFFERS=$WITH_FLATBUFFERS -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK -DBUTIL_USE_CPU_FREQUENCY=$WITH_CPU_FREQUENCY" # Avoid over-optimizations of TLS variables by GCC>=4.8 # See: https://github.com/apache/brpc/issues/1693 @@ -506,6 +509,12 @@ if [ "$SYSTEM" = "Darwin" ]; then fi fi +if [ $WITH_FLATBUFFERS != 0 ]; then + FLATBUFFERS_HDR=$(find_dir_of_header_or_die flatbuffers/flatbuffers.h) || exit 1 + append_to_output_headers "$FLATBUFFERS_HDR" + print_success "Found FlatBuffers headers: $FLATBUFFERS_HDR" +fi + if [ $WITH_THRIFT != 0 ]; then THRIFT_LIB=$(find_dir_of_lib_or_die thriftnb) THRIFT_HDR=$(find_dir_of_header_or_die thrift/Thrift.h) @@ -692,6 +701,11 @@ cat << EOF > src/butil/config.h #endif #define BRPC_WITH_GLOG $WITH_GLOG +#ifdef BRPC_WITH_FLATBUFFERS +#undef BRPC_WITH_FLATBUFFERS +#endif +#define BRPC_WITH_FLATBUFFERS $WITH_FLATBUFFERS + #ifdef BUTIL_USE_CPU_FREQUENCY #undef BUTIL_USE_CPU_FREQUENCY #endif @@ -714,6 +728,7 @@ print_info "C++ std: $CXXFLAGS" print_info "System: $SYSTEM" if [ $WITH_GLOG -ne 0 ]; then print_info "With glog: yes"; fi if [ $WITH_THRIFT -ne 0 ]; then print_info "With thrift: yes"; fi +if [ $WITH_FLATBUFFERS -ne 0 ]; then print_info "With FlatBuffers: yes (headers only)"; fi if [ $WITH_RDMA -ne 0 ]; then print_info "With RDMA: yes"; fi if [ $WITH_URMA -ne 0 ]; then print_info "With URMA: yes"; fi if [ $WITH_MESALINK -ne 0 ]; then print_info "With MesaLink: yes"; fi diff --git a/docs/en/flatbuffers.md b/docs/en/flatbuffers.md new file mode 100644 index 0000000000..7ff86accbe --- /dev/null +++ b/docs/en/flatbuffers.md @@ -0,0 +1,123 @@ +# FlatBuffers messages + +bRPC provides optional IOBuf-backed FlatBuffers messages, builders, and service +descriptors. The message-construction approach builds on +[apache/brpc#3196](https://github.com/apache/brpc/pull/3196). + +This component does not register an `fb_rpc` transport or add FlatBuffers +integration to `brpc::Channel` and `brpc::Server`. Service-generation and +in-process dispatch tests are not network RPC or performance benchmarks. + +## Build + +FlatBuffers support is disabled by default. The message runtime needs only +FlatBuffers headers; it does not link a FlatBuffers library. Tests need a `flatc` +matching those headers. Keep upstream's generated version assertions intact: +regenerate the header rather than weakening the assertion. + +For example, with GoogleTest sources installed under `/usr/src/googletest`: + +```sh +cmake -S . -B build -DWITH_FLATBUFFERS=ON -DBUILD_UNIT_TESTS=ON \ + -DBUILD_BRPC_TOOLS=OFF -DDOWNLOAD_GTEST=OFF \ + -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest +cmake --build build --target brpc_flatbuffers_unittest -j6 +ctest --test-dir build -R '^brpc_flatbuffers_unittest$' --output-on-failure +``` + +For other installations set `FLATBUFFERS_INCLUDE_DIR`, +`FLATBUFFERS_FLATC_EXECUTABLE`, and `BRPC_SYSTEM_GTEST_SOURCE_DIR` as needed. +The project's usual test dependencies still apply. + +Make accepts `--with-flatbuffers` on `config_brpc.sh`; its tests accept +`FLATC=/path/to/flatc`. Bazel accepts `--define=BRPC_WITH_FLATBUFFERS=true`, +with matching FlatBuffers 25.2.10 runtime and compiler dependencies. Bzlmod +imports the same checksum-pinned archive as WORKSPACE: `runtime_cc` and `flatc` +do not need FlatBuffers' external gRPC module, which would otherwise conflict +with bRPC's pinned BoringSSL even when the feature is disabled. + +Public headers live in `brpc/flatbuffers/`, matching namespace +`brpc::flatbuffers`. Include `message.h` for construction and `service.h` for +service descriptors/interfaces. `BRPC_WITH_FLATBUFFERS` in `butil/config.h` +is always 0 or 1; test it with `#if`, not `#ifdef`. + +Flatc 2.0.x emits unqualified `flatbuffers::` names. Use a business schema +namespace outside `brpc` (for example `myapp.rpc`) to avoid shadowing by +`brpc::flatbuffers`; do not rely on include order. Flatc 25.2.10 emits fully +qualified names instead. + +## Message construction and ownership + +Use upstream `flatc --cpp` to generate the schema's `*_generated.h`. Pass a +`brpc::flatbuffers::MessageBuilder` to the generated `Create...` functions, +call `Finish(root)`, and finally call `ReleaseMessage()`. + +* `ReleaseMessage()` does not copy payload bytes. The returned move-only Message + owns an IOBuf block reference and survives builder reuse or destruction. +* Message/builder moves leave the source reusable. Moving a shared-string builder + discards its optional deduplication cache; existing offsets remain valid. +* Importing an ordinary `::flatbuffers::FlatBufferBuilder` copies its payload and + scratch while preserving unfinished table state. The original allocator frees + the original storage, including owned custom allocators. No `free`/`delete[]` + guess or assumption about spare bytes before the payload is made. +* Use MessageBuilder's own move, swap, and release operations. Do not transfer it + through a base-class cast or use inherited raw-buffer release operations: a raw + FlatBuffers detached buffer would retain the address of its member allocator. +* 64 zero-initialized bytes precede a released payload. They may be shortened + with `reduce_meta_size_and_get_buf`; growing them is rejected without mutation. + Payload addresses and bytes are unchanged by shortening metadata. +* Serialization accepts a const Message and retains its storage in the output + IOBuf. The buffer is shared, not copy-on-write: do not mutate payload/metadata + while another reader or serialized buffer is using it. +* Allocation sizes are checked before narrowing to SingleIOBuf's uint32_t size. + Allocation failure is fatal, including release builds, independently of + bRPC's `crash_on_fatal_log` setting. These paths explicitly abort rather than + relying on `CHECK`/`LOG(FATAL)`. Upstream `vector_downward` cannot safely + continue with a null allocation result. + +`ParseFbFromIOBUF` checks sizes/framing and retains independent ownership. It +shares a contiguous input when the payload address is 64-byte aligned; fragmented +or insufficiently aligned input is copied into aligned storage. A builder's +allocation is 64-byte aligned, but the final payload need only have the alignment +required by its schema, so not every local message qualifies for receive-side +zero-copy. Alignments above 64 bytes are not supported. + +**Framing is not schema verification.** Call `msg.Verify()` before +`GetRoot()` or `GetMutableRoot()` on received data. Optional +FlatBuffers strings/vectors can still be null in a valid message. Failed framing +checks leave the prior message intact. + +## Service IDs and generation + +`BrpcDescriptorTable` contains a namespace, service name, whitespace-separated +method names, and explicit method IDs. IDs must be unique nonnegative int32 values. +An empty ID list assigns ordinal IDs to manually constructed descriptors; +generated services require explicit IDs: + +```fbs +rpc_service BenchmarkService { + First(Request):Response (id: 2); + Second(Request):Response (id: 5); +} +``` + +* `descriptor.method(position)` enumerates methods in declaration order. +* `method.index()` is the stable wire ID, not its array position. +* `descriptor.FindMethodByIndex(id)` looks up sparse wire IDs. A transport must + use this lookup rather than indexing a dense array with the wire ID. +* Never recycle a removed method ID for a different method. Removing or reordering + declarations leaves surviving explicit IDs unchanged. +* Namespace `a.b` and `a.b.` normalize to the same service name; empty namespace + means global scope. Method full names include their service. The service hash + uses the canonical full name and MurmurHash3 seed 1. Keep service names stable + when persisting or transmitting these IDs. +* Descriptors cannot be reinitialized after success. They own methods with RAII; + generated accessors use function-local static initialization for thread safety. + +The companion generator in `tools/flatbuffers/` uses the upstream parser to +produce service bindings. It is independently built; only that optional tool +needs `libflatbuffers`. See its [README](../../tools/flatbuffers/README.md) for +commands and limitations. Generated dispatch verifies requests, rejects +unknown/foreign methods, and runs non-null completion callbacks on failure, +including unimplemented methods. Successful implementations own completion and +must run their callback exactly once. diff --git a/src/brpc/flatbuffers/message.cpp b/src/brpc/flatbuffers/message.cpp new file mode 100644 index 0000000000..c9bc775707 --- /dev/null +++ b/src/brpc/flatbuffers/message.cpp @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "brpc/flatbuffers/message.h" + +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include "butil/logging.h" + +namespace brpc { +namespace flatbuffers { +namespace { + +void CheckOrAbort(bool condition, const char* error) { + if (BAIDU_UNLIKELY(!condition)) { + // brpc's crash_on_fatal_log flag is false by default. A CHECK alone + // cannot enforce FlatBuffers' non-null allocator contract. + LOG(ERROR) << error; + std::abort(); + } +} + +uint8_t* AlignPayload(uint8_t* data) { + const uintptr_t address = reinterpret_cast(data); + const size_t padding = (kBufferAlignment - address % kBufferAlignment) % + kBufferAlignment; + return data + padding; +} + +} // namespace + +SlabAllocator& SlabAllocator::operator=(SlabAllocator&& other) noexcept { + if (this != &other) { + SlabAllocator tmp(std::move(other)); + swap(tmp); + } + return *this; +} + +void SlabAllocator::swap(SlabAllocator& other) noexcept { + _iobuf.swap(other._iobuf); + std::swap(_data, other._data); + std::swap(_capacity, other._capacity); +} + +uint8_t* SlabAllocator::allocate(size_t size) { + CheckOrAbort(size > 0 && size < FLATBUFFERS_MAX_BUFFER_SIZE, + "Invalid FlatBuffers allocation size"); + CheckOrAbort(_data == nullptr, "Allocator already has a live buffer"); + butil::SingleIOBuf storage; + const uint32_t allocation_size = static_cast( + size + kDefaultMetaSize + kBufferAlignment - 1); + uint8_t* raw = static_cast(storage.allocate(allocation_size)); + CheckOrAbort(raw != nullptr, "Fail to allocate FlatBuffers storage"); + _data = AlignPayload(raw + kDefaultMetaSize); + _capacity = size; + _iobuf.swap(storage); + return _data; +} + +void SlabAllocator::deallocate(uint8_t* p, size_t /*size*/) { + if (!p) { + return; + } + CheckOrAbort(p == _data, "Invalid FlatBuffers deallocation pointer"); + _iobuf.reset(); + _data = nullptr; + _capacity = 0; +} + +uint8_t* SlabAllocator::reallocate_downward( + uint8_t* old_p, size_t old_size, size_t new_size, + size_t in_use_back, size_t in_use_front) { + CheckOrAbort(old_p != nullptr && old_p == _data && old_size == _capacity, + "Invalid FlatBuffers reallocation buffer"); + CheckOrAbort(new_size > old_size, "FlatBuffers reallocation must grow"); + CheckOrAbort(in_use_back <= old_size && in_use_front <= old_size - in_use_back, + "Invalid FlatBuffers scratch or payload size"); + // Keep the old allocation alive until BOTH data and scratch are copied. + SlabAllocator replacement; + uint8_t* data = replacement.allocate(new_size); + if (in_use_back) { + memcpy(data + new_size - in_use_back, + old_p + old_size - in_use_back, in_use_back); + } + if (in_use_front) { + memcpy(data, old_p, in_use_front); + } + swap(replacement); + return _data; +} + +Message::Message(const butil::IOBuf::BlockRef& ref, uint32_t meta_size, + uint32_t msg_size) + : _iobuf(ref), _meta_size(meta_size), _msg_size(msg_size) {} + +Message& Message::operator=(Message&& other) noexcept { + if (this != &other) { + Clear(); + Swap(other); + } + return *this; +} + +void Message::Swap(Message& other) noexcept { + _iobuf.swap(other._iobuf); + std::swap(_meta_size, other._meta_size); + std::swap(_msg_size, other._msg_size); +} + +void Message::MergeFrom(const Message& /*other*/) { + CheckOrAbort(false, "FlatBuffers Message is move-only; use move assignment"); +} + +void Message::Clear() { + _iobuf.reset(); + _meta_size = 0; + _msg_size = 0; +} + +const uint8_t* Message::data() const { + const uint8_t* raw = static_cast(_iobuf.get_begin()); + return raw ? raw + _meta_size : nullptr; +} + +bool Message::parse_msg_from_iobuf(const butil::IOBuf& buf, size_t msg_size, + size_t meta_size) { + const size_t total = buf.size(); + // Subtraction avoids overflow, and the limits cover SingleIOBuf's uint32_t + // sizes, including allocation overhead, before any narrowing conversion. + if (msg_size == 0 || total >= FLATBUFFERS_MAX_BUFFER_SIZE || + meta_size > total || msg_size != total - meta_size) { + return false; + } + butil::SingleIOBuf storage; + const butil::StringPiece first = buf.backing_block(0); + if (first.size() == total && + reinterpret_cast(first.data() + meta_size) % + kBufferAlignment == 0) { + if (!storage.assign(buf, static_cast(total))) { + return false; + } + } else { + uint8_t* raw = static_cast(storage.allocate( + static_cast(total + kBufferAlignment - 1))); + if (!raw) { + return false; + } + uint8_t* begin = AlignPayload(raw + meta_size) - meta_size; + buf.copy_to(begin, total); + const butil::IOBuf::BlockRef& ref = storage.get_cur_ref(); + butil::IOBuf::BlockRef aligned_ref = { + ref.offset + static_cast(begin - raw), + static_cast(total), ref.block}; + butil::SingleIOBuf aligned(aligned_ref); + storage.swap(aligned); + } + _iobuf.swap(storage); + _meta_size = static_cast(meta_size); + _msg_size = static_cast(msg_size); + return true; +} + +bool Message::append_msg_to_iobuf(butil::IOBuf& buf) const { + if (!data() || !_msg_size) { + return false; + } + _iobuf.append_to(&buf); + return true; +} + +void* Message::reduce_meta_size_and_get_buf(uint32_t new_size) { + if (!data() || new_size > _meta_size) { + errno = EINVAL; + return nullptr; + } + if (new_size != _meta_size) { + const uint32_t offset = _meta_size - new_size; + const butil::IOBuf::BlockRef& ref = _iobuf.get_cur_ref(); + butil::IOBuf::BlockRef sub_ref = { + ref.offset + offset, ref.length - offset, ref.block}; + butil::SingleIOBuf slice(sub_ref); + _iobuf.swap(slice); + _meta_size = new_size; + } + return mutable_buf_begin(); +} + +MessageBuilder::MessageBuilder(size_t initial_size) + : ::flatbuffers::FlatBufferBuilder( + initial_size, &slab_allocator_, false, kBufferAlignment) { + CheckOrAbort(initial_size > 0 && initial_size < FLATBUFFERS_MAX_BUFFER_SIZE, + "Invalid FlatBuffers initial size"); +} + +MessageBuilder::MessageBuilder(MessageBuilder&& other) : MessageBuilder() { + Swap(other); +} + +MessageBuilder& MessageBuilder::operator=(MessageBuilder&& other) { + if (this != &other) { + MessageBuilder tmp(std::move(other)); + Swap(tmp); + } + return *this; +} + +void MessageBuilder::ClearStringPool() { + // FlatBuffers' shared-string comparator stores a vector_downward pointer. + // Discard only this optional cache when moving between vector objects. + delete string_pool; + string_pool = nullptr; +} + +void MessageBuilder::Swap(MessageBuilder& other) { + if (this == &other) { + return; + } + ClearStringPool(); + other.ClearStringPool(); + slab_allocator_.swap(other.slab_allocator_); + ::flatbuffers::FlatBufferBuilder::Swap(other); + // Each builder must continue to refer to its OWN allocator member. + buf_.swap_allocator(other.buf_); +} + +MessageBuilder::MessageBuilder(::flatbuffers::FlatBufferBuilder&& src) + : ::flatbuffers::FlatBufferBuilder(std::move(src)) { + ClearStringPool(); + CheckOrAbort(minalign_ <= kBufferAlignment, + "Unsupported FlatBuffers alignment"); + decltype(buf_) replacement( + buf_.capacity() ? buf_.capacity() : 1024, + &slab_allocator_, false, kBufferAlignment); + if (buf_.capacity()) { + replacement.push(buf_.data(), buf_.size()); + for (::flatbuffers::uoffset_t i = 0; i < buf_.scratch_size(); ++i) { + replacement.scratch_push_small(buf_.scratch_data()[i]); + } + } + // The temporary returns the original buffer to its actual allocator. + buf_.swap(replacement); +} + +MessageBuilder& MessageBuilder::operator=(::flatbuffers::FlatBufferBuilder&& src) { + if (static_cast<::flatbuffers::FlatBufferBuilder*>(this) != &src) { + MessageBuilder tmp(std::move(src)); + Swap(tmp); + } + return *this; +} + +Message MessageBuilder::ReleaseMessage() { + CheckOrAbort(finished && buf_.size() > 0, + "Finish the FlatBuffer before releasing a message"); + const uint32_t msg_size = static_cast(buf_.size()); + const uint8_t* msg_data = buf_.data(); + const butil::SingleIOBuf& storage = slab_allocator_._iobuf; + const uint8_t* raw = static_cast(storage.get_begin()); + CheckOrAbort(raw != nullptr, "Missing FlatBuffers storage"); + CheckOrAbort(msg_data >= raw + kDefaultMetaSize, + "Missing FlatBuffers metadata prefix"); + const size_t begin = msg_data - raw - kDefaultMetaSize; + CheckOrAbort(begin + kDefaultMetaSize + msg_size <= storage.get_length(), + "FlatBuffers message exceeds storage"); + const butil::IOBuf::BlockRef& ref = storage.get_cur_ref(); + const butil::IOBuf::BlockRef sub_ref = { + ref.offset + static_cast(begin), + kDefaultMetaSize + msg_size, ref.block}; + Message msg(sub_ref, kDefaultMetaSize, msg_size); + // Never expose stale scratch or allocator bytes in the reserved prefix. + memset(msg.mutable_buf_begin(), 0, kDefaultMetaSize); + Reset(); + return msg; +} + +bool ParseFbFromIOBUF(Message* msg, size_t msg_size, const butil::IOBuf& buf, + size_t meta_size) { + return msg && msg->parse_msg_from_iobuf(buf, msg_size, meta_size); +} + +bool SerializeFbToIOBUF(const Message* msg, butil::IOBuf& buf) { + return msg && msg->append_msg_to_iobuf(buf); +} + +} // namespace flatbuffers +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS diff --git a/src/brpc/flatbuffers/message.h b/src/brpc/flatbuffers/message.h new file mode 100644 index 0000000000..7304118747 --- /dev/null +++ b/src/brpc/flatbuffers/message.h @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_FLATBUFFERS_MESSAGE_H +#define BRPC_FLATBUFFERS_MESSAGE_H + +#include "butil/config.h" + +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include +#include "butil/single_iobuf.h" +#include "brpc/nonreflectable_message.h" + +namespace brpc { +namespace flatbuffers { + +// Space preceding a payload, available to a future RPC transport. +constexpr uint32_t kDefaultMetaSize = 64; +// IOBuf slices need not start at an aligned address. The allocator corrects it. +constexpr size_t kBufferAlignment = 64; + +class MessageBuilder; + +// FlatBufferBuilder cannot recover from a null allocation. Allocation failure +// and sizes outside FlatBuffers' offset range are fatal, even in release builds. +class SlabAllocator : public ::flatbuffers::Allocator { +public: + SlabAllocator() : _data(nullptr), _capacity(0) {} + SlabAllocator(const SlabAllocator&) = delete; + SlabAllocator& operator=(const SlabAllocator&) = delete; + SlabAllocator(SlabAllocator&& other) noexcept : SlabAllocator() { + swap(other); + } + SlabAllocator& operator=(SlabAllocator&& other) noexcept; + + uint8_t* allocate(size_t size) override; + void deallocate(uint8_t* p, size_t size) override; + uint8_t* reallocate_downward(uint8_t* old_p, size_t old_size, + size_t new_size, size_t in_use_back, + size_t in_use_front) override; + void swap(SlabAllocator& other) noexcept; + +private: + butil::SingleIOBuf _iobuf; + uint8_t* _data; + size_t _capacity; + friend class MessageBuilder; +}; + +// Construct the allocator before the FlatBufferBuilder base uses it, and +// destroy it after that base has returned its buffer. +struct SlabAllocatorMember { + SlabAllocator slab_allocator_; +}; + +// A move-only message. Parsing checks framing, not the schema: Verify() +// must succeed before reading data received from an untrusted peer. +class Message : public NonreflectableMessage { +public: + Message() : _meta_size(0), _msg_size(0) {} + Message(const Message&) = delete; + Message& operator=(const Message&) = delete; + Message(Message&& other) noexcept : Message() { Swap(other); } + Message& operator=(Message&& other) noexcept; + + void MergeFrom(const Message&) override; + void Clear() override; + void Swap(Message& other) noexcept; + + const uint8_t* data() const; + void* mutable_data() { return const_cast(data()); } + void* mutable_buf_begin() { + return const_cast(_iobuf.get_begin()); + } + void* reduce_meta_size_and_get_buf(uint32_t new_size); + uint32_t get_meta_size() const { return _meta_size; } + size_t size() const { return _msg_size; } + + template + bool Verify() const { + if (!data() || size() < sizeof(::flatbuffers::uoffset_t) || + size() >= FLATBUFFERS_MAX_BUFFER_SIZE) { + return false; + } + ::flatbuffers::Verifier verifier(data(), size()); + return verifier.VerifyBuffer(nullptr); + } + + // These accessors require a schema-verified buffer. + template const T* GetRoot() const { + return data() ? ::flatbuffers::GetRoot(data()) : nullptr; + } + template T* GetMutableRoot() { + return data() ? ::flatbuffers::GetMutableRoot(mutable_data()) : nullptr; + } + + // Failure leaves the old message unchanged. A fragmented or unaligned + // payload is copied into aligned storage; aligned contiguous input is shared. + bool parse_msg_from_iobuf(const butil::IOBuf& buf, size_t msg_size, + size_t meta_size); + bool append_msg_to_iobuf(butil::IOBuf& buf) const; + +private: + Message(const butil::IOBuf::BlockRef& ref, uint32_t meta_size, + uint32_t msg_size); + butil::SingleIOBuf _iobuf; + uint32_t _meta_size; + uint32_t _msg_size; + friend class MessageBuilder; +}; + +class MessageBuilder : private SlabAllocatorMember, + public ::flatbuffers::FlatBufferBuilder { +public: + explicit MessageBuilder(size_t initial_size = 1024); + MessageBuilder(const MessageBuilder&) = delete; + MessageBuilder& operator=(const MessageBuilder&) = delete; + MessageBuilder(MessageBuilder&& other); + MessageBuilder& operator=(MessageBuilder&& other); + + // Importing a foreign builder copies its payload and scratch data, retaining + // build state. Its original allocator frees the source allocation correctly. + // In particular, no free()/delete[] guess or spare-prefix assumption is made. + explicit MessageBuilder(::flatbuffers::FlatBufferBuilder&& src); + MessageBuilder& operator=(::flatbuffers::FlatBufferBuilder&& src); + + void Swap(MessageBuilder& other); + // Requires Finish(). The returned message shares storage without copying; + // both this builder and a moved-from builder may immediately be reused. + Message ReleaseMessage(); + +private: + void ClearStringPool(); + // The inherited release methods would retain a pointer to our member + // allocator after this builder dies. Only ReleaseMessage is supported. + using ::flatbuffers::FlatBufferBuilder::Release; + using ::flatbuffers::FlatBufferBuilder::ReleaseRaw; + void ReleaseBufferPointer() = delete; + using ::flatbuffers::FlatBufferBuilder::SwapBufAllocator; +}; + +bool ParseFbFromIOBUF(Message* msg, size_t msg_size, const butil::IOBuf& buf, + size_t meta_size = 0); +bool SerializeFbToIOBUF(const Message* msg, butil::IOBuf& buf); + +} // namespace flatbuffers +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS +#endif // BRPC_FLATBUFFERS_MESSAGE_H diff --git a/src/brpc/flatbuffers/service.cpp b/src/brpc/flatbuffers/service.cpp new file mode 100644 index 0000000000..8bd1a9c2ce --- /dev/null +++ b/src/brpc/flatbuffers/service.cpp @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "brpc/flatbuffers/service.h" + +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include +#include "butil/third_party/murmurhash3/murmurhash3.h" + +namespace brpc { +namespace flatbuffers { +namespace { + +bool IsIdentifier(const std::string& name) { + if (name.empty()) { + return false; + } + for (size_t i = 0; i < name.size(); ++i) { + const char c = name[i]; + if (c != '_' && !(c >= 'a' && c <= 'z') && + !(c >= 'A' && c <= 'Z') && + !(i != 0 && c >= '0' && c <= '9')) { + return false; + } + } + return true; +} + +bool IsNamespace(const std::string& name) { + if (name.empty()) { + return true; + } + size_t begin = 0; + do { + const size_t end = name.find('.', begin); + if (!IsIdentifier(name.substr(begin, end - begin))) { + return false; + } + if (end == std::string::npos) { + return true; + } + begin = end + 1; + } while (begin < name.size()); + return false; +} + +} // namespace + +int ServiceDescriptor::init(const BrpcDescriptorTable& table) { + if (!_methods.empty()) { + errno = EALREADY; + return -1; + } + std::string prefix = table.prefix; + if (!prefix.empty() && prefix.back() == '.') { + prefix.pop_back(); + if (prefix.empty()) { + errno = EINVAL; + return -1; + } + } + if (!IsIdentifier(table.service_name) || !IsNamespace(prefix)) { + errno = EINVAL; + return -1; + } + std::istringstream input(table.method_name_list); + std::vector names; + std::set unique_names; + std::string name; + while (input >> name) { + if (!IsIdentifier(name) || !unique_names.insert(name).second) { + errno = EINVAL; + return -1; + } + names.push_back(name); + } + if (names.empty() || names.size() > static_cast( + std::numeric_limits::max()) || + (!table.method_ids.empty() && table.method_ids.size() != names.size())) { + errno = EINVAL; + return -1; + } + std::set unique_ids; + for (int id : table.method_ids) { + if (id < 0 || !unique_ids.insert(id).second) { + errno = EINVAL; + return -1; + } + } + _name = table.service_name; + _full_name = prefix.empty() ? _name : prefix + "." + _name; + butil::MurmurHash3_x86_32(_full_name.data(), _full_name.size(), 1, &_index); + _methods.reserve(names.size()); + for (size_t i = 0; i < names.size(); ++i) { + const int id = table.method_ids.empty() ? static_cast(i) + : table.method_ids[i]; + _methods.emplace_back(new MethodDescriptor(names[i], this, id)); + } + return 0; +} + +const MethodDescriptor* ServiceDescriptor::method(int position) const { + if (position < 0 || static_cast(position) >= _methods.size()) { + errno = EINVAL; + return nullptr; + } + return _methods[position].get(); +} + +const MethodDescriptor* ServiceDescriptor::FindMethodByIndex(int id) const { + for (const auto& method : _methods) { + if (method->index() == id) { + return method.get(); + } + } + errno = EINVAL; + return nullptr; +} + +MethodDescriptor::MethodDescriptor(const std::string& name, + const ServiceDescriptor* service, int index) + : _name(name), _full_name(service->full_name() + "." + name), + _service(service), _index(index) {} + +int parse_service_descriptors(const BrpcDescriptorTable& table, + ServiceDescriptor** out) { + if (!out) { + errno = EINVAL; + return -1; + } + std::unique_ptr descriptor(new ServiceDescriptor); + if (descriptor->init(table) != 0) { + return -1; + } + *out = descriptor.release(); + return 0; +} + +} // namespace flatbuffers +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS diff --git a/src/brpc/flatbuffers/service.h b/src/brpc/flatbuffers/service.h new file mode 100644 index 0000000000..4423170be9 --- /dev/null +++ b/src/brpc/flatbuffers/service.h @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_FLATBUFFERS_SERVICE_H +#define BRPC_FLATBUFFERS_SERVICE_H + +#include "butil/config.h" + +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include + +namespace google { +namespace protobuf { +class Closure; +class RpcController; +} // namespace protobuf +} // namespace google + +namespace brpc { +namespace flatbuffers { +class Message; +class ServiceDescriptor; + +struct BrpcDescriptorTable { + // A namespace, optionally followed by one period. Empty means global scope. + std::string prefix; + std::string service_name; + // Whitespace-separated method names in declaration order. + std::string method_name_list; + // Stable wire IDs, NOT array positions. Empty retains legacy ordinal IDs. + // New schemas should specify every ID; IDs must be nonnegative and unique. + std::vector method_ids; +}; + +class MethodDescriptor { +public: + MethodDescriptor(const std::string& name, const ServiceDescriptor* service, + int index); + const std::string& name() const { return _name; } + const std::string& full_name() const { return _full_name; } + int index() const { return _index; } + const ServiceDescriptor* service() const { return _service; } + +private: + std::string _name; + std::string _full_name; + const ServiceDescriptor* _service; + int _index; +}; + +// Immutable after successful initialization. Owns all method descriptors. +class ServiceDescriptor { +public: + ServiceDescriptor() : _index(0) {} + ServiceDescriptor(const ServiceDescriptor&) = delete; + ServiceDescriptor& operator=(const ServiceDescriptor&) = delete; + int init(const BrpcDescriptorTable& table); + const std::string& name() const { return _name; } + const std::string& full_name() const { return _full_name; } + // MurmurHash3 of the canonical fully-qualified service name, seed 1. + uint32_t index() const { return _index; } + int method_count() const { return static_cast(_methods.size()); } + // Dense declaration-order lookup, for enumeration and generated stubs. + const MethodDescriptor* method(int position) const; + // Sparse stable-ID lookup, for wire dispatch. Missing IDs return nullptr. + const MethodDescriptor* FindMethodByIndex(int id) const; + +private: + std::string _name; + std::string _full_name; + uint32_t _index; + std::vector > _methods; +}; + +// On success the caller owns *out; on failure *out is unchanged. +int parse_service_descriptors(const BrpcDescriptorTable& table, + ServiceDescriptor** out); + +class RpcChannel { +public: + RpcChannel() = default; + virtual ~RpcChannel() = default; + RpcChannel(const RpcChannel&) = delete; + RpcChannel& operator=(const RpcChannel&) = delete; + virtual void FBCallMethod(const MethodDescriptor* method, + google::protobuf::RpcController* controller, + const Message* request, Message* response, + google::protobuf::Closure* done) = 0; +}; + +class Service { +public: + Service() = default; + virtual ~Service() = default; + Service(const Service&) = delete; + Service& operator=(const Service&) = delete; + enum ChannelOwnership { STUB_OWNS_CHANNEL, STUB_DOESNT_OWN_CHANNEL }; + virtual const ServiceDescriptor* GetDescriptor() = 0; + // Implementations must validate the method and request, report errors on + // controller and run a non-null done exactly once, including failure paths. + virtual void FBCallMethod(const MethodDescriptor* method, + google::protobuf::RpcController* controller, + const Message* request, Message* response, + google::protobuf::Closure* done) = 0; +}; + +} // namespace flatbuffers +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS +#endif // BRPC_FLATBUFFERS_SERVICE_H diff --git a/test/BUILD.bazel b/test/BUILD.bazel index 1aecc39987..a2c291eb7e 100644 --- a/test/BUILD.bazel +++ b/test/BUILD.bazel @@ -229,11 +229,47 @@ root_runfiles( ], ) +FLATBUFFERS_COMPATIBILITY = select({ + "//:brpc_with_flatbuffers": [], + "//conditions:default": ["@platforms//:incompatible"], +}) + +genrule( + name = "flatbuffers_test_messages", + srcs = ["flatbuffers_message.fbs"], + outs = ["flatbuffers_message_generated.h"], + cmd = select({ + "//:brpc_with_flatbuffers": "$(location @com_github_google_flatbuffers//:flatc) --cpp -o $(@D) $(location flatbuffers_message.fbs)", + "//conditions:default": "exit 1", + }), + target_compatible_with = FLATBUFFERS_COMPATIBILITY, + tools = select({ + "//:brpc_with_flatbuffers": ["@com_github_google_flatbuffers//:flatc"], + "//conditions:default": [], + }), +) + +cc_test( + name = "brpc_flatbuffers_unittest", + srcs = [ + "brpc_flatbuffers_unittest.cpp", + ":flatbuffers_test_messages", + ], + copts = COPTS, + includes = ["."], + target_compatible_with = FLATBUFFERS_COMPATIBILITY, + deps = [ + "//:brpc", + "@com_google_googletest//:gtest_main", + ], +) + generate_unittests( name = "brpc_unittests", - srcs = glob([ - "brpc_*_unittest.cpp", - ]), + srcs = glob( + ["brpc_*_unittest.cpp"], + exclude = ["brpc_flatbuffers_unittest.cpp"], + ), deps = [ ":gperftools_helper", "//:brpc", diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 71baf85df4..ee01ad8165 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -268,8 +268,35 @@ foreach(BTHREAD_UT ${BTHREAD_UNITTESTS}) add_test(NAME ${BTHREAD_UT_WE} COMMAND ${BTHREAD_UT_WE}) endforeach() +# FlatBuffers tests use the release library, independently of the debug suite. +if(WITH_FLATBUFFERS) + find_program(FLATBUFFERS_FLATC_EXECUTABLE NAMES flatc) + if(NOT FLATBUFFERS_FLATC_EXECUTABLE) + message(FATAL_ERROR + "FlatBuffers tests require an installed flatc; set FLATBUFFERS_FLATC_EXECUTABLE.") + endif() + set(FLATBUFFERS_TEST_HEADER + "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_message_generated.h") + add_custom_command( + OUTPUT "${FLATBUFFERS_TEST_HEADER}" + COMMAND "${FLATBUFFERS_FLATC_EXECUTABLE}" --cpp + -o "${CMAKE_CURRENT_BINARY_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/flatbuffers_message.fbs" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/flatbuffers_message.fbs" + "${FLATBUFFERS_FLATC_EXECUTABLE}" + COMMENT "Generating FlatBuffers test messages" + VERBATIM) + add_executable(brpc_flatbuffers_unittest + brpc_flatbuffers_unittest.cpp "${FLATBUFFERS_TEST_HEADER}") + target_link_libraries(brpc_flatbuffers_unittest PRIVATE + brpc-static brpc_test_config gtest_main ${DYNAMIC_LIB}) + add_test(NAME brpc_flatbuffers_unittest COMMAND brpc_flatbuffers_unittest) +endif() + # brpc tests file(GLOB BRPC_UNITTESTS "brpc_*_unittest.cpp") +list(REMOVE_ITEM BRPC_UNITTESTS + "${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_unittest.cpp") foreach(BRPC_UT ${BRPC_UNITTESTS}) get_filename_component(BRPC_UT_WE ${BRPC_UT} NAME_WE) add_executable(${BRPC_UT_WE} ${BRPC_UT} $) diff --git a/test/Makefile b/test/Makefile index de4d566bcd..e0cca7b819 100644 --- a/test/Makefile +++ b/test/Makefile @@ -181,6 +181,9 @@ TEST_BTHREAD_SOURCES = $(wildcard bthread_*unittest.cpp) TEST_BTHREAD_OBJS = $(addsuffix .o, $(basename $(TEST_BTHREAD_SOURCES))) TEST_BRPC_SOURCES = $(wildcard brpc_*unittest.cpp) +ifneq ($(WITH_FLATBUFFERS),1) +TEST_BRPC_SOURCES := $(filter-out brpc_flatbuffers_unittest.cpp,$(TEST_BRPC_SOURCES)) +endif TEST_BRPC_OBJS = $(addsuffix .o, $(basename $(TEST_BRPC_SOURCES))) TEST_PROTO_SOURCES = $(wildcard *.proto) @@ -194,7 +197,7 @@ all: $(TEST_BINS) .PHONY:clean clean:clean_bins @echo "> Cleaning" - rm -rf $(TEST_BUTIL_OBJS) $(TEST_BVAR_OBJS) $(TEST_BTHREAD_OBJS) $(TEST_BRPC_OBJS) $(TEST_PROTO_OBJS) $(TEST_PROTO_SOURCES:.proto=.pb.h) $(TEST_PROTO_SOURCES:.proto=.pb.cc) + rm -rf $(TEST_BUTIL_OBJS) $(TEST_BVAR_OBJS) $(TEST_BTHREAD_OBJS) $(TEST_BRPC_OBJS) $(TEST_PROTO_OBJS) $(TEST_PROTO_SOURCES:.proto=.pb.h) $(TEST_PROTO_SOURCES:.proto=.pb.cc) flatbuffers_message_generated.h $(MAKE) -C.. clean_debug .PHONY:clean_bins @@ -235,6 +238,23 @@ else ifeq ($(SYSTEM),Darwin) $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) $^ $(GTEST_STATIC_LINKINGS) $(UT_DYNAMIC_LINKINGS) endif +ifeq ($(WITH_FLATBUFFERS),1) +FLATC ?= flatc + +flatbuffers_message_generated.h:flatbuffers_message.fbs + $(FLATC) --cpp -o . $< + +brpc_flatbuffers_unittest.o:flatbuffers_message_generated.h + +brpc_flatbuffers_unittest:brpc_flatbuffers_unittest.o | libbrpc.dbg.$(SOEXT) + @echo "> Linking $@" +ifeq ($(SYSTEM),Linux) + $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(UT_DYNAMIC_LINKINGS) +else ifeq ($(SYSTEM),Darwin) + $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) $^ $(GTEST_STATIC_LINKINGS) $(UT_DYNAMIC_LINKINGS) +endif +endif + brpc_%_unittest:$(TEST_PROTO_OBJS) brpc_%_unittest.o | libbrpc.dbg.$(SOEXT) @echo "> Linking $@" ifeq ($(SYSTEM),Linux) diff --git a/test/brpc_flatbuffers_unittest.cpp b/test/brpc_flatbuffers_unittest.cpp new file mode 100644 index 0000000000..e4defbc369 --- /dev/null +++ b/test/brpc_flatbuffers_unittest.cpp @@ -0,0 +1,528 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "butil/config.h" +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include +#include +#include +#include +#include +#include "brpc/flatbuffers/message.h" +#include "brpc/flatbuffers/service.h" +#include "flatbuffers_message_generated.h" + +#if !BRPC_WITH_GLOG +namespace logging { +DECLARE_bool(crash_on_fatal_log); +} +#endif + +namespace { +using brpc::flatbuffers::Message; +using brpc::flatbuffers::MessageBuilder; +using brpc::flatbuffers::SlabAllocator; +using brpc::flatbuffers::ServiceDescriptor; +using brpc::flatbuffers::BrpcDescriptorTable; +using brpc_fbtest::Payload; + +void FinishPayload(::flatbuffers::FlatBufferBuilder& builder, + const std::string& text, int64_t value = 42) { + auto str = builder.CreateString(text); + std::vector numbers = {1, -2, 3, 4000}; + auto values = builder.CreateVector(numbers); + builder.Finish(brpc_fbtest::CreatePayload(builder, value, str, values)); +} + +Message MakeMessage(size_t length = 16, int64_t value = 42) { + MessageBuilder builder(8); + FinishPayload(builder, std::string(length, 'x'), value); + return builder.ReleaseMessage(); +} + +void ExpectPayload(const Message& msg, size_t length, int64_t value = 42) { + ASSERT_TRUE(msg.Verify()); + const Payload* root = msg.GetRoot(); + ASSERT_NE(nullptr, root); + EXPECT_EQ(value, root->value()); + ASSERT_NE(nullptr, root->message()); + EXPECT_EQ(std::string(length, 'x'), root->message()->str()); + ASSERT_NE(nullptr, root->values()); + ASSERT_EQ(4u, root->values()->size()); + EXPECT_EQ(-2, root->values()->Get(1)); +} + +TEST(FlatbuffersTest, EmptyAndClear) { + Message msg; + EXPECT_EQ(nullptr, msg.data()); + EXPECT_EQ(nullptr, msg.GetRoot()); + EXPECT_EQ(nullptr, msg.GetMutableRoot()); + EXPECT_EQ(nullptr, msg.reduce_meta_size_and_get_buf(0)); + EXPECT_FALSE(msg.Verify()); + EXPECT_EQ(0u, msg.size()); + butil::IOBuf wire; + EXPECT_FALSE(brpc::flatbuffers::SerializeFbToIOBUF(&msg, wire)); + EXPECT_FALSE(brpc::flatbuffers::SerializeFbToIOBUF(nullptr, wire)); + EXPECT_FALSE(brpc::flatbuffers::ParseFbFromIOBUF(nullptr, 0, wire)); + msg = MakeMessage(); + msg.Clear(); + msg.Clear(); + EXPECT_EQ(nullptr, msg.data()); + EXPECT_EQ(0u, msg.get_meta_size()); + EXPECT_EQ(0u, msg.size()); +} + +TEST(FlatbuffersTest, ReleaseLifetimeAndGrowth) { + for (size_t length : {0u, 16u, 63u, 1024u, 8192u, 32768u}) { + SCOPED_TRACE(length); + Message msg; + const uint8_t* payload = nullptr; + { + MessageBuilder builder(8); + FinishPayload(builder, std::string(length, 'x')); + payload = builder.GetBufferPointer(); + msg = builder.ReleaseMessage(); + EXPECT_EQ(payload, msg.data()); + EXPECT_EQ(0u, builder.GetSize()); + FinishPayload(builder, "replacement"); + } + ExpectPayload(msg, length); + EXPECT_EQ(std::string(brpc::flatbuffers::kDefaultMetaSize, '\0'), + std::string(static_cast(msg.mutable_buf_begin()), + msg.get_meta_size())); + } +} + +TEST(FlatbuffersTest, MessageMoveAndSwap) { + Message first = MakeMessage(16, 1); + const uint8_t* ptr = first.data(); + Message second(std::move(first)); + EXPECT_EQ(ptr, second.data()); + EXPECT_EQ(nullptr, first.data()); + EXPECT_EQ(0u, first.size()); + EXPECT_EQ(0u, first.get_meta_size()); + EXPECT_FALSE(first.Verify()); + first = MakeMessage(8192, 2); + second = std::move(first); + EXPECT_EQ(nullptr, first.data()); + ExpectPayload(second, 8192, 2); + Message* alias = &second; + second = std::move(*alias); + ExpectPayload(second, 8192, 2); + first = MakeMessage(16, 3); + first.Swap(second); + ExpectPayload(first, 8192, 2); + ExpectPayload(second, 16, 3); + Message empty; + second = std::move(empty); + EXPECT_EQ(nullptr, second.data()); + EXPECT_EQ(nullptr, empty.data()); +} + +TEST(FlatbuffersTest, BuilderMoveAndReuse) { + MessageBuilder first(8); + auto str = first.CreateString(std::string(8192, 'x')); + MessageBuilder second(std::move(first)); + second.Finish(brpc_fbtest::CreatePayload(second, 8, str)); + Message msg = second.ReleaseMessage(); + ASSERT_TRUE(msg.Verify()); + EXPECT_EQ(8192u, msg.GetRoot()->message()->size()); + FinishPayload(first, std::string(16, 'x')); + ExpectPayload(first.ReleaseMessage(), 16); + FinishPayload(second, std::string(63, 'x')); + first = std::move(second); + MessageBuilder* alias = &first; + first = std::move(*alias); + ExpectPayload(first.ReleaseMessage(), 63); + FinishPayload(second, std::string(1024, 'x')); + ExpectPayload(second.ReleaseMessage(), 1024); +} + +TEST(FlatbuffersTest, BuilderSwapFinishedAndUnfinished) { + MessageBuilder first; + FinishPayload(first, std::string(16, 'x')); + MessageBuilder second; + auto str = second.CreateString(std::string(63, 'x')); + first.Swap(second); + ExpectPayload(second.ReleaseMessage(), 16); + first.Finish(brpc_fbtest::CreatePayload(first, 9, str)); + Message msg = first.ReleaseMessage(); + ASSERT_TRUE(msg.Verify()); + EXPECT_EQ(9, msg.GetRoot()->value()); + EXPECT_EQ(63u, msg.GetRoot()->message()->size()); +} + +TEST(FlatbuffersTest, SharedStringsAfterMoveAndImport) { + MessageBuilder first(8); + auto old = first.CreateSharedString("shared"); + MessageBuilder second(std::move(first)); + auto same = second.CreateSharedString("shared"); + auto other = second.CreateSharedString("other"); + std::vector<::flatbuffers::Offset<::flatbuffers::String> > strings = { + old, same, other}; + second.Finish(second.CreateVector(strings)); + Message msg = second.ReleaseMessage(); + auto root = ::flatbuffers::GetRoot< + ::flatbuffers::Vector<::flatbuffers::Offset<::flatbuffers::String> > >( + msg.data()); + EXPECT_EQ("shared", root->Get(0)->str()); + EXPECT_EQ("shared", root->Get(1)->str()); + EXPECT_EQ("other", root->Get(2)->str()); + ::flatbuffers::FlatBufferBuilder foreign; + auto text = foreign.CreateSharedString("shared"); + MessageBuilder imported(std::move(foreign)); + imported.CreateSharedString("after import"); + imported.Finish(brpc_fbtest::CreatePayload(imported, 1, text)); + ASSERT_TRUE(imported.ReleaseMessage().Verify()); +} + +class CountingAllocator : public ::flatbuffers::Allocator { +public: + CountingAllocator(int* allocations, int* frees, int* destructors) + : _allocations(allocations), _frees(frees), _destructors(destructors) {} + ~CountingAllocator() override { ++*_destructors; } + uint8_t* allocate(size_t n) override { + ++*_allocations; + return new uint8_t[n]; + } + void deallocate(uint8_t* p, size_t) override { + ++*_frees; + delete[] p; + } +private: + int* _allocations; + int* _frees; + int* _destructors; +}; + +TEST(FlatbuffersTest, ImportForeignAllocatorAndScratch) { + int allocations = 0; + int frees = 0; + int destructors = 0; + Message msg; + { + ::flatbuffers::FlatBufferBuilder foreign(8, + new CountingAllocator(&allocations, &frees, &destructors), true); + auto text = foreign.CreateString(std::string(8192, 'x')); + const auto table = foreign.StartTable(); + foreign.AddOffset(Payload::VT_MESSAGE, text); + foreign.AddElement(Payload::VT_VALUE, 123, 0); + MessageBuilder imported(std::move(foreign)); + EXPECT_EQ(allocations, frees); + EXPECT_EQ(1, destructors); + imported.Finish(::flatbuffers::Offset(imported.EndTable(table))); + msg = imported.ReleaseMessage(); + FinishPayload(foreign, "reuse source"); + } + ASSERT_TRUE(msg.Verify()); + EXPECT_EQ(123, msg.GetRoot()->value()); + EXPECT_EQ(8192u, msg.GetRoot()->message()->size()); + EXPECT_EQ(allocations, frees); + EXPECT_EQ(1, destructors); +} + +TEST(FlatbuffersTest, ImportBorrowedAllocatorDoesNotOwnAllocator) { + int allocations = 0; + int frees = 0; + int destructors = 0; + Message msg; + { + CountingAllocator allocator(&allocations, &frees, &destructors); + ::flatbuffers::FlatBufferBuilder foreign(8, &allocator, false); + FinishPayload(foreign, std::string(1024, 'x')); + MessageBuilder imported(std::move(foreign)); + EXPECT_EQ(allocations, frees); + EXPECT_EQ(0, destructors); + msg = imported.ReleaseMessage(); + } + EXPECT_EQ(1, destructors); + ExpectPayload(msg, 1024); +} + +TEST(FlatbuffersTest, ImportFinishedDefaultBuilderAndEmpty) { + ::flatbuffers::FlatBufferBuilder foreign(8); + FinishPayload(foreign, std::string(8192, 'x')); + MessageBuilder imported(std::move(foreign)); + ExpectPayload(imported.ReleaseMessage(), 8192); + ::flatbuffers::FlatBufferBuilder empty; + imported = std::move(empty); + FinishPayload(imported, std::string(16, 'x')); + ExpectPayload(imported.ReleaseMessage(), 16); + FinishPayload(imported, std::string(63, 'x')); + ::flatbuffers::FlatBufferBuilder& alias = imported; + imported = std::move(alias); + ExpectPayload(imported.ReleaseMessage(), 63); +} + +TEST(FlatbuffersTest, AllocatorFrontBackAndMove) { + SlabAllocator first; + uint8_t* data = first.allocate(64); + EXPECT_EQ(0u, reinterpret_cast(data) % + brpc::flatbuffers::kBufferAlignment); + memset(data, 'f', 16); + memset(data + 48, 'b', 16); + SlabAllocator second(std::move(first)); + size_t previous = 64; + for (size_t next : {128u, 512u, 8192u, 16384u}) { + data = second.reallocate_downward(data, previous, next, 16, 16); + EXPECT_EQ(std::string(16, 'f'), std::string((char*)data, 16)); + EXPECT_EQ(std::string(16, 'b'), std::string((char*)data + next - 16, 16)); + previous = next; + } + first = std::move(second); + SlabAllocator* alias = &first; + first = std::move(*alias); + first.deallocate(data, 16384); + second.deallocate(nullptr, 0); + data = second.allocate(32); + second.deallocate(data, 32); +} + +TEST(FlatbuffersTest, SerializeConstAndRetainWireStorage) { + butil::IOBuf wire; + { + const Message msg = MakeMessage(8192); + ASSERT_TRUE(brpc::flatbuffers::SerializeFbToIOBUF(&msg, wire)); + EXPECT_EQ(msg.size() + msg.get_meta_size(), wire.size()); + } + Message parsed; + const size_t meta = brpc::flatbuffers::kDefaultMetaSize; + ASSERT_TRUE(brpc::flatbuffers::ParseFbFromIOBUF( + &parsed, wire.size() - meta, wire, meta)); + wire.clear(); + ExpectPayload(parsed, 8192); +} + +TEST(FlatbuffersTest, ParseAlignedStorageIsSharedAndRetained) { + Message source = MakeMessage(); + void* raw = nullptr; + ASSERT_EQ(0, posix_memalign(&raw, brpc::flatbuffers::kBufferAlignment, + source.size())); + memcpy(raw, source.data(), source.size()); + int frees = 0; + butil::IOBuf wire; + ASSERT_EQ(0, wire.append_user_data(raw, source.size(), [&frees](void* p) { + ++frees; + free(p); + })); + Message parsed; + ASSERT_TRUE(parsed.parse_msg_from_iobuf(wire, source.size(), 0)); + EXPECT_EQ(raw, parsed.data()); + wire.clear(); + EXPECT_EQ(0, frees); + ExpectPayload(parsed, 16); + parsed.Clear(); + EXPECT_EQ(1, frees); +} + +TEST(FlatbuffersTest, ParseFragmentedUnalignedAndReplace) { + for (size_t length : {16u, 8192u, 32768u}) { + Message source = MakeMessage(length); + butil::IOBuf wire; + const size_t split = source.size() / 2; + void* first = malloc(split); + void* second = malloc(source.size() - split); + memcpy(first, source.data(), split); + memcpy(second, source.data() + split, source.size() - split); + ASSERT_EQ(0, wire.append_user_data(first, split, free)); + ASSERT_EQ(0, wire.append_user_data(second, source.size() - split, free)); + ASSERT_EQ(2u, wire.backing_block_num()); + Message parsed = MakeMessage(1); + ASSERT_TRUE(parsed.parse_msg_from_iobuf(wire, source.size(), 0)); + ASSERT_TRUE(parsed.parse_msg_from_iobuf(wire, source.size(), 0)); + wire.clear(); + ExpectPayload(parsed, length); + EXPECT_EQ(0u, reinterpret_cast(parsed.data()) % + brpc::flatbuffers::kBufferAlignment); + + char* bytes = static_cast(malloc(source.size() + 1)); + bytes[0] = 'h'; + memcpy(bytes + 1, source.data(), source.size()); + ASSERT_EQ(0, wire.append_user_data(bytes, source.size() + 1, free)); + ASSERT_TRUE(parsed.parse_msg_from_iobuf(wire, source.size(), 1)); + EXPECT_NE(bytes + 1, reinterpret_cast(parsed.data())); + wire.clear(); + ExpectPayload(parsed, length); + EXPECT_EQ(1u, parsed.get_meta_size()); + } +} + +TEST(FlatbuffersTest, ParseRejectsBadFramingWithoutChangingMessage) { + Message msg = MakeMessage(16); + const uint8_t* data = msg.data(); + butil::IOBuf buf; + buf.append("abcd"); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, 0, 4)); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, 4, 1)); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, 5, 0)); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, 1, 5)); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, std::numeric_limits::max(), 5)); + EXPECT_FALSE(msg.parse_msg_from_iobuf(buf, 5, std::numeric_limits::max())); + EXPECT_EQ(data, msg.data()); + ExpectPayload(msg, 16); +} + +TEST(FlatbuffersTest, InvalidPayloadAndOptionalString) { + Message msg; + for (size_t size : {1u, 3u, 4u, 16u, 64u}) { + butil::IOBuf buf; + buf.append(std::string(size, '\xff')); + ASSERT_TRUE(msg.parse_msg_from_iobuf(buf, size, 0)); + EXPECT_FALSE(msg.Verify()); + } + MessageBuilder builder; + builder.Finish(brpc_fbtest::CreatePayload(builder)); + msg = builder.ReleaseMessage(); + ASSERT_TRUE(msg.Verify()); + EXPECT_EQ(nullptr, msg.GetRoot()->message()); + // The schema allows an absent string; callers must not dereference it. + msg = MakeMessage(); + memset(msg.mutable_data(), 0xff, sizeof(::flatbuffers::uoffset_t)); + EXPECT_FALSE(msg.Verify()); +} + +TEST(FlatbuffersTest, ShrinkMetadataDoesNotMovePayload) { + Message msg = MakeMessage(16); + const uint8_t* data = msg.data(); + EXPECT_EQ(nullptr, msg.reduce_meta_size_and_get_buf(65)); + ASSERT_NE(nullptr, msg.reduce_meta_size_and_get_buf(36)); + memset(msg.mutable_buf_begin(), 'm', 36); + EXPECT_EQ(data, msg.data()); + ExpectPayload(msg, 16); + EXPECT_EQ(data, msg.reduce_meta_size_and_get_buf(0)); + EXPECT_EQ(data, msg.reduce_meta_size_and_get_buf(0)); + EXPECT_EQ(nullptr, msg.reduce_meta_size_and_get_buf(1)); + EXPECT_EQ(0u, msg.get_meta_size()); + butil::IOBuf wire; + ASSERT_TRUE(msg.append_msg_to_iobuf(wire)); + EXPECT_EQ(msg.size(), wire.size()); +} + +void* FailBlockAllocation(size_t) { return nullptr; } + +class FlatbuffersDeathTest : public ::testing::Test { +protected: + void SetUp() override { + _old_style = GTEST_FLAG_GET(death_test_style); + GTEST_FLAG_SET(death_test_style, "threadsafe"); +#if !BRPC_WITH_GLOG + _old = logging::FLAGS_crash_on_fatal_log; + logging::FLAGS_crash_on_fatal_log = false; +#endif + } + void TearDown() override { + GTEST_FLAG_SET(death_test_style, _old_style); +#if !BRPC_WITH_GLOG + logging::FLAGS_crash_on_fatal_log = _old; +#endif + } +private: + bool _old = false; + std::string _old_style; +}; + +TEST_F(FlatbuffersDeathTest, AllocationFailureIsChecked) { + EXPECT_DEATH({ + butil::iobuf::blockmem_allocate = FailBlockAllocation; + SlabAllocator allocator; + allocator.allocate(1024 * 1024); + }, "Fail to allocate"); + EXPECT_DEATH({ + SlabAllocator allocator; + uint8_t* data = allocator.allocate(1024 * 1024); + butil::iobuf::blockmem_allocate = FailBlockAllocation; + allocator.reallocate_downward(data, 1024 * 1024, 2 * 1024 * 1024, 8, 8); + }, "Fail to allocate"); +} + +TEST_F(FlatbuffersDeathTest, RejectsUnsafeAllocatorInputsAndUnfinishedRelease) { + EXPECT_DEATH({ SlabAllocator a; a.allocate(0); }, ""); + EXPECT_DEATH({ SlabAllocator a; a.allocate(std::numeric_limits::max()); }, ""); + EXPECT_DEATH({ + SlabAllocator a; + uint8_t* p = a.allocate(8); + a.reallocate_downward(p, 8, 16, 8, 1); + }, ""); + EXPECT_DEATH({ MessageBuilder builder; builder.ReleaseMessage(); }, "Finish"); + EXPECT_DEATH({ Message msg; Message other; msg.MergeFrom(other); }, "move-only"); +} + +TEST(FlatbuffersDescriptorTest, SparseStableIDsAndCanonicalNames) { + ServiceDescriptor first; + ASSERT_EQ(0, first.init({"test.", "Benchmark", "One Two Three", {2, 5, 1}})); + EXPECT_EQ("test.Benchmark", first.full_name()); + EXPECT_EQ(3, first.method_count()); + EXPECT_EQ(2, first.method(0)->index()); + EXPECT_EQ(5, first.method(1)->index()); + EXPECT_EQ("test.Benchmark.Two", first.method(1)->full_name()); + EXPECT_EQ(&first, first.method(0)->service()); + EXPECT_EQ(first.method(1), first.FindMethodByIndex(5)); + EXPECT_EQ(nullptr, first.FindMethodByIndex(3)); + EXPECT_EQ(nullptr, first.method(-1)); + EXPECT_EQ(nullptr, first.method(3)); + ServiceDescriptor reduced; + ASSERT_EQ(0, reduced.init({"test", "Benchmark", "Three Two", {1, 5}})); + EXPECT_EQ(first.index(), reduced.index()); + EXPECT_EQ(first.FindMethodByIndex(5)->index(), reduced.method(1)->index()); + EXPECT_EQ(first.FindMethodByIndex(5)->full_name(), reduced.method(1)->full_name()); + EXPECT_NE(0, first.init({"test", "Other", "Call", {3}})); + EXPECT_EQ("test.Benchmark", first.full_name()); +} + +TEST(FlatbuffersDescriptorTest, LegacyGlobalAndWhitespace) { + ServiceDescriptor desc; + ASSERT_EQ(0, desc.init({"", "Global", " First\tSecond\n Third ", {}})); + EXPECT_EQ("Global", desc.full_name()); + EXPECT_EQ(3, desc.method_count()); + EXPECT_EQ(0, desc.method(0)->index()); + EXPECT_EQ(2, desc.method(2)->index()); + EXPECT_EQ("Global.First", desc.method(0)->full_name()); +} + +TEST(FlatbuffersDescriptorTest, InvalidTablesAndFailureOwnership) { + const std::vector invalid = { + {"test", "", "A", {}}, {"test", "Service", "", {}}, + {"test", "Service", "A A", {1, 2}}, + {"test", "Service", "A B", {1, 1}}, + {"test", "Service", "A B", {1}}, + {"test", "Service", "A B", {-1, 2}}, + {"test..", "Service", "A", {}}, {".", "Service", "A", {}}, + {"test", "Bad.Name", "A", {}}, {"test", "Service", "1Bad", {}} + }; + ServiceDescriptor desc; + for (const auto& table : invalid) { + EXPECT_NE(0, desc.init(table)); + EXPECT_EQ(0, desc.method_count()); + ServiceDescriptor* output = &desc; + EXPECT_NE(0, brpc::flatbuffers::parse_service_descriptors(table, &output)); + EXPECT_EQ(&desc, output); + } + ASSERT_EQ(0, desc.init({"test", "Service", "A", {0}})); + EXPECT_NE(0, brpc::flatbuffers::parse_service_descriptors( + {"test", "Service", "A", {0}}, nullptr)); + ServiceDescriptor* output = nullptr; + ASSERT_EQ(0, brpc::flatbuffers::parse_service_descriptors( + {"test", "Service", "A", {0}}, &output)); + std::unique_ptr owner(output); + EXPECT_EQ("test.Service", owner->full_name()); +} + +} // namespace +#endif // BRPC_WITH_FLATBUFFERS diff --git a/test/flatbuffers_codegen/CMakeLists.txt b/test/flatbuffers_codegen/CMakeLists.txt new file mode 100644 index 0000000000..86b8cb694e --- /dev/null +++ b/test/flatbuffers_codegen/CMakeLists.txt @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +find_package(Protobuf REQUIRED) +find_program(FLATC_EXECUTABLE flatc) +if(NOT FLATC_EXECUTABLE) + message(FATAL_ERROR "Codegen tests need the official flatc executable") +endif() +get_filename_component(BRPC_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../.." ABSOLUTE) +set(GENERATED_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated") +file(MAKE_DIRECTORY "${GENERATED_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/include/butil") +# Keep test-only configuration in this build tree, never in the repository. +set(WITH_FLATBUFFERS_VAL 1) +configure_file("${BRPC_SOURCE_DIR}/config.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/include/butil/config.h") +set(CODEGEN_INCLUDE_DIRS + "${CMAKE_CURRENT_BINARY_DIR}/include" + "${BRPC_SOURCE_DIR}/src" + "${GENERATED_DIR}" + "${FLATBUFFERS_INCLUDE_DIR}" + ${Protobuf_INCLUDE_DIRS}) +add_custom_command( + OUTPUT "${GENERATED_DIR}/echo.brpc.fb.cpp" + "${GENERATED_DIR}/echo.brpc.fb.h" + "${GENERATED_DIR}/echo_generated.h" + COMMAND "${FLATC_EXECUTABLE}" --cpp -o "${GENERATED_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/echo.fbs" + COMMAND brpc_flatc -o "${GENERATED_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/echo.fbs" + DEPENDS brpc_flatc "${CMAKE_CURRENT_SOURCE_DIR}/echo.fbs" + VERBATIM) +add_library(brpc_codegen_compile OBJECT "${GENERATED_DIR}/echo.brpc.fb.cpp") +target_include_directories(brpc_codegen_compile PRIVATE ${CODEGEN_INCLUDE_DIRS}) +target_compile_features(brpc_codegen_compile PRIVATE cxx_std_14) + +add_test(NAME flatbuffers_codegen_acceptance + COMMAND "${CMAKE_COMMAND}" + "-DGENERATOR=$" + "-DFLATC=${FLATC_EXECUTABLE}" + "-DSCHEMA=${CMAKE_CURRENT_SOURCE_DIR}/echo.fbs" + "-DWORK=${CMAKE_CURRENT_BINARY_DIR}/acceptance" + "-DCXX=${CMAKE_CXX_COMPILER}" + "-DINCLUDE_DIRS=${CODEGEN_INCLUDE_DIRS}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/acceptance.cmake") + +set(BRPC_CODEGEN_BRPC_LIBRARY "" CACHE FILEPATH + "Optional existing FlatBuffers-enabled libbrpc for runtime acceptance") +set(BRPC_CODEGEN_EXTRA_LIBRARIES "" CACHE STRING + "Additional link dependencies of the supplied libbrpc") +if(BRPC_CODEGEN_BRPC_LIBRARY) + find_package(Threads REQUIRED) + find_package(OpenSSL REQUIRED) + find_package(ZLIB REQUIRED) + find_library(CODEGEN_GFLAGS_LIBRARY gflags) + find_library(CODEGEN_LEVELDB_LIBRARY leveldb) + add_executable(brpc_codegen_runtime runtime.cpp $) + target_include_directories(brpc_codegen_runtime PRIVATE ${CODEGEN_INCLUDE_DIRS}) + target_compile_features(brpc_codegen_runtime PRIVATE cxx_std_14) + target_link_libraries(brpc_codegen_runtime PRIVATE + "${BRPC_CODEGEN_BRPC_LIBRARY}" ${Protobuf_LIBRARIES} + ${CODEGEN_GFLAGS_LIBRARY} ${CODEGEN_LEVELDB_LIBRARY} + OpenSSL::SSL OpenSSL::Crypto ZLIB::ZLIB Threads::Threads + ${CMAKE_DL_LIBS} ${BRPC_CODEGEN_EXTRA_LIBRARIES}) + add_test(NAME flatbuffers_codegen_runtime COMMAND brpc_codegen_runtime) +endif() diff --git a/test/flatbuffers_codegen/acceptance.cmake b/test/flatbuffers_codegen/acceptance.cmake new file mode 100644 index 0000000000..cbf09dfeb3 --- /dev/null +++ b/test/flatbuffers_codegen/acceptance.cmake @@ -0,0 +1,269 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +file(READ "${SCHEMA}" schema_text) +file(MAKE_DIRECTORY "${WORK}") + +function(expect_rejected name replacement) + set(directory "${WORK}/${name}") + file(MAKE_DIRECTORY "${directory}") + string(REPLACE "(id: 7)" "${replacement}" text "${schema_text}") + file(WRITE "${directory}/echo.fbs" "${text}") + execute_process(COMMAND "${GENERATOR}" -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if("${result}" STREQUAL "0" OR error STREQUAL "") + message(FATAL_ERROR "${name}: invalid schema was accepted or lacked diagnostic") + endif() + if(EXISTS "${directory}/echo.brpc.fb.h" OR EXISTS "${directory}/echo.brpc.fb.cpp") + message(FATAL_ERROR "${name}: invalid schema emitted service files") + endif() + message(STATUS "Rejected ${name}: ${error}") +endfunction() + +function(expect_rejected_rpc_name name) + set(directory "${WORK}/reserved_${name}") + file(MAKE_DIRECTORY "${directory}") + file(REMOVE "${directory}/echo.brpc.fb.h" "${directory}/echo.brpc.fb.cpp") + string(REPLACE "Inspect(" "${name}(" text "${schema_text}") + file(WRITE "${directory}/echo.fbs" "${text}") + execute_process(COMMAND "${GENERATOR}" -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if("${result}" STREQUAL "0") + message(SEND_ERROR "RPC ${name}: generator incorrectly accepted a fixed Stub field name") + return() + endif() + if(NOT error MATCHES "method name collides with generated API: ${name}") + message(FATAL_ERROR "RPC ${name}: expected a name-collision diagnostic: ${error}") + endif() + if(EXISTS "${directory}/echo.brpc.fb.h" OR EXISTS "${directory}/echo.brpc.fb.cpp") + message(FATAL_ERROR "RPC ${name}: rejected schema emitted service files") + endif() + message(STATUS "Rejected RPC ${name}: ${error}") +endfunction() + +function(expect_compiles name text) + set(directory "${WORK}/${name}") + file(MAKE_DIRECTORY "${directory}") + file(WRITE "${directory}/echo.fbs" "${text}") + execute_process(COMMAND "${FLATC}" --cpp -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: official flatc failed: ${error}") + endif() + execute_process(COMMAND "${GENERATOR}" -o "${directory}" "${directory}/echo.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: brpc_flatc failed: ${error}") + endif() + set(includes "-I${directory}") + foreach(include_dir IN LISTS INCLUDE_DIRS) + list(APPEND includes "-I${include_dir}") + endforeach() + execute_process(COMMAND "${CXX}" -std=c++14 ${includes} + -c "${directory}/echo.brpc.fb.cpp" -o "${directory}/echo.o" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: generated code did not compile: ${error}") + endif() + # The generated header must also compile without incidental prior includes. + file(WRITE "${directory}/header.cpp" "#include \"echo.brpc.fb.h\"\n") + execute_process(COMMAND "${CXX}" -std=c++14 ${includes} + -c "${directory}/header.cpp" -o "${directory}/header.o" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: generated header is not self-contained: ${error}") + endif() + foreach(suffix brpc.fb.h brpc.fb.cpp) + file(READ "${directory}/echo.${suffix}" generated) + if(NOT generated MATCHES "Licensed to the Apache Software Foundation") + message(FATAL_ERROR "${name}: ${suffix} lacks Apache license") + endif() + if(generated MATCHES "FLATBUFFERS_VERSION") + message(FATAL_ERROR "${name}: service output pins FlatBuffers version") + endif() + endforeach() + message(STATUS "Compiled ${name}") +endfunction() + +expect_rejected_rpc_name(channel_) +expect_rejected_rpc_name(owned_channel_) +expect_rejected(missing_id "") +expect_rejected(duplicate_id "(id: 41)") +expect_rejected(negative_id "(id: -1)") +expect_rejected(overflow_id "(id: 2147483648)") +expect_rejected(string_id "(id: \"7\")") +expect_rejected(streaming "(id: 7, streaming: \"server\")") +expect_compiles(sparse_ids "${schema_text}") + +set(shadow_names request response controller done method std BrpcFlatbuffersFail + STUB_OWNS_CHANNEL STUB_DOESNT_OWN_CHANNEL ChannelOwnership) +set(shadow_schema "namespace codegen.names;\ntable Request { text:string; }\ntable Response { text:string; }\nrpc_service Names {\n") +set(method_id 0) +foreach(name IN LISTS shadow_names) + math(EXPR method_id "${method_id} + 7") + string(APPEND shadow_schema " ${name}(Request):Response (id: ${method_id});\n") +endforeach() +string(APPEND shadow_schema "}\nrpc_service ChannelOwnership { Ping(Request):Response (id: 3); }\n") +expect_compiles(shadowed_names "${shadow_schema}") + +# Supply RUNTIME_LIBRARIES when invoking this script directly to exercise the +# new names against an existing FlatBuffers-enabled bRPC library as well. +if(RUNTIME_LIBRARIES) + set(directory "${WORK}/shadowed_names") + set(runtime_source [=[ +#include "echo.brpc.fb.h" +#include + +using ::brpc::flatbuffers::Message; +using ::google::protobuf::Closure; +using ::google::protobuf::RpcController; + +class Completion : public Closure { +public: + void Run() override { ++runs; } + int runs = 0; +}; + +class Implementation : public ::codegen::names::Names { +public: + int selected = 0; +]=]) + set(method_id 0) + foreach(name IN LISTS shadow_names) + math(EXPR method_id "${method_id} + 7") + string(APPEND runtime_source + " void ${name}(RpcController*, const Message*, Message*, Closure* completion) override { selected = ${method_id}; completion->Run(); }\n") + endforeach() + string(APPEND runtime_source [=[ +}; + +class LocalChannel : public ::brpc::flatbuffers::RpcChannel { +public: + explicit LocalChannel(::brpc::flatbuffers::Service* service) : service_(service) {} + void FBCallMethod(const ::brpc::flatbuffers::MethodDescriptor* method, + RpcController* controller, const Message* request, + Message* response, Closure* done) override { + service_->FBCallMethod(method, controller, request, response, done); + } +private: + ::brpc::flatbuffers::Service* service_; +}; + +int main() { + ::brpc::flatbuffers::MessageBuilder builder; + builder.Finish(::codegen::names::CreateRequest(builder)); + Message request = builder.ReleaseMessage(); + Message response; + Implementation implementation; + LocalChannel channel(&implementation); + ::codegen::names::Names::Stub stub(&channel); + ::codegen::names::Names defaults; + LocalChannel default_channel(&defaults); + ::codegen::names::Names::Stub default_stub(&default_channel); + ::codegen::names::Names::Stub disconnected(nullptr); + Completion completion; + int expected_runs = 0; +]=]) + set(method_id 0) + foreach(name IN LISTS shadow_names) + math(EXPR method_id "${method_id} + 7") + string(APPEND runtime_source + " stub.${name}(nullptr, &request, &response, &completion);\n" + " if (implementation.selected != ${method_id} || completion.runs != ++expected_runs) return 1;\n" + " default_stub.${name}(nullptr, &request, &response, &completion);\n" + " if (completion.runs != ++expected_runs) return 2;\n" + " disconnected.${name}(nullptr, &request, &response, &completion);\n" + " if (completion.runs != ++expected_runs) return 3;\n") + endforeach() + string(APPEND runtime_source [=[ + ::codegen::names::ChannelOwnership service; + LocalChannel service_channel(&service); + ::codegen::names::ChannelOwnership::Stub named_stub(&service_channel, + ::brpc::flatbuffers::Service::STUB_DOESNT_OWN_CHANNEL); + named_stub.Ping(nullptr, &request, &response, &completion); + if (completion.runs != ++expected_runs) return 4; + ::codegen::names::Names::Stub owned_stub(new LocalChannel(&implementation), + ::brpc::flatbuffers::Service::STUB_OWNS_CHANNEL); + owned_stub.request(nullptr, &request, &response, &completion); + if (implementation.selected != 7 || completion.runs != ++expected_runs) return 5; + std::cout << "Shadowed-name dispatch, failures, and constructors passed\n"; + return 0; +} +]=]) + file(WRITE "${directory}/runtime.cpp" "${runtime_source}") + set(includes "-I${directory}") + foreach(include_dir IN LISTS INCLUDE_DIRS) + list(APPEND includes "-I${include_dir}") + endforeach() + execute_process(COMMAND "${CXX}" -std=c++14 ${includes} + "${directory}/runtime.cpp" "${directory}/echo.o" ${RUNTIME_LIBRARIES} + -o "${directory}/runtime" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "Shadowed-name runtime link failed: ${error}") + endif() + execute_process(COMMAND "${directory}/runtime" + RESULT_VARIABLE result OUTPUT_VARIABLE output ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "Shadowed-name runtime failed: ${output}${error}") + endif() + message(STATUS "${output}") +endif() + +string(REPLACE + " Repeat(Request):Response (id: 41);\n Inspect(Request):Response (id: 7);" + " Inspect(Request):Response (id: 7);\n Repeat(Request):Response (id: 41);" + reordered "${schema_text}") +expect_compiles(reordered "${reordered}") +file(READ "${WORK}/reordered/echo.brpc.fb.cpp" reordered_source) +if(NOT reordered_source MATCHES "case 41:" OR NOT reordered_source MATCHES "case 7:") + message(FATAL_ERROR "Reordering changed wire IDs") +endif() +string(FIND "${reordered_source}" "void Echo_Stub::Repeat(" repeat_begin) +if(repeat_begin LESS 0) + message(FATAL_ERROR "Reordered stub lacks Repeat") +endif() +string(SUBSTRING "${reordered_source}" ${repeat_begin} -1 repeat_body) +string(FIND "${repeat_body}" "\n}\n" repeat_end) +string(SUBSTRING "${repeat_body}" 0 ${repeat_end} repeat_body) +if(NOT repeat_body MATCHES "method\\(1\\)") + message(FATAL_ERROR "Reordered stub does not use its dense method position") +endif() + +string(REPLACE "namespace codegen.example;" "" global_schema "${schema_text}") +expect_compiles(global_namespace "${global_schema}") +string(REPLACE "(id: 7)" "(id: 0)" zero_schema "${schema_text}") +expect_compiles(zero_id "${zero_schema}") +string(REPLACE "(id: 7)" "(id: 2147483647)" maximum_schema "${schema_text}") +expect_compiles(maximum_id "${maximum_schema}") + +# Include schemas are read through the official Parser and not re-emitted. +file(MAKE_DIRECTORY "${WORK}/included") +file(WRITE "${WORK}/included/types.fbs" + "namespace shared; table Input { note:string; } table Output { note:string; }\n" + "rpc_service Imported { Ping(Input):Output (id: 3); }\n") +execute_process(COMMAND "${FLATC}" --cpp -o "${WORK}/included" + "${WORK}/included/types.fbs" RESULT_VARIABLE result) +if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "Cannot generate included table definitions") +endif() +expect_compiles(included + "include \"types.fbs\"; namespace api; rpc_service Local { Call(shared.Input):shared.Output (id: 17); }") +file(READ "${WORK}/included/echo.brpc.fb.h" included_header) +if(included_header MATCHES "class Imported") + message(FATAL_ERROR "Included service was emitted twice") +endif() diff --git a/test/flatbuffers_codegen/echo.fbs b/test/flatbuffers_codegen/echo.fbs new file mode 100644 index 0000000000..d023205f30 --- /dev/null +++ b/test/flatbuffers_codegen/echo.fbs @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace codegen.example; + +table Request { + text:string; +} + +table Response { + text:string; +} + +rpc_service Echo { + Repeat(Request):Response (id: 41); + Inspect(Request):Response (id: 7); +} + +rpc_service Other { + Repeat(Request):Response (id: 41); +} + +root_type Request; diff --git a/test/flatbuffers_codegen/runtime.cpp b/test/flatbuffers_codegen/runtime.cpp new file mode 100644 index 0000000000..4b923939ba --- /dev/null +++ b/test/flatbuffers_codegen/runtime.cpp @@ -0,0 +1,308 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "echo.brpc.fb.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +#define CODEGEN_CHECK(condition) \ + do { \ + if (!(condition)) { \ + throw std::runtime_error("check failed: " #condition); \ + } \ + } while (0) + +using brpc::flatbuffers::Message; +using brpc::flatbuffers::MessageBuilder; +using brpc::flatbuffers::MethodDescriptor; +using brpc::flatbuffers::ServiceDescriptor; +using codegen::example::Echo; +using codegen::example::Other; +using codegen::example::Request; +using codegen::example::Response; +using google::protobuf::Closure; +using google::protobuf::RpcController; + +class Controller : public RpcController { +public: + void Reset() override { failures = 0; error.clear(); } + bool Failed() const override { return failures != 0; } + std::string ErrorText() const override { return error; } + void StartCancel() override {} + void SetFailed(const std::string& reason) override { ++failures; error = reason; } + bool IsCanceled() const override { return false; } + void NotifyOnCancel(Closure*) override {} + int failures = 0; + std::string error; +}; + +class Done : public Closure { +public: + void Run() override { ++runs; } + int runs = 0; +}; + +class DeletingDone : public Closure { +public: + explicit DeletingDone(int* runs) : runs_(runs) {} + void Run() override { ++*runs_; delete this; } +private: + int* runs_; +}; + +Message MakeRequest(const char* text) { + MessageBuilder builder; + const auto value = text ? builder.CreateString(text) : + ::flatbuffers::Offset<::flatbuffers::String>(); + builder.Finish(codegen::example::CreateRequest(builder, value)); + return builder.ReleaseMessage(); +} + +class Implementation : public Echo { +public: + void Repeat(RpcController*, const Message* request, + Message* response, Closure* done) override { + ++calls; + selected = 41; + Reply(request, response, done); + } + void Inspect(RpcController*, const Message* request, + Message* response, Closure* done) override { + ++calls; + selected = 7; + Reply(request, response, done); + } + void Reply(const Message* request, Message* response, Closure* done) { + const auto* input = request->GetRoot(); + MessageBuilder builder; + const auto value = input->text() ? builder.CreateString(input->text()->str()) : + ::flatbuffers::Offset<::flatbuffers::String>(); + builder.Finish(codegen::example::CreateResponse(builder, value)); + *response = builder.ReleaseMessage(); + if (done) { + done->Run(); + } + } + int calls = 0; + int selected = -1; +}; + +class LocalChannel : public brpc::flatbuffers::RpcChannel { +public: + explicit LocalChannel(Echo* service, int* destroyed = nullptr) + : service_(service), destroyed_(destroyed) {} + ~LocalChannel() override { if (destroyed_) { ++*destroyed_; } } + void FBCallMethod(const MethodDescriptor* method, RpcController* controller, + const Message* request, Message* response, Closure* done) override { + last_method = method; + service_->FBCallMethod(method, controller, request, response, done); + } + const MethodDescriptor* last_method = nullptr; +private: + Echo* service_; + int* destroyed_; +}; + +void TestConcurrentDescriptors() { + const int thread_count = 24; + std::atomic start(false); + std::atomic invalid(0); + std::vector descriptors(thread_count); + std::vector threads; + for (int i = 0; i < thread_count; ++i) { + threads.emplace_back([&, i] { + while (!start.load()) { + std::this_thread::yield(); + } + for (int n = 0; n < 1000; ++n) { + descriptors[i] = Echo::descriptor(); + if (descriptors[i]->method_count() != 2 || + descriptors[i]->method(0)->index() != 41 || + descriptors[i]->method(1)->index() != 7) { + ++invalid; + } + } + }); + } + start.store(true); + for (auto& thread : threads) { + thread.join(); + } + CODEGEN_CHECK(invalid.load() == 0); + for (const auto* descriptor : descriptors) { + CODEGEN_CHECK(descriptor == descriptors.front()); + } + const auto* descriptor = descriptors.front(); + CODEGEN_CHECK(descriptor->full_name() == "codegen.example.Echo"); + CODEGEN_CHECK(descriptor->FindMethodByIndex(41) == descriptor->method(0)); + CODEGEN_CHECK(descriptor->FindMethodByIndex(7) == descriptor->method(1)); + CODEGEN_CHECK(descriptor->FindMethodByIndex(0) == nullptr); + CODEGEN_CHECK(descriptor->method(0)->service() == descriptor); +} + +void ExpectFailure(const std::function& call) { + Controller controller; + Done done; + call(&controller, &done); + CODEGEN_CHECK(controller.Failed()); + CODEGEN_CHECK(controller.failures == 1); + CODEGEN_CHECK(!controller.ErrorText().empty()); + CODEGEN_CHECK(done.runs == 1); +} + +void TestDispatchAndErrors() { + Implementation service; + LocalChannel channel(&service); + Echo::Stub stub(&channel); + const auto* descriptor = Echo::descriptor(); + CODEGEN_CHECK(stub.GetDescriptor() == descriptor); + CODEGEN_CHECK(stub.channel() == &channel); + for (const char* text : {static_cast(nullptr), "hello"}) { + Message request = MakeRequest(text); + for (int position = 0; position < 2; ++position) { + Controller controller; + Done done; + Message response; + if (position == 0) { + stub.Repeat(&controller, &request, &response, &done); + } else { + stub.Inspect(&controller, &request, &response, &done); + } + CODEGEN_CHECK(!controller.Failed()); + CODEGEN_CHECK(done.runs == 1); + CODEGEN_CHECK(channel.last_method == descriptor->method(position)); + CODEGEN_CHECK(service.selected == (position == 0 ? 41 : 7)); + CODEGEN_CHECK(response.Verify()); + const auto* output = response.GetRoot(); + CODEGEN_CHECK(text ? output->text() && output->text()->str() == text : + output->text() == nullptr); + } + } + const int calls = service.calls; + Message request = MakeRequest(nullptr); + Message response; + Message invalid; + const MethodDescriptor unknown("Unknown", descriptor, 99); + const MethodDescriptor forged("Forged", descriptor, 41); + for (const auto* method : {static_cast(nullptr), + Other::descriptor()->method(0), &unknown, &forged}) { + ExpectFailure([&](Controller* controller, Done* done) { + service.FBCallMethod(method, controller, &request, &response, done); + }); + } + for (int position = 0; position < 2; ++position) { + ExpectFailure([&](Controller* controller, Done* done) { + service.FBCallMethod(descriptor->method(position), controller, + &invalid, &response, done); + }); + } + ExpectFailure([&](Controller* controller, Done* done) { + service.FBCallMethod(descriptor->method(0), controller, nullptr, &response, done); + }); + ExpectFailure([&](Controller* controller, Done* done) { + service.FBCallMethod(descriptor->method(0), controller, &request, nullptr, done); + }); + CODEGEN_CHECK(service.calls == calls); + + Echo unimplemented; + for (int position = 0; position < 2; ++position) { + ExpectFailure([&](Controller* controller, Done* done) { + unimplemented.FBCallMethod(descriptor->method(position), controller, + &request, &response, done); + }); + } + ExpectFailure([&](Controller* controller, Done* done) { + unimplemented.Repeat(controller, &request, &response, done); + }); + Echo::Stub disconnected(nullptr); + ExpectFailure([&](Controller* controller, Done* done) { + disconnected.Repeat(controller, &request, &response, done); + }); + ExpectFailure([&](Controller* controller, Done* done) { + disconnected.Inspect(controller, &request, &response, done); + }); + Controller controller; + service.FBCallMethod(nullptr, &controller, &request, &response, nullptr); + CODEGEN_CHECK(controller.failures == 1); + Done done; + service.FBCallMethod(nullptr, nullptr, &request, &response, &done); + CODEGEN_CHECK(done.runs == 1); + int deleted_runs = 0; + service.FBCallMethod(nullptr, nullptr, &request, &response, + new DeletingDone(&deleted_runs)); + CODEGEN_CHECK(deleted_runs == 1); +} + +void TestOwnershipAndAsyncCompletion() { + Implementation service; + int destroyed = 0; + { + Echo::Stub owner(new LocalChannel(&service, &destroyed), + brpc::flatbuffers::Service::STUB_OWNS_CHANNEL); + } + CODEGEN_CHECK(destroyed == 1); + { + LocalChannel borrowed(&service, &destroyed); + { + Echo::Stub stub(&borrowed); + } + CODEGEN_CHECK(destroyed == 1); + } + CODEGEN_CHECK(destroyed == 2); + + class Deferred : public Echo { + public: + void Repeat(RpcController*, const Message*, Message*, Closure* done) override { + pending = done; + } + Closure* pending = nullptr; + } deferred; + Message request = MakeRequest(nullptr); + Message response; + Controller controller; + Done done; + deferred.FBCallMethod(Echo::descriptor()->method(0), &controller, + &request, &response, &done); + CODEGEN_CHECK(!controller.Failed()); + CODEGEN_CHECK(done.runs == 0); + CODEGEN_CHECK(deferred.pending == &done); + deferred.pending->Run(); + CODEGEN_CHECK(done.runs == 1); +} + +} // namespace + +int main() { + try { + TestConcurrentDescriptors(); + TestDispatchAndErrors(); + TestOwnershipAndAsyncCompletion(); + std::cout << "Descriptor, sparse dispatch, verifier, callback and ownership checks passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/test/flatbuffers_message.fbs b/test/flatbuffers_message.fbs new file mode 100644 index 0000000000..0288499125 --- /dev/null +++ b/test/flatbuffers_message.fbs @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace brpc_fbtest; + +table Payload { + value:long; + message:string; + values:[int]; +} + +root_type Payload; diff --git a/tools/flatbuffers/CMakeLists.txt b/tools/flatbuffers/CMakeLists.txt new file mode 100644 index 0000000000..05179ac090 --- /dev/null +++ b/tools/flatbuffers/CMakeLists.txt @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cmake_minimum_required(VERSION 3.10) +project(brpc_flatc LANGUAGES CXX) + +find_path(FLATBUFFERS_INCLUDE_DIR flatbuffers/idl.h) +find_library(FLATBUFFERS_LIBRARY NAMES flatbuffers) +if(NOT FLATBUFFERS_INCLUDE_DIR OR NOT FLATBUFFERS_LIBRARY) + message(FATAL_ERROR "Install official FlatBuffers development headers and libflatbuffers") +endif() + +add_executable(brpc_flatc brpc_flatc.cpp) +target_compile_features(brpc_flatc PRIVATE cxx_std_11) +target_include_directories(brpc_flatc SYSTEM PRIVATE "${FLATBUFFERS_INCLUDE_DIR}") +target_link_libraries(brpc_flatc PRIVATE "${FLATBUFFERS_LIBRARY}") +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(brpc_flatc PRIVATE -Wall -Wextra -Wpedantic) +endif() + +include(CTest) +if(BUILD_TESTING) + add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../../test/flatbuffers_codegen" + "${CMAKE_CURRENT_BINARY_DIR}/tests") +endif() diff --git a/tools/flatbuffers/README.md b/tools/flatbuffers/README.md new file mode 100644 index 0000000000..b862490cea --- /dev/null +++ b/tools/flatbuffers/README.md @@ -0,0 +1,134 @@ +# Standalone FlatBuffers service generator + +`brpc_flatc` generates the bRPC **service abstraction**, not a network protocol or +an adapter for `brpc::Channel`. It uses the installed, official +`flatbuffers/idl.h` Parser and `libflatbuffers`; it does not download or modify a +FlatBuffers fork. The official `flatc --cpp` remains responsible for table types. + +## Build and generate + +The tool needs a C++11 compiler, CMake, and official FlatBuffers development +headers/library. To build only the generator: + +```sh +cmake -S tools/flatbuffers -B /tmp/brpc-codegen-build -DBUILD_TESTING=OFF +cmake --build /tmp/brpc-codegen-build -j2 +mkdir -p /tmp/brpc-generated +flatc --cpp -o /tmp/brpc-generated test/flatbuffers_codegen/echo.fbs +/tmp/brpc-codegen-build/brpc_flatc -o /tmp/brpc-generated test/flatbuffers_codegen/echo.fbs +``` + +The output directory must already exist. The command line is: + +```text +brpc_flatc [-I include_dir]... [-o existing_output_dir] schema.fbs +``` + +For `echo.fbs`, the two tools produce: + +- Official `flatc`: `echo_generated.h`. +- `brpc_flatc`: `echo.brpc.fb.h` and `echo.brpc.fb.cpp`, both Apache-licensed. + +Compile the generated `.cpp` into the application with C++14 or newer and bRPC +configured with FlatBuffers support. Keep the generated header, official table header, and bRPC +headers on the include path. The generated service files do not pin a +FlatBuffers version; use an official `flatc` matching your installed table +headers, as required by official generated code. + +Schema includes are resolved relative to the input file and through repeated +`-I` options (`-Idir` also works). Run official `flatc --cpp` for each included +schema to obtain its table header. Run `brpc_flatc` separately for each schema +that declares services; imported services are not emitted again. Pass the same +include directories to both tools. This tool processes one `.fbs` per invocation +and follows the official default `*_generated.h` naming convention. + +## Explicit, stable IDs + +```fbs +namespace example.api; + +table Request { text:string; } +table Response { text:string; } + +rpc_service Echo { + Repeat(Request):Response (id: 41); + Inspect(Request):Response (id: 7); +} +``` + +Every RPC method must have an explicit integer `(id: N)` in `[0, 2147483647]`. +IDs must be unique **within a service**. Missing, negative, duplicate, quoted, +or out-of-range IDs fail generation with a diagnostic and nonzero exit status. +Reordering declarations never changes their wire IDs. This does not require +explicit IDs on FlatBuffers table fields. + +The generated names are `example::api::Echo`, +`example::api::Echo_Stub`, and the alias `example::api::Echo::Stub`. +They are **not** the old `EchoStub` spelling. Methods take protobuf +`RpcController`/`Closure` and `brpc::flatbuffers::Message` parameters. Derive from +`Echo` and override the schema methods; supply an implementation of +`brpc::flatbuffers::RpcChannel` to a stub. Channel ownership defaults to borrowed; +`Service::STUB_OWNS_CHANNEL` transfers ownership to the stub. + +This intentionally bounded generator supports unary table RPCs, multiple +services/methods, namespaces, included request/response tables, and absent +optional strings. Streaming and C++ keyword names in service/type/namespace +positions are rejected rather than silently misgenerated. Method names that +collide with generated service APIs are also rejected. Schema file basenames +may contain ASCII letters, digits, underscores, dots, and hyphens. + +## Descriptor and completion contracts + +- `Echo::descriptor()` owns a `ServiceDescriptor` in a function-local static + RAII holder. C++11 initialization is thread-safe; the holder and its owned + method descriptors are destroyed normally at process exit. There is no + leaked singleton allocation or unsynchronized lazy initialization. +- The descriptor table contains the namespace prefix, service name, ordered + method names, and explicit IDs. Stubs use `method(position)`; service dispatch + switches on the stable `method->index()`. +- Service dispatch checks descriptor ownership and identity, non-null request + and response, and `request->Verify()` before calling user code. +- Invalid calls, null stub channels, and default unimplemented methods call + `controller->SetFailed()` when a controller exists, then run non-null `done` + exactly once. Null controllers and callbacks are tolerated on failure. +- Successful dispatch transfers completion responsibility to the user method. + The generated dispatcher does not run `done` again, so asynchronous service + implementations remain possible. User methods and channel implementations + must themselves honor the exactly-once completion contract. + +## Independent acceptance tests + +No root build file needs modification. The default standalone build enables +CTest and additionally needs official `flatc` and protobuf development headers: + +```sh +cmake -S tools/flatbuffers -B /tmp/brpc-codegen-build -DBUILD_TESTING=ON +cmake --build /tmp/brpc-codegen-build -j2 +ctest --test-dir /tmp/brpc-codegen-build --output-on-failure +``` + +The build compiles the generated service against the real bRPC headers. CTest +rejects missing/duplicate/negative/overflow/string IDs and streaming RPCs, then +compiles the service and self-contained header for sparse IDs, reordered +methods, global/nested namespaces, zero/max IDs, and included table types. Test +configuration headers and generated artifacts stay in the standalone build +directory, not in the source tree. + +To enable additional behavior tests against an **already built** +FlatBuffers-enabled bRPC library: + +```sh +cmake -S tools/flatbuffers -B /tmp/brpc-codegen-build \ + -DBUILD_TESTING=ON \ + -DBRPC_CODEGEN_BRPC_LIBRARY=/path/to/libbrpc.a +cmake --build /tmp/brpc-codegen-build -j2 +ctest --test-dir /tmp/brpc-codegen-build --output-on-failure +``` + +Use `BRPC_CODEGEN_EXTRA_LIBRARIES` for additional dependencies of a custom bRPC +build. The runtime test covers concurrent first descriptor access, sparse +stub/dispatch mapping, absent and present strings, invalid/foreign/forged +methods, failed request verification, null arguments, unimplemented methods, +exactly-once error callbacks (including self-deleting closures), deferred +completion, and channel ownership. It uses a local abstract channel; no network +RPC compatibility is claimed. diff --git a/tools/flatbuffers/brpc_flatc.cpp b/tools/flatbuffers/brpc_flatc.cpp new file mode 100644 index 0000000000..b8a9f11b8a --- /dev/null +++ b/tools/flatbuffers/brpc_flatc.cpp @@ -0,0 +1,486 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +const char kLicense[] = R"(// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Generated by brpc_flatc. Do not edit. + +)"; + +const char kArguments[] = + "::google::protobuf::RpcController* controller,\n" + " const ::brpc::flatbuffers::Message* request,\n" + " ::brpc::flatbuffers::Message* response,\n" + " ::google::protobuf::Closure* done"; + +struct Service { + const flatbuffers::ServiceDef* definition; + std::vector ids; +}; + +std::string JoinNamespace(const flatbuffers::Definition& definition, + const std::string& separator) { + std::string result; + if (definition.defined_namespace) { + for (const auto& component : definition.defined_namespace->components) { + if (!result.empty()) { + result += separator; + } + result += component; + } + } + return result; +} + +std::string Qualified(const flatbuffers::Definition& definition) { + const std::string ns = JoinNamespace(definition, "::"); + return "::" + (ns.empty() ? "" : ns + "::") + definition.name; +} + +bool IsCppIdentifier(const std::string& name) { + static const std::set keywords = { + "alignas", "alignof", "and", "and_eq", "asm", "auto", "bitand", + "bitor", "bool", "break", "case", "catch", "char", "char16_t", + "char32_t", "class", "compl", "concept", "const", "const_cast", + "consteval", "constexpr", "constinit", "continue", "co_await", + "co_return", "co_yield", "decltype", "default", "delete", "do", + "double", "dynamic_cast", "else", "enum", "explicit", "export", + "extern", "false", "float", "for", "friend", "goto", "if", + "inline", "int", "long", "mutable", "namespace", "new", + "noexcept", "not", "not_eq", "nullptr", "operator", "or", "or_eq", + "private", "protected", "public", "register", "reinterpret_cast", + "requires", "return", "short", "signed", "sizeof", "static", + "static_assert", "static_cast", "struct", "switch", "template", + "this", "thread_local", "throw", "true", "try", "typedef", + "typeid", "typename", "union", "unsigned", "using", "virtual", + "void", "volatile", "wchar_t", "while", "xor", "xor_eq" + }; + return !name.empty() && keywords.count(name) == 0; +} + +bool ValidateName(const flatbuffers::Definition& definition, + std::string* error) { + if (!IsCppIdentifier(definition.name)) { + *error = "C++ keyword is not supported: " + definition.name; + return false; + } + if (definition.defined_namespace) { + for (const auto& component : definition.defined_namespace->components) { + if (!IsCppIdentifier(component)) { + *error = "C++ keyword namespace is not supported: " + component; + return false; + } + } + } + return true; +} + +bool ParseId(const flatbuffers::Value& value, int32_t* id) { + if (value.constant.empty() || !flatbuffers::IsInteger(value.type.base_type)) { + return false; + } + int64_t number = 0; + for (char c : value.constant) { + if (c < '0' || c > '9') { + return false; + } + number = number * 10 + (c - '0'); + if (number > std::numeric_limits::max()) { + return false; + } + } + *id = static_cast(number); + return true; +} + +bool CollectServices(const flatbuffers::Parser& parser, + std::vector* services, std::string* error) { + for (const auto* definition : parser.services_.vec) { + // Included schemas are generated separately, just as with flatc --cpp. + if (definition->generated) { + continue; + } + if (!ValidateName(*definition, error)) { + return false; + } + Service service = {definition, {}}; + std::set ids; + if (definition->calls.vec.empty()) { + *error = "rpc_service must contain at least one method: " + + definition->name; + return false; + } + for (const auto* call : definition->calls.vec) { + if (!ValidateName(*call, error) || + !ValidateName(*call->request, error) || + !ValidateName(*call->response, error)) { + return false; + } + if (call->name == definition->name || + call->name == definition->name + "_Stub" || + call->name == "descriptor" || call->name == "GetDescriptor" || + call->name == "FBCallMethod" || call->name == "Stub" || + call->name == "channel" || call->name == "channel_" || + call->name == "owned_channel_") { + *error = "method name collides with generated API: " + call->name; + return false; + } + const auto* attribute = call->attributes.Lookup("id"); + int32_t id = 0; + if (!attribute || !ParseId(*attribute, &id)) { + *error = definition->name + "." + call->name + + " requires an explicit nonnegative int32 (id: N)"; + return false; + } + if (!ids.insert(id).second) { + *error = "duplicate method id " + std::to_string(id) + + " in " + definition->name; + return false; + } + if (call->attributes.Lookup("streaming")) { + *error = "streaming RPC is not supported: " + call->name; + return false; + } + service.ids.push_back(id); + } + services->push_back(service); + } + if (services->empty()) { + *error = "input schema contains no rpc_service to generate"; + return false; + } + return true; +} + +void OpenNamespace(const flatbuffers::Definition& definition, + std::ostream& out) { + if (definition.defined_namespace) { + for (const auto& component : definition.defined_namespace->components) { + out << "namespace " << component << " {\n"; + } + } + out << '\n'; +} + +void CloseNamespace(const flatbuffers::Definition& definition, + std::ostream& out) { + if (definition.defined_namespace) { + const auto& components = definition.defined_namespace->components; + for (auto it = components.rbegin(); it != components.rend(); ++it) { + out << "} // namespace " << *it << '\n'; + } + } + out << '\n'; +} + +void GenerateHeader(const Service& service, std::ostream& out) { + const auto& definition = *service.definition; + const std::string& name = definition.name; + OpenNamespace(definition, out); + out << "class " << name << "_Stub;\n\n" + << "class " << name << " : public ::brpc::flatbuffers::Service {\n" + << "public:\n" + << " typedef " << name << "_Stub Stub;\n" + << " static const ::brpc::flatbuffers::ServiceDescriptor* descriptor();\n" + << " const ::brpc::flatbuffers::ServiceDescriptor* GetDescriptor() override;\n" + << " void FBCallMethod(\n" + << " const ::brpc::flatbuffers::MethodDescriptor* method,\n" + << " " << kArguments << ") override;\n"; + for (const auto* call : definition.calls.vec) { + out << " virtual void " << call->name << "(\n" + << " " << kArguments << ");\n"; + } + out << "};\n\n" + << "class " << name << "_Stub : public " << name << " {\n" + << "public:\n" + << " explicit " << name << "_Stub(\n" + << " ::brpc::flatbuffers::RpcChannel* channel,\n" + << " ::brpc::flatbuffers::Service::ChannelOwnership ownership =\n" + << " ::brpc::flatbuffers::Service::STUB_DOESNT_OWN_CHANNEL);\n" + << " ::brpc::flatbuffers::RpcChannel* channel() const { return channel_; }\n"; + for (const auto* call : definition.calls.vec) { + out << " void " << call->name << "(\n" + << " " << kArguments << ") override;\n"; + } + out << "\nprivate:\n" + << " ::brpc::flatbuffers::RpcChannel* channel_;\n" + << " ::std::unique_ptr<::brpc::flatbuffers::RpcChannel> owned_channel_;\n" + << "};\n\n"; + CloseNamespace(definition, out); +} + +void GenerateSource(const Service& service, std::ostream& out) { + const auto& definition = *service.definition; + const std::string& name = definition.name; + OpenNamespace(definition, out); + out << "const ::brpc::flatbuffers::ServiceDescriptor* " << name + << "::descriptor() {\n" + << " struct Holder {\n" + << " ::brpc::flatbuffers::ServiceDescriptor value;\n" + << " Holder() {\n" + << " const ::brpc::flatbuffers::BrpcDescriptorTable table = {\n" + << " \"" << JoinNamespace(definition, ".") << "\", \"" + << name << "\",\n \""; + for (size_t i = 0; i < definition.calls.vec.size(); ++i) { + out << (i == 0 ? "" : " ") << definition.calls.vec[i]->name; + } + out << "\", {"; + for (size_t i = 0; i < service.ids.size(); ++i) { + out << (i == 0 ? "" : ", ") << service.ids[i]; + } + out << "}\n };\n" + << " if (value.init(table) != 0) {\n" + << " throw ::std::runtime_error(\"invalid generated service descriptor\");\n" + << " }\n" + << " }\n" + << " };\n" + << " static const Holder holder;\n" + << " return &holder.value;\n" + << "}\n\n" + << "const ::brpc::flatbuffers::ServiceDescriptor* " << name + << "::GetDescriptor() {\n" + << " return descriptor();\n" + << "}\n\n" + << "void " << name << "::FBCallMethod(\n" + << " const ::brpc::flatbuffers::MethodDescriptor* method,\n" + << " " << kArguments << ") {\n" + << " if (!method || method->service() != descriptor() ||\n" + << " descriptor()->FindMethodByIndex(method->index()) != method) {\n" + << " ::BrpcFlatbuffersFail(controller, done, \"invalid service method\");\n" + << " return;\n" + << " }\n" + << " if (!request || !response) {\n" + << " ::BrpcFlatbuffersFail(controller, done, \"null request or response\");\n" + << " return;\n" + << " }\n" + << " switch (method->index()) {\n"; + for (size_t i = 0; i < definition.calls.vec.size(); ++i) { + const auto& call = *definition.calls.vec[i]; + out << " case " << service.ids[i] << ":\n" + << " if (!request->Verify<" << Qualified(*call.request) + << ">()) {\n" + << " ::BrpcFlatbuffersFail(controller, done, \"invalid " + << call.request->name << " request\");\n" + << " return;\n" + << " }\n" + << " this->" << call.name << "(controller, request, response, done);\n" + << " return;\n"; + } + out << " default:\n" + << " ::BrpcFlatbuffersFail(controller, done, \"unknown method id\");\n" + << " return;\n" + << " }\n" + << "}\n\n"; + for (const auto* call : definition.calls.vec) { + out << "void " << name << "::" << call->name << "(\n" + << " " << kArguments << ") {\n" + << " (void)request;\n" + << " (void)response;\n" + << " ::BrpcFlatbuffersFail(controller, done, \"method not implemented: " + << name << "." << call->name << "\");\n" + << "}\n\n"; + } + out << name << "_Stub::" << name << "_Stub(\n" + << " ::brpc::flatbuffers::RpcChannel* channel,\n" + << " ::brpc::flatbuffers::Service::ChannelOwnership ownership)\n" + << " : channel_(channel),\n" + << " owned_channel_(ownership == ::brpc::flatbuffers::Service::STUB_OWNS_CHANNEL\n" + << " ? channel : nullptr) {}\n\n"; + for (size_t i = 0; i < definition.calls.vec.size(); ++i) { + out << "void " << name << "_Stub::" << definition.calls.vec[i]->name + << "(\n " << kArguments << ") {\n" + << " if (!channel_) {\n" + << " ::BrpcFlatbuffersFail(controller, done, \"null RPC channel\");\n" + << " return;\n" + << " }\n" + << " channel_->FBCallMethod(descriptor()->method(" << i + << "), controller, request, response, done);\n" + << "}\n\n"; + } + CloseNamespace(definition, out); +} + +bool WriteFile(const std::string& path, const std::string& contents) { + std::ofstream stream(path.c_str(), std::ios::binary | std::ios::trunc); + stream << contents; + stream.close(); + return !stream.fail(); +} + +void Usage(std::ostream& out) { + out << "Usage: brpc_flatc [-I include_dir]... [-o existing_output_dir] schema.fbs\n" + << "Run official flatc --cpp separately to produce schema_generated.h.\n"; +} + +int Run(int argc, char** argv) { + std::string input; + std::string output_dir = "."; + std::vector include_dirs; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "--help" || arg == "-h") { + Usage(std::cout); + return 0; + } + if (arg == "-o" || arg == "-I") { + if (++i == argc) { + Usage(std::cerr); + return 1; + } + if (arg == "-o") { + output_dir = argv[i]; + } else { + include_dirs.push_back(argv[i]); + } + } else if (arg.compare(0, 2, "-I") == 0 && arg.size() > 2) { + include_dirs.push_back(arg.substr(2)); + } else if (arg.empty() || arg[0] == '-' || !input.empty()) { + Usage(std::cerr); + return 1; + } else { + input = arg; + } + } + if (input.size() < 5 || input.substr(input.size() - 4) != ".fbs" || + output_dir.empty()) { + Usage(std::cerr); + return 1; + } + const size_t slash = input.find_last_of("/\\"); + const std::string basename = input.substr(slash == std::string::npos ? 0 : slash + 1); + const std::string stem = basename.substr(0, basename.size() - 4); + if (stem.empty() || stem.find_first_not_of( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.-") != + std::string::npos) { + std::cerr << "brpc_flatc: unsupported schema filename\n"; + return 1; + } + include_dirs.push_back(slash == std::string::npos ? "." : + input.substr(0, slash == 0 ? 1 : slash)); + std::vector include_paths; + for (const auto& directory : include_dirs) { + include_paths.push_back(directory.c_str()); + } + include_paths.push_back(nullptr); + std::ifstream stream(input.c_str(), std::ios::binary); + if (!stream) { + std::cerr << "brpc_flatc: cannot read " << input << '\n'; + return 1; + } + const std::string schema((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); + if (stream.bad() || schema.find('\0') != std::string::npos) { + std::cerr << "brpc_flatc: invalid schema input\n"; + return 1; + } + flatbuffers::Parser parser; + if (!parser.Parse(schema.c_str(), include_paths.data(), input.c_str())) { + std::cerr << parser.error_ << '\n'; + return 1; + } + std::vector services; + std::string error; + if (!CollectServices(parser, &services, &error)) { + std::cerr << "brpc_flatc: " << error << '\n'; + return 1; + } + std::string guard = "BRPC_FLATBUFFERS_GENERATED_"; + for (char c : stem) { + guard += c >= 'a' && c <= 'z' ? static_cast(c - 'a' + 'A') : + ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ? c : '_'); + } + guard += "_H_"; + std::ostringstream header; + header << kLicense << "#ifndef " << guard << "\n#define " << guard << "\n\n" + << "#include \n" + << "#include \n" + << "#include \"brpc/flatbuffers/message.h\"\n" + << "#include \"brpc/flatbuffers/service.h\"\n" + << "#include \"" << stem << "_generated.h\"\n\n" + << "#if !BRPC_WITH_FLATBUFFERS\n" + << "#error \"Generated services require BRPC_WITH_FLATBUFFERS\"\n" + << "#endif\n\n"; + std::ostringstream source; + source << kLicense << "#include \"" << stem << ".brpc.fb.h\"\n\n" + << "#include \n\n" + << "namespace {\n" + << "void BrpcFlatbuffersFail(::google::protobuf::RpcController* controller,\n" + << " ::google::protobuf::Closure* done,\n" + << " const char* reason) {\n" + << " if (controller) {\n" + << " controller->SetFailed(reason);\n" + << " }\n" + << " if (done) {\n" + << " done->Run();\n" + << " }\n" + << "}\n" + << "} // namespace\n\n"; + for (const auto& service : services) { + GenerateHeader(service, header); + GenerateSource(service, source); + } + header << "#endif // " << guard << '\n'; + if (output_dir.back() != '/') { + output_dir += '/'; + } + if (!WriteFile(output_dir + stem + ".brpc.fb.h", header.str()) || + !WriteFile(output_dir + stem + ".brpc.fb.cpp", source.str())) { + std::cerr << "brpc_flatc: cannot write output in " << output_dir << '\n'; + return 1; + } + return 0; +} + +} // namespace + +int main(int argc, char** argv) { + try { + return Run(argc, argv); + } catch (const std::exception& error) { + std::cerr << "brpc_flatc: " << error.what() << '\n'; + return 1; + } +} From 6e5a3b3248fc10671873497fc56574eb4ef8b5c9 Mon Sep 17 00:00:00 2001 From: xulei25 Date: Mon, 21 Sep 2026 15:00:39 +0800 Subject: [PATCH 2/3] Fix FlatBuffers generation and acceptance Keep service header guards distinct across punctuation, case and equal schema basenames in different namespaces without embedding checkout paths. Test standalone headers, both include orders and relocation. Select C++14 or C++17 according to Protobuf and use its exported include paths and link target for standalone acceptance. Rebuild and pass both codegen acceptance tests before committing. Build on message construction in 3f2550c6 and RPC support in 338fe839, continuing apache/brpc#3196 and apache/brpc#3197 over the SingleIOBuf foundation from apache/brpc#3062. --- test/flatbuffers_codegen/CMakeLists.txt | 32 +++++++-- test/flatbuffers_codegen/acceptance.cmake | 86 ++++++++++++++++++++++- tools/flatbuffers/brpc_flatc.cpp | 21 ++++-- 3 files changed, 126 insertions(+), 13 deletions(-) diff --git a/test/flatbuffers_codegen/CMakeLists.txt b/test/flatbuffers_codegen/CMakeLists.txt index 86b8cb694e..971197d843 100644 --- a/test/flatbuffers_codegen/CMakeLists.txt +++ b/test/flatbuffers_codegen/CMakeLists.txt @@ -15,7 +15,20 @@ # specific language governing permissions and limitations # under the License. -find_package(Protobuf REQUIRED) +set(protobuf_MODULE_COMPATIBLE ON CACHE BOOL "Expose Protobuf compatibility variables for codegen tests" FORCE) +find_package(Protobuf CONFIG QUIET) +set(CODEGEN_PROTOBUF_CONFIG ${Protobuf_FOUND}) +if(NOT Protobuf_FOUND) + find_package(Protobuf REQUIRED) +endif() +set(CODEGEN_CXX_STANDARD 14) +if(Protobuf_VERSION VERSION_GREATER 4.21) + set(CODEGEN_CXX_STANDARD 17) + if(NOT CODEGEN_PROTOBUF_CONFIG) + message(FATAL_ERROR "Protobuf 5+ needs its CMake config package for codegen tests") + endif() +endif() +message(STATUS "Codegen tests: Protobuf ${Protobuf_VERSION}, C++${CODEGEN_CXX_STANDARD}") find_program(FLATC_EXECUTABLE flatc) if(NOT FLATC_EXECUTABLE) message(FATAL_ERROR "Codegen tests need the official flatc executable") @@ -33,6 +46,16 @@ set(CODEGEN_INCLUDE_DIRS "${GENERATED_DIR}" "${FLATBUFFERS_INCLUDE_DIR}" ${Protobuf_INCLUDE_DIRS}) +get_target_property(CODEGEN_PROTOBUF_INCLUDE_DIRS protobuf::libprotobuf INTERFACE_INCLUDE_DIRECTORIES) +if(CODEGEN_PROTOBUF_INCLUDE_DIRS) + list(APPEND CODEGEN_INCLUDE_DIRS ${CODEGEN_PROTOBUF_INCLUDE_DIRS}) +endif() +if(TARGET absl::base) + get_target_property(CODEGEN_ABSL_INCLUDE_DIRS absl::base INTERFACE_INCLUDE_DIRECTORIES) + if(CODEGEN_ABSL_INCLUDE_DIRS) + list(APPEND CODEGEN_INCLUDE_DIRS ${CODEGEN_ABSL_INCLUDE_DIRS}) + endif() +endif() add_custom_command( OUTPUT "${GENERATED_DIR}/echo.brpc.fb.cpp" "${GENERATED_DIR}/echo.brpc.fb.h" @@ -44,7 +67,7 @@ add_custom_command( VERBATIM) add_library(brpc_codegen_compile OBJECT "${GENERATED_DIR}/echo.brpc.fb.cpp") target_include_directories(brpc_codegen_compile PRIVATE ${CODEGEN_INCLUDE_DIRS}) -target_compile_features(brpc_codegen_compile PRIVATE cxx_std_14) +target_compile_features(brpc_codegen_compile PRIVATE cxx_std_${CODEGEN_CXX_STANDARD}) add_test(NAME flatbuffers_codegen_acceptance COMMAND "${CMAKE_COMMAND}" @@ -53,6 +76,7 @@ add_test(NAME flatbuffers_codegen_acceptance "-DSCHEMA=${CMAKE_CURRENT_SOURCE_DIR}/echo.fbs" "-DWORK=${CMAKE_CURRENT_BINARY_DIR}/acceptance" "-DCXX=${CMAKE_CXX_COMPILER}" + "-DCXX_STANDARD=${CODEGEN_CXX_STANDARD}" "-DINCLUDE_DIRS=${CODEGEN_INCLUDE_DIRS}" -P "${CMAKE_CURRENT_SOURCE_DIR}/acceptance.cmake") @@ -68,9 +92,9 @@ if(BRPC_CODEGEN_BRPC_LIBRARY) find_library(CODEGEN_LEVELDB_LIBRARY leveldb) add_executable(brpc_codegen_runtime runtime.cpp $) target_include_directories(brpc_codegen_runtime PRIVATE ${CODEGEN_INCLUDE_DIRS}) - target_compile_features(brpc_codegen_runtime PRIVATE cxx_std_14) + target_compile_features(brpc_codegen_runtime PRIVATE cxx_std_${CODEGEN_CXX_STANDARD}) target_link_libraries(brpc_codegen_runtime PRIVATE - "${BRPC_CODEGEN_BRPC_LIBRARY}" ${Protobuf_LIBRARIES} + "${BRPC_CODEGEN_BRPC_LIBRARY}" protobuf::libprotobuf ${CODEGEN_GFLAGS_LIBRARY} ${CODEGEN_LEVELDB_LIBRARY} OpenSSL::SSL OpenSSL::Crypto ZLIB::ZLIB Threads::Threads ${CMAKE_DL_LIBS} ${BRPC_CODEGEN_EXTRA_LIBRARIES}) diff --git a/test/flatbuffers_codegen/acceptance.cmake b/test/flatbuffers_codegen/acceptance.cmake index cbf09dfeb3..118b271755 100644 --- a/test/flatbuffers_codegen/acceptance.cmake +++ b/test/flatbuffers_codegen/acceptance.cmake @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. +if(NOT DEFINED CXX_STANDARD) + set(CXX_STANDARD 14) +endif() file(READ "${SCHEMA}" schema_text) file(MAKE_DIRECTORY "${WORK}") @@ -73,7 +76,7 @@ function(expect_compiles name text) foreach(include_dir IN LISTS INCLUDE_DIRS) list(APPEND includes "-I${include_dir}") endforeach() - execute_process(COMMAND "${CXX}" -std=c++14 ${includes} + execute_process(COMMAND "${CXX}" "-std=c++${CXX_STANDARD}" ${includes} -c "${directory}/echo.brpc.fb.cpp" -o "${directory}/echo.o" RESULT_VARIABLE result ERROR_VARIABLE error) if(NOT "${result}" STREQUAL "0") @@ -81,7 +84,7 @@ function(expect_compiles name text) endif() # The generated header must also compile without incidental prior includes. file(WRITE "${directory}/header.cpp" "#include \"echo.brpc.fb.h\"\n") - execute_process(COMMAND "${CXX}" -std=c++14 ${includes} + execute_process(COMMAND "${CXX}" "-std=c++${CXX_STANDARD}" ${includes} -c "${directory}/header.cpp" -o "${directory}/header.o" RESULT_VARIABLE result ERROR_VARIABLE error) if(NOT "${result}" STREQUAL "0") @@ -99,6 +102,83 @@ function(expect_compiles name text) message(STATUS "Compiled ${name}") endfunction() +function(expect_distinct_headers name first_stem second_stem) + set(directory "${WORK}/header_guards_${name}") + set(includes) + foreach(include_dir IN LISTS INCLUDE_DIRS) + list(APPEND includes "-I${include_dir}") + endforeach() + foreach(side first second) + set(stem "${${side}_stem}") + set(output "${directory}/${side}") + file(MAKE_DIRECTORY "${output}") + string(REPLACE "namespace codegen.example;" "namespace guard_${name}.${side};" + text "${schema_text}") + if(side STREQUAL "first") + set(first_text "${text}") + endif() + file(WRITE "${output}/${stem}.fbs" "${text}") + execute_process(COMMAND "${FLATC}" --cpp -o "${output}" "${output}/${stem}.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}/${side}: official flatc failed: ${error}") + endif() + execute_process(COMMAND "${GENERATOR}" -o "${output}" "${output}/${stem}.fbs" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}/${side}: brpc_flatc failed: ${error}") + endif() + file(WRITE "${output}/single.cpp" + "#include \"${stem}.brpc.fb.h\"\n::guard_${name}::${side}::Echo* service = nullptr;\n") + execute_process(COMMAND "${CXX}" "-std=c++${CXX_STANDARD}" ${includes} + -c "${output}/single.cpp" -o "${output}/single.o" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}/${side}: standalone header failed: ${error}") + endif() + endforeach() + foreach(reverse FALSE TRUE) + set(first "#include \"first/${first_stem}.brpc.fb.h\"\n") + set(second "#include \"second/${second_stem}.brpc.fb.h\"\n") + if(reverse) + set(headers "${second}${first}") + else() + set(headers "${first}${second}") + endif() + file(WRITE "${directory}/combined_${reverse}.cpp" + "${headers}::guard_${name}::first::Echo* first_echo = nullptr;\n::guard_${name}::second::Echo* second_echo = nullptr;\n") + execute_process(COMMAND "${CXX}" "-std=c++${CXX_STANDARD}" ${includes} + -c "${directory}/combined_${reverse}.cpp" + -o "${directory}/combined_${reverse}.o" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: combined headers (reverse=${reverse}) failed: ${error}") + endif() + endforeach() + # A checkout/output directory change must not alter generated identifiers. + set(relocated "${directory}/relocated") + file(MAKE_DIRECTORY "${relocated}") + file(WRITE "${relocated}/${first_stem}.fbs" "${first_text}") + execute_process(COMMAND "${GENERATOR}" -o . "${first_stem}.fbs" + WORKING_DIRECTORY "${relocated}" + RESULT_VARIABLE result ERROR_VARIABLE error) + if(NOT "${result}" STREQUAL "0") + message(FATAL_ERROR "${name}: relocated generation failed: ${error}") + endif() + foreach(suffix brpc.fb.h brpc.fb.cpp) + file(READ "${directory}/first/${first_stem}.${suffix}" original) + file(READ "${relocated}/${first_stem}.${suffix}" regenerated) + if(NOT "${original}" STREQUAL "${regenerated}") + message(FATAL_ERROR "${name}: ${suffix} depends on checkout/output paths") + endif() + endforeach() + message(STATUS "Distinct, relocatable header guards: ${name}") +endfunction() + +expect_distinct_headers(punctuation foo-bar foo_bar) +expect_distinct_headers(letter_case FooBar foobar) +expect_distinct_headers(same_basename echo echo) + expect_rejected_rpc_name(channel_) expect_rejected_rpc_name(owned_channel_) expect_rejected(missing_id "") @@ -209,7 +289,7 @@ int main() { foreach(include_dir IN LISTS INCLUDE_DIRS) list(APPEND includes "-I${include_dir}") endforeach() - execute_process(COMMAND "${CXX}" -std=c++14 ${includes} + execute_process(COMMAND "${CXX}" "-std=c++${CXX_STANDARD}" ${includes} "${directory}/runtime.cpp" "${directory}/echo.o" ${RUNTIME_LIBRARIES} -o "${directory}/runtime" RESULT_VARIABLE result ERROR_VARIABLE error) diff --git a/tools/flatbuffers/brpc_flatc.cpp b/tools/flatbuffers/brpc_flatc.cpp index b8a9f11b8a..70c5da9b7c 100644 --- a/tools/flatbuffers/brpc_flatc.cpp +++ b/tools/flatbuffers/brpc_flatc.cpp @@ -80,6 +80,16 @@ std::string Qualified(const flatbuffers::Definition& definition) { return "::" + (ns.empty() ? "" : ns + "::") + definition.name; } +std::string GuardComponent(const std::string& value) { + static const char digits[] = "0123456789ABCDEF"; + std::string result; + for (unsigned char c : value) { + result += digits[c >> 4]; + result += digits[c & 15]; + } + return result; +} + bool IsCppIdentifier(const std::string& name) { static const std::set keywords = { "alignas", "alignof", "and", "and_eq", "asm", "auto", "bitand", @@ -427,12 +437,11 @@ int Run(int argc, char** argv) { std::cerr << "brpc_flatc: " << error << '\n'; return 1; } - std::string guard = "BRPC_FLATBUFFERS_GENERATED_"; - for (char c : stem) { - guard += c >= 'a' && c <= 'z' ? static_cast(c - 'a' + 'A') : - ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ? c : '_'); - } - guard += "_H_"; + // Coexisting service headers cannot define the same qualified service. + // Preserve filename bytes and that service identity, without tying output + // to checkout paths or folding punctuation and case into the same guard. + const std::string guard = "BRPC_FLATBUFFERS_GENERATED_" + GuardComponent(stem) + + "_" + GuardComponent(Qualified(*services.front().definition)) + "_H_"; std::ostringstream header; header << kLicense << "#ifndef " << guard << "\n#define " << guard << "\n\n" << "#include \n" From 332a1e159b4e929eccf8beb5f0b7be038e1ff0b0 Mon Sep 17 00:00:00 2001 From: xulei25 Date: Thu, 24 Sep 2026 11:41:07 +0800 Subject: [PATCH 3/3] Document FlatBuffers messages in Chinese Add a Chinese version of the FlatBuffers message and codegen guide for the focused message PR. Explain why the Bzlmod path keeps the checksum-pinned FlatBuffers archive instead of bazel_dep: the BCR module imports gRPC and language/tooling dependencies while bRPC only needs runtime_cc and flatc. --- MODULE.bazel | 8 +-- docs/cn/flatbuffers.md | 108 +++++++++++++++++++++++++++++++++++++++++ docs/en/flatbuffers.md | 2 + 3 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 docs/cn/flatbuffers.md diff --git a/MODULE.bazel b/MODULE.bazel index 4f72f9b07b..abe905655d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -80,9 +80,11 @@ git_repository( commit = '564ee727a55523d4351a8fb3c94292b388ebb924', # v26.06.0_CAM ) -# runtime_cc and flatc do not need FlatBuffers' gRPC module dependency, which -# would otherwise conflict with brpc's BoringSSL version even when disabled. -# Keep the archive and checksum in sync with WORKSPACE. +# Do not use `bazel_dep(name = 'flatbuffers', version = '25.2.10')` here. +# The BCR module pulls gRPC and JS/Go/Swift rule dependencies for FlatBuffers' +# full upstream build. bRPC only needs runtime_cc and flatc, and gRPC would +# otherwise introduce another BoringSSL version even when BRPC_WITH_FLATBUFFERS +# is false. Keep this archive and checksum in sync with WORKSPACE. flatbuffers_http_archive = use_repo_rule( '@bazel_tools//tools/build_defs/repo:http.bzl', 'http_archive', diff --git a/docs/cn/flatbuffers.md b/docs/cn/flatbuffers.md new file mode 100644 index 0000000000..75ac9e819c --- /dev/null +++ b/docs/cn/flatbuffers.md @@ -0,0 +1,108 @@ +# FlatBuffers 消息 + +[English version](../en/flatbuffers.md) + +bRPC 提供可选的、基于 IOBuf 的 FlatBuffers 消息、构造器和服务描述符。 +消息构造方案基于 [apache/brpc#3196](https://github.com/apache/brpc/pull/3196)。 + +该组件不会注册 `fb_rpc` 传输协议，也不会向 `brpc::Channel` 和 `brpc::Server` +增加 FlatBuffers 集成。服务代码生成和进程内分派测试不是网络 RPC，也不是性能 benchmark。 + +## 构建 + +FlatBuffers 支持默认关闭。消息运行库只需要 FlatBuffers 头文件，不链接 +FlatBuffers 库。测试需要与头文件版本匹配的 `flatc`。请保留上游生成代码中的 +版本断言；版本不一致时应重新生成头文件，而不是削弱断言。 + +例如，GoogleTest 源码安装在 `/usr/src/googletest` 时： + +```sh +cmake -S . -B build -DWITH_FLATBUFFERS=ON -DBUILD_UNIT_TESTS=ON \ + -DBUILD_BRPC_TOOLS=OFF -DDOWNLOAD_GTEST=OFF \ + -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest +cmake --build build --target brpc_flatbuffers_unittest -j6 +ctest --test-dir build -R '^brpc_flatbuffers_unittest$' --output-on-failure +``` + +其他安装方式可按需设置 `FLATBUFFERS_INCLUDE_DIR`、 +`FLATBUFFERS_FLATC_EXECUTABLE` 和 `BRPC_SYSTEM_GTEST_SOURCE_DIR`。 +项目原有测试依赖仍然适用。 + +Make 可在 `config_brpc.sh` 中传入 `--with-flatbuffers`；测试可通过 +`FLATC=/path/to/flatc` 指定官方生成器。Bazel 可传入 +`--define=BRPC_WITH_FLATBUFFERS=true`，并使用匹配的 FlatBuffers 25.2.10 +运行时和编译器依赖。Bzlmod 导入与 WORKSPACE 相同的 checksum 固定归档： +`runtime_cc` 和 `flatc` 不需要 FlatBuffers 的外部 gRPC 模块，否则即使该功能关闭， +也可能与 bRPC 固定的 BoringSSL 版本冲突。 + +公共头文件位于 `brpc/flatbuffers/`，命名空间为 `brpc::flatbuffers`。 +构造消息包含 `message.h`，使用服务描述符和接口包含 `service.h`。 +`butil/config.h` 中的 `BRPC_WITH_FLATBUFFERS` 始终为 0 或 1；应用应使用 +`#if` 判断，而不是 `#ifdef`。 + +Flatc 2.0.x 会生成未全限定的 `flatbuffers::` 名称。业务 schema 应使用 +`brpc` 之外的 namespace（例如 `myapp.rpc`），避免被 `brpc::flatbuffers` +遮蔽；不要依赖 include 顺序。Flatc 25.2.10 会生成全限定名称。 + +## 消息构造和所有权 + +使用上游 `flatc --cpp` 生成 schema 对应的 `*_generated.h`。将 +`brpc::flatbuffers::MessageBuilder` 传给生成的 `Create...` 函数，调用 +`Finish(root)`，最后调用 `ReleaseMessage()`。 + +* `ReleaseMessage()` 不拷贝 payload 字节。返回的 move-only Message 拥有 IOBuf + block 引用，并且在 builder 复用或析构后仍然有效。 +* Message 和 builder 移动后，源对象仍可复用。移动带 shared string 的 builder 会丢弃 + 其可选去重缓存；已经生成的 offset 仍有效。 +* 导入普通 `::flatbuffers::FlatBufferBuilder` 会复制其 payload 和 scratch，并保留 + 未完成 table 的状态。原始 allocator 负责释放原有存储，包括其拥有的自定义 allocator。 + 实现不会猜测该用 `free` 还是 `delete[]`，也不会假设 payload 前存在额外空间。 +* 使用 MessageBuilder 自身的 move、swap 和 release 操作。不要通过基类 cast 转移它， + 也不要使用继承来的 raw-buffer release 操作：原始 FlatBuffers detached buffer 会保留 + 指向成员 allocator 的地址。 +* released payload 前有 64 字节零初始化空间。可用 `reduce_meta_size_and_get_buf` + 缩短这段空间；增长会被拒绝且不修改对象。缩短 metadata 不会改变 payload 地址或字节。 +* 序列化接受 const Message，并在输出 IOBuf 中保留其存储引用。该 buffer 是共享的， + 不是 copy-on-write：当其他读者或已序列化 buffer 仍在使用时，不要修改 payload/metadata。 +* 分配大小在收窄为 SingleIOBuf 的 uint32_t 长度前会先检查。分配失败在 release build + 中同样 fatal，不受 bRPC `crash_on_fatal_log` 设置影响。这些路径会显式 abort， + 而不是依赖 `CHECK`/`LOG(FATAL)`。上游 `vector_downward` 无法在 null allocation 后安全继续。 + +`ParseFbFromIOBUF` 检查长度和 framing，并保留独立所有权。当 payload 地址 64 字节对齐时， +它会共享连续输入；输入分片或对齐不足时会复制到对齐存储。builder 分配为 64 字节对齐， +但最终 payload 只需满足 schema 要求的对齐，因此并非所有本地消息都适合接收侧零拷贝。 +不支持超过 64 字节的对齐要求。 + +**Framing 不是 schema 验证。** 对不可信对端收到的数据，应先调用 +`msg.Verify()`，再调用 `GetRoot()` 或 +`GetMutableRoot()`。合法消息中的可选 FlatBuffers string/vector 仍可能为 null。 +framing 检查失败会保持原消息不变。 + +## Service ID 和代码生成 + +`BrpcDescriptorTable` 包含 namespace、service 名称、以空白分隔的方法名，以及显式方法 ID。 +ID 必须是唯一的非负 int32。空 ID 列表会为手写 descriptor 分配 ordinal ID；生成服务必须 +使用显式 ID： + +```fbs +rpc_service BenchmarkService { + First(Request):Response (id: 2); + Second(Request):Response (id: 5); +} +``` + +* `descriptor.method(position)` 按声明顺序枚举方法。 +* `method.index()` 是稳定的 wire ID，不是数组下标。 +* `descriptor.FindMethodByIndex(id)` 用于查找稀疏 wire ID。传输层必须用该查找，不能用 + wire ID 直接索引稠密数组。 +* 不要把已删除的方法 ID 复用于另一个方法。删除或重排声明不应改变仍存在方法的显式 ID。 +* namespace `a.b` 和 `a.b.` 会规范化为相同 service 名称；空 namespace 表示全局作用域。 + 方法全名包含 service。service hash 使用规范化后的全名和 MurmurHash3 seed 1。 + 当这些 ID 被持久化或在线路上传输时，应保持 service 名称稳定。 +* Descriptor 成功初始化后不能重新初始化。它们通过 RAII 拥有 method；生成的 accessor + 使用函数局部静态初始化以保证线程安全。 + +`tools/flatbuffers/` 中的配套生成器使用上游 parser 生成服务绑定。它独立构建；只有该 +可选工具需要 `libflatbuffers`。命令和限制见其 [README](../../tools/flatbuffers/README.md)。 +生成的 dispatch 会校验请求、拒绝未知或外来方法，并在失败时执行非空 completion callback， +包括未实现方法。成功实现拥有 completion，并且必须恰好执行一次回调。 diff --git a/docs/en/flatbuffers.md b/docs/en/flatbuffers.md index 7ff86accbe..92c48da23e 100644 --- a/docs/en/flatbuffers.md +++ b/docs/en/flatbuffers.md @@ -1,5 +1,7 @@ # FlatBuffers messages +[中文版](../cn/flatbuffers.md) + bRPC provides optional IOBuf-backed FlatBuffers messages, builders, and service descriptors. The message-construction approach builds on [apache/brpc#3196](https://github.com/apache/brpc/pull/3196).