Add optional FlatBuffers RPC support

.github/workflows/flatbuffers-on.yml

@@ -0,0 +1,86 @@
+name: FlatBuffers ON
+
+on:
+  push:
+    branches: [master]
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  build-test:
+    name: FB ON (${{ matrix.os }}, ${{ matrix.system }}, ${{ matrix.compiler }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+      matrix:
+        include:
+          - {os: ubuntu-22.04, system: cmake, compiler: gcc, cc: gcc, cxx: g++, cache: ccache}
+          - {os: ubuntu-22.04, system: cmake, compiler: clang, cc: clang, cxx: clang++, cache: ccache}
+          - {os: ubuntu-22.04, system: make, compiler: gcc, cc: gcc, cxx: g++, cache: ccache}
+          - {os: ubuntu-22.04, system: make, compiler: clang, cc: clang, cxx: clang++, cache: ccache}
+          - {os: ubuntu-22.04, system: bazel, compiler: gcc, cc: gcc, cxx: g++, cache: bazel}
+          - {os: macos-latest, system: cmake, compiler: apple-clang, cc: clang, cxx: clang++, cache: ccache}
+    env:
+      CC: ${{ matrix.cc }}
+      CXX: ${{ matrix.cxx }}
+      FB_WORK: ${{ runner.temp }}/brpc-fb-${{ matrix.system }}-${{ matrix.compiler }}
+      BUILD_SYSTEM: ${{ matrix.system }}
+    steps:
+      - uses: actions/checkout@v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Validate the gate's report checks
+        run: python3 .github/scripts/test_flatbuffers_on.py
+
+      - name: Install focused Linux dependencies
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential cmake ninja-build clang ccache pkg-config python3 curl ca-certificates git \
+            libssl-dev libgflags-dev libprotobuf-dev libprotoc-dev protobuf-compiler \
+            libleveldb-dev libgoogle-perftools-dev zlib1g-dev
+
+      - name: Install focused macOS dependencies
+        if: runner.os == 'macOS'
+        run: brew install cmake ninja pkg-config openssl@3 gflags leveldb protobuf@29 abseil
+
+      - uses: ./.github/actions/setup-build-cache
+        with:
+          kind: ${{ matrix.cache }}
+          cache-key: fb-on-${{ matrix.system }}-${{ matrix.compiler }}
+
+      - name: Build and run the FB ON gates
+        shell: bash
+        run: |
+          args=(--build-system "$BUILD_SYSTEM" --work "$FB_WORK" --jobs 2)
+          if [[ "$RUNNER_OS" == macOS ]]; then
+            for package in protobuf@29 abseil openssl@3 gflags leveldb; do
+              args+=(--dependency-prefix "$(brew --prefix "$package")")
+            done
+            args+=(--protoc "$(brew --prefix protobuf@29)/bin/protoc")
+          fi
+          python3 .github/scripts/flatbuffers-on.py "${args[@]}"
+
+      - name: Upload commands, logs and test reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: flatbuffers-on-${{ matrix.os }}-${{ matrix.system }}-${{ matrix.compiler }}
+          path: ${{ env.FB_WORK }}/evidence/
+          if-no-files-found: warn
+          retention-days: 14

.gitignore

@@ -17,8 +17,10 @@
 /test/output
 build/
 
-# Ignore hidden files
+# Ignore hidden files, but keep GitHub workflows and helpers discoverable.
 .*
+!/.github/
+/.github/scripts/__pycache__/
 *.swp
 
 # Ignore auto-generated files

BUILD.bazel

@@ -20,6 +20,12 @@ licenses(["notice"])  # Apache v2
 
 exports_files(["LICENSE"])
 
+config_setting(
+    name = "brpc_with_flatbuffers",
+    define_values = {"BRPC_WITH_FLATBUFFERS": "true"},
+    visibility = ["//visibility:public"],
+)
+
 COPTS = [
     "-fno-omit-frame-pointer",
 ] + select({
@@ -45,6 +51,9 @@ DEFINES = [
  }) + select({
      "//bazel/config:brpc_with_thrift": ["ENABLE_THRIFT_FRAMED_PROTOCOL=1"],
      "//conditions:default": [],
+ }) + select({
+     ":brpc_with_flatbuffers": ["BRPC_WITH_FLATBUFFERS=1"],
+     "//conditions:default": ["BRPC_WITH_FLATBUFFERS=0"],
  }) + select({
      "//bazel/config:brpc_with_thrift_legacy_version": [],
      "//conditions:default": ["THRIFT_STDCXX=std"],
@@ -125,6 +134,14 @@ genrule(
               "//conditions:default": "0",
           }) +
           """
+#ifdef BRPC_WITH_FLATBUFFERS
+#undef BRPC_WITH_FLATBUFFERS
+#endif
+#define BRPC_WITH_FLATBUFFERS """ + select({
+              ":brpc_with_flatbuffers": "1",
+              "//conditions:default": "0",
+          }) +
+          """
 #ifdef BUTIL_USE_CPU_FREQUENCY
 #undef BUTIL_USE_CPU_FREQUENCY
 #endif
@@ -539,6 +556,11 @@ brpc_proto_library(
     visibility = ["//visibility:public"],
 )
 
+FLATBUFFERS_SRC_PATTERNS = [
+    "src/brpc/flatbuffers/*.cpp",
+    "src/brpc/policy/flatbuffers_protocol.cpp",
+]
+
 URMA_SRC_PATTERNS = [
     "src/brpc/urma/*.cpp",
     "src/brpc/urma/**/*.cpp",
@@ -562,7 +584,7 @@ BRPC_BASE_SRCS = glob(
         "src/brpc/policy/thrift_protocol.cpp",
         "src/brpc/event_dispatcher_epoll.cpp",
         "src/brpc/event_dispatcher_kqueue.cpp",
-    ] + URMA_SRC_PATTERNS,
+    ] + URMA_SRC_PATTERNS + FLATBUFFERS_SRC_PATTERNS,
 )
 
 cc_library(
@@ -573,6 +595,9 @@ cc_library(
             "src/brpc/**/thrift*.cpp",
         ]),
         "//conditions:default": [],
+    }) + select({
+        ":brpc_with_flatbuffers": glob(FLATBUFFERS_SRC_PATTERNS),
+        "//conditions:default": [],
     }) + select({
         "//bazel/config:brpc_with_urma_use_real": URMA_SRCS,
         "//bazel/config:brpc_with_urma": URMA_SRCS + [
@@ -612,6 +637,9 @@ cc_library(
             "@org_apache_thrift//:thrift",
         ],
         "//conditions:default": [],
+    }) + select({
+        ":brpc_with_flatbuffers": ["@com_github_google_flatbuffers//:runtime_cc"],
+        "//conditions:default": [],
     }),
 )
 

CMakeLists.txt

@@ -23,6 +23,7 @@ option(WITH_MESALINK "With MesaLink" OFF)
 option(WITH_BORINGSSL "With BoringSSL" OFF)
 option(WITH_DEBUG_SYMBOLS "With debug symbols" ON)
 option(WITH_THRIFT "With thrift framed protocol supported" OFF)
+option(WITH_FLATBUFFERS "With FlatBuffers message and RPC support (headers only)" OFF)
 option(WITH_BTHREAD_TRACER "With bthread tracer supported" OFF)
 option(WITH_SNAPPY "With snappy" OFF)
 option(WITH_RDMA "With RDMA" OFF)
@@ -82,6 +83,17 @@ if(WITH_GLOG)
     set(BRPC_WITH_GLOG 1)
 endif()
 
+set(WITH_FLATBUFFERS_VAL "0")
+if(WITH_FLATBUFFERS)
+    find_path(FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h)
+    if(NOT FLATBUFFERS_INCLUDE_DIR)
+        message(FATAL_ERROR
+            "WITH_FLATBUFFERS requires FlatBuffers headers; set FLATBUFFERS_INCLUDE_DIR.")
+    endif()
+    set(WITH_FLATBUFFERS_VAL "1")
+    list(APPEND BRPC_COMMON_INCLUDE_DIRS ${FLATBUFFERS_INCLUDE_DIR})
+endif()
+
 set(WITH_CPU_FREQUENCY_VAL "0")
 if(WITH_CPU_FREQUENCY)
     set(WITH_CPU_FREQUENCY_VAL "1")
@@ -177,6 +189,7 @@ endif()
 
 list(APPEND BRPC_COMMON_DEFINITIONS
     BRPC_WITH_GLOG=${WITH_GLOG_VAL}
+    BRPC_WITH_FLATBUFFERS=${WITH_FLATBUFFERS_VAL}
     BRPC_WITH_RDMA=${WITH_RDMA_VAL}
     BRPC_WITH_URMA=${WITH_URMA_VAL}
     BRPC_WITH_UBRING=${WITH_UBRING_VAL}
@@ -670,6 +683,10 @@ file(GLOB_RECURSE BTHREAD_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/b
 file(GLOB_RECURSE JSON2PB_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/json2pb/*.cpp")
 file(GLOB_RECURSE BRPC_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/brpc/*.cpp")
 file(GLOB_RECURSE THRIFT_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/brpc/thrift*.cpp")
+if(NOT WITH_FLATBUFFERS)
+    list(FILTER BRPC_SOURCES EXCLUDE REGEX "/brpc/flatbuffers/.*\\.cpp$")
+    list(REMOVE_ITEM BRPC_SOURCES "${PROJECT_SOURCE_DIR}/src/brpc/policy/flatbuffers_protocol.cpp")
+endif()
 file(GLOB_RECURSE EXCLUDE_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/brpc/event_dispatcher_*.cpp")
 
 # When building with the real liburma, exclude the link-time mock so its urma_*

MODULE.bazel

@@ -79,3 +79,17 @@ git_repository(
     remote = 'https://atomgit.com/openeuler/umdk.git',
     commit = '564ee727a55523d4351a8fb3c94292b388ebb924',  # v26.06.0_CAM
 )
+
+# runtime_cc and flatc do not need FlatBuffers' gRPC module dependency, which
+# would otherwise conflict with brpc's BoringSSL version even when disabled.
+# Keep the archive and checksum in sync with WORKSPACE.
+flatbuffers_http_archive = use_repo_rule(
+    '@bazel_tools//tools/build_defs/repo:http.bzl',
+    'http_archive',
+)
+flatbuffers_http_archive(
+    name = 'com_github_google_flatbuffers',
+    sha256 = 'b9c2df49707c57a48fc0923d52b8c73beb72d675f9d44b2211e4569be40a7421',
+    strip_prefix = 'flatbuffers-25.2.10',
+    urls = ['https://github.com/google/flatbuffers/archive/refs/tags/v25.2.10.tar.gz'],
+)

Makefile

@@ -204,6 +204,9 @@ JSON2PB_SOURCES = $(foreach d,$(JSON2PB_DIRS),$(wildcard $(addprefix $(d)/*,$(SR
 JSON2PB_OBJS = $(addsuffix .o, $(basename $(JSON2PB_SOURCES))) 
 
 BRPC_DIRS = src/brpc src/brpc/details src/brpc/builtin src/brpc/policy src/brpc/policy/mysql src/brpc/rdma
+ifeq ($(WITH_FLATBUFFERS),1)
+BRPC_DIRS += src/brpc/flatbuffers
+endif
 ifeq ($(WITH_URMA),1)
 BRPC_DIRS += src/brpc/urma
 endif
@@ -213,6 +216,9 @@ BRPC_SOURCES_ALL = $(foreach d,$(BRPC_DIRS),$(wildcard $(addprefix $(d)/*,$(SRCE
 ifeq ($(URMA_USE_MOCK),0)
 BRPC_SOURCES_ALL := $(filter-out src/brpc/urma/mock_urma.cpp,$(BRPC_SOURCES_ALL))
 endif
+ifneq ($(WITH_FLATBUFFERS),1)
+BRPC_SOURCES_ALL := $(filter-out src/brpc/policy/flatbuffers_protocol.cpp,$(BRPC_SOURCES_ALL))
+endif
 BRPC_SOURCES = $(filter-out $(THRIFT_SOURCES) $(EXCLUDE_SOURCES), $(BRPC_SOURCES_ALL))
 BRPC_PROTOS = $(filter %.proto,$(BRPC_SOURCES))
 BRPC_CFAMILIES = $(filter-out %.proto %.pb.cc,$(BRPC_SOURCES))

README.md

@@ -31,6 +31,7 @@ You can use it to:
 * Read [overview](docs/en/overview.md) to know where bRPC can be used and its advantages.
 * Read [getting started](docs/en/getting_started.md) for building steps and play with [examples](https://github.com/apache/brpc/tree/master/example/).
 * Docs:
+  * [Enable FlatBuffers RPC and verify the example](docs/en/flatbuffers.md)
   * [Performance benchmark](docs/en/benchmark.md)
   * [bvar](docs/en/bvar.md)
     * [bvar_c++](docs/en/bvar_c++.md)

README_cn.md

@@ -32,6 +32,7 @@
 * 通过[概述](docs/cn/overview.md)了解哪里可以用bRPC及其优势。
 * 阅读[编译步骤](docs/cn/getting_started.md)了解如何开始使用, 之后可以运行一下[示例程序](https://github.com/apache/brpc/tree/master/example/).
 * 文档:
+  * [启用 FlatBuffers RPC 并通过 example 验证](docs/cn/flatbuffers.md)
   * [性能测试](docs/cn/benchmark.md)
   * [bvar](docs/cn/bvar.md)
     * [bvar_c++](docs/cn/bvar_c++.md)

WORKSPACE

@@ -150,6 +150,14 @@ http_archive(
     urls = ["https://github.com/google/crc32c/archive/1.1.2.tar.gz"],
 )
 
+# Optional FlatBuffers support uses runtime_cc; keep this version in sync with MODULE.bazel.
+http_archive(
+    name = "com_github_google_flatbuffers",
+    integrity = "sha256-ucLfSXB8V6SPwJI9UrjHO+ty1nX51EsiEeRWm+QKdCE=",
+    strip_prefix = "flatbuffers-25.2.10",
+    urls = ["https://github.com/google/flatbuffers/archive/refs/tags/v25.2.10.tar.gz"],
+)
+
 http_archive(
     name = "com_github_google_glog",  # 2021-05-07T23:06:39Z
     patch_args = ["-p1"],

config.h.in

@@ -21,6 +21,11 @@
 #endif
 #cmakedefine BRPC_WITH_GLOG @WITH_GLOG_VAL@
 
+#ifdef BRPC_WITH_FLATBUFFERS
+#undef BRPC_WITH_FLATBUFFERS
+#endif
+#define BRPC_WITH_FLATBUFFERS @WITH_FLATBUFFERS_VAL@
+
 #ifdef BUTIL_USE_CPU_FREQUENCY
 #undef BUTIL_USE_CPU_FREQUENCY
 #endif

config_brpc.sh

@@ -54,9 +54,10 @@ else
     LDD=ldd
 fi
 
-TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-rdma,with-urma,with-urma-mock,without-urma-mock,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"`
+TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-flatbuffers,with-rdma,with-urma,with-urma-mock,without-urma-mock,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"`
 WITH_GLOG=0
 WITH_THRIFT=0
+WITH_FLATBUFFERS=0
 WITH_RDMA=0
 WITH_URMA=0
 URMA_MOCK_MODE=auto
@@ -91,6 +92,7 @@ while true; do
         --cxx ) CXX=$2; shift 2 ;;
         --with-glog ) WITH_GLOG=1; shift 1 ;;
         --with-thrift) WITH_THRIFT=1; shift 1 ;;
+        --with-flatbuffers) WITH_FLATBUFFERS=1; shift 1 ;;
         --with-rdma) WITH_RDMA=1; shift 1 ;;
         --with-urma) WITH_URMA=1; shift 1 ;;
         --with-urma-mock) URMA_MOCK_MODE=on; shift 1 ;;
@@ -479,14 +481,15 @@ append_to_output "HDRS=$($ECHO $HDRS)"
 append_to_output "LIBS=$($ECHO $LIBS)"
 append_to_output "PROTOC=$PROTOC"
 append_to_output "PROTOBUF_HDR=$PROTOBUF_HDR"
+append_to_output "WITH_FLATBUFFERS=$WITH_FLATBUFFERS"
 append_to_output "CC=$CC"
 append_to_output "CXX=$CXX"
 append_to_output "GCC_VERSION=$GCC_VERSION"
 append_to_output "STATIC_LINKINGS=$STATIC_LINKINGS"
 append_to_output "DYNAMIC_LINKINGS=$DYNAMIC_LINKINGS"
 
 # CPP means C PreProcessing, not C PlusPlus
-CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK -DBUTIL_USE_CPU_FREQUENCY=$WITH_CPU_FREQUENCY"
+CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_WITH_FLATBUFFERS=$WITH_FLATBUFFERS -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK -DBUTIL_USE_CPU_FREQUENCY=$WITH_CPU_FREQUENCY"
 
 # Avoid over-optimizations of TLS variables by GCC>=4.8
 # See: https://github.com/apache/brpc/issues/1693
@@ -506,6 +509,12 @@ if [ "$SYSTEM" = "Darwin" ]; then
     fi
 fi
 
+if [ $WITH_FLATBUFFERS != 0 ]; then
+    FLATBUFFERS_HDR=$(find_dir_of_header_or_die flatbuffers/flatbuffers.h) || exit 1
+    append_to_output_headers "$FLATBUFFERS_HDR"
+    print_success "Found FlatBuffers headers: $FLATBUFFERS_HDR"
+fi
+
 if [ $WITH_THRIFT != 0 ]; then
     THRIFT_LIB=$(find_dir_of_lib_or_die thriftnb)
     THRIFT_HDR=$(find_dir_of_header_or_die thrift/Thrift.h)
@@ -692,6 +701,11 @@ cat << EOF > src/butil/config.h
 #endif
 #define BRPC_WITH_GLOG $WITH_GLOG
 
+#ifdef BRPC_WITH_FLATBUFFERS
+#undef BRPC_WITH_FLATBUFFERS
+#endif
+#define BRPC_WITH_FLATBUFFERS $WITH_FLATBUFFERS
+
 #ifdef BUTIL_USE_CPU_FREQUENCY
 #undef BUTIL_USE_CPU_FREQUENCY
 #endif
@@ -714,6 +728,7 @@ print_info "C++ std:   $CXXFLAGS"
 print_info "System:    $SYSTEM"
 if [ $WITH_GLOG -ne 0 ]; then print_info "With glog: yes"; fi
 if [ $WITH_THRIFT -ne 0 ]; then print_info "With thrift: yes"; fi
+if [ $WITH_FLATBUFFERS -ne 0 ]; then print_info "With FlatBuffers: yes (headers only)"; fi
 if [ $WITH_RDMA -ne 0 ]; then print_info "With RDMA: yes"; fi
 if [ $WITH_URMA -ne 0 ]; then print_info "With URMA: yes"; fi
 if [ $WITH_MESALINK -ne 0 ]; then print_info "With MesaLink: yes"; fi