Cherry pick commits from main to REL_2_STABLE, Sep'26

.github/workflows/apache-rat-audit.yml

@@ -55,6 +55,8 @@ jobs:
         uses: actions/checkout@v7
         with:
           fetch-depth: 1
+          # The release tarball includes submodules; audit the same tree.
+          submodules: recursive
 
       - name: Set up Java and Maven
         uses: actions/setup-java@v5
@@ -115,7 +117,7 @@ jobs:
       - name: Check for binary files
         run: |
           echo "Checking for binary files..."
-          echo "Checking extensions: class, jar, tar, tgz, zip, exe, dll, so, gz, bz2"
+          echo "Checking extensions: class, jar, tar, tgz, zip, exe, dll, so, gz, bz2, deb, rpm, gppkg, bin"
           echo "----------------------------------------------------------------------"
 
           # Binary file allowlist, see README.apache.md
@@ -127,10 +129,12 @@ jobs:
             "src/bin/gpfdist/regress/data/gpfdist2/gz_multi_chunk_2.tbl.gz"
             "src/bin/gpfdist/regress/data/gpfdist2/lineitem.tbl.bz2"
             "src/bin/gpfdist/regress/data/gpfdist2/lineitem.tbl.gz"
+            "gpMgmt/test/behave/mgmt_utils/steps/data/sample.gppkg"
+            "src/bin/pgevent/MSG00001.bin"
           )
 
           # Check for specific binary file extensions
-          binary_extensions="class jar tar tgz zip exe dll so gz bz2"
+          binary_extensions="class jar tar tgz zip exe dll so gz bz2 deb rpm gppkg bin"
           echo "BINARY_EXTENSIONS=${binary_extensions}" >> $GITHUB_ENV
           binary_results=""
           binaryfiles_found=false

.gitignore

@@ -73,4 +73,9 @@ lib*.pc
 /compile_commands.json
 /tmp_install/
 /.cache/
-/install/
+/install/
+/portlock/
+
+# Windows Explorer thumbnail caches
+Thumbs.db
+ehthumbs.db

.gitmodules

@@ -11,9 +11,6 @@
 [submodule "contrib/pax_storage/src/cpp/contrib/googlebench"]
 	path = contrib/pax_storage/src/cpp/contrib/googlebench
 	url = https://github.com/google/benchmark.git
-[submodule "dependency/yyjson"]
-	path = dependency/yyjson
-	url = https://github.com/ibireme/yyjson.git
 
 [submodule "gpcontrib/yezzey"]
 	path = gpcontrib/yezzey

LICENSE

@@ -213,13 +213,6 @@ The PostgreSQL software includes:
       src/bin/pg_controldata/pg_controldata.c
       copyright (c) Oliver Elphick <olly@lfix.co.uk>, 2001
 
-----------------------------
-   BSD-4-Clause (University of California-Specific)
-
-      src/include/port/win32_msvc/glob.h
-      src/port/glob.c
-      see licenses/LICENSE-glob.txt
-
 ----------------------------
    BSD 3 Clause License
 
@@ -235,7 +228,8 @@ The PostgreSQL software includes:
       src/backend/utils/mb/wstrncmp.c
       see licenses/LICENSE-bsd3-wstrncmp.txt
 
-      src/backend/libpq/sha2.h
+      src/common/sha2.c
+      src/common/sha2_int.h
       see licenses/LICENSE-bsd3-sha2.txt
 
 ----------------------------
@@ -251,7 +245,7 @@ The PostgreSQL software includes:
 ----------------------------
    Perl Artistic License 2.0 (exception)
 
-      src/pl/plperl/ppport.h: 
+      src/pl/plperl/ppport.h
 
       Pursuant to https://issues.apache.org/jira/browse/LEGAL-79 and
       PL/Perl's use of a generated header file, we declare this file to
@@ -334,6 +328,27 @@ The Greenplum Database software includes:
       gpcontrib/gpcloud/test/googletest
       see licenses/LICENSE-googletest.txt
 
+      gpcontrib/pg_hint_plan/*
+      see licenses/LICENSE-pg-hint-plan.txt
+
+----------------------------
+   Internet Systems Consortium/Internet Software Consortium (ISC) LICENSE
+
+      gpMgmt/bin/lib/pexpect/__init__.py
+      gpMgmt/bin/lib/pexpect/pxssh.py
+      see licenses/LICENSE-pexpect.txt
+
+----------------------------
+   MIT LICENSE
+
+      gpcontrib/gpcloud/lib/http_parser.cpp
+      gpcontrib/gpcloud/lib/http_parser.h
+      see licenses/LICENSE-http-parser.txt
+
+      gpcontrib/gpcloud/lib/ini.cpp
+      gpcontrib/gpcloud/lib/ini.h
+      see licenses/LICENSE-ini.txt
+
 ----------------------------
    PostgreSQL License
 
@@ -373,9 +388,6 @@ Apache Cloudberry includes codes from
 ----------------------------
    MIT LICENSE
 
-      dependency/yyjson
-      see licenses/LICENSE-yyjson.txt
-
       contrib/pax_storage/src/cpp/cpp-stub
       see licenses/LICENSE-cpp-stub.txt
 

NOTICE

@@ -2,4 +2,11 @@ Apache Cloudberry (Incubating)
 Copyright 2024-2026 The Apache Software Foundation
 
 This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
+The Apache Software Foundation (http://www.apache.org/).
+
+---------------------------------------------------------------------------
+
+This product includes software originally developed by VMware.
+
+Greenplum Database
+Copyright (c) 2004-2020 VMware, Inc. or its affiliates. All Rights Reserved.

README.apache.md

@@ -50,3 +50,31 @@ The following compressed files are included in the source tree. These files are
 - src/bin/gpfdist/regress/data/gpfdist2/gz_multi_chunk_2.tbl.gz
 - src/bin/gpfdist/regress/data/gpfdist2/lineitem.tbl.bz2
 - src/bin/gpfdist/regress/data/gpfdist2/lineitem.tbl.gz
+
+## Binary Files in Source
+
+A source release should not carry compiled artifacts, so the licence audit
+workflow fails on binary file extensions unless the file is allowlisted. The
+following are allowed, with the reason for each.
+
+- gpMgmt/test/behave/mgmt_utils/steps/data/sample.gppkg
+
+  A small sample package that the `gppkg` behave suite installs and removes.
+  The tests exercise package handling itself, so the fixture has to be a real
+  package.
+
+- src/bin/pgevent/MSG00001.bin
+
+  Inherited from PostgreSQL, where it is also shipped. It is the output of the
+  Microsoft Message Compiler and is referenced from `pgmsgevent.rc` when
+  building the Windows event log DLL; `src/bin/pgevent/README` describes how it
+  is produced. Cloudberry does not build it: `src/bin/Makefile` only puts
+  `pgevent` in `SUBDIRS` when `PORTNAME` is `win32`.
+
+The PAX Python API test data under
+`contrib/pax_storage/src/api/python3/test/` (`test.file1` through `test.file9`,
+plus `test.file3.vm1`, `test.file3.vm2` and `test.file7.toast`) is also binary:
+each file is a PAX-format data file covering a particular set of column types,
+read by `paxpy_test.py`. These names carry no recognised extension, so the
+workflow's extension-based check does not see them; they are recorded here so
+the set is documented rather than invisible.

README.md

@@ -52,9 +52,9 @@
 ## Introduction
 
 Apache Cloudberry (Incubating), created by the original developers of
-Greenplum Database, is one advanced and mature open-source Massively Parallel
-Processing (MPP) database, which evolves from the open-source version of the
-Pivotal Greenplum Database®️ but features a newer PostgreSQL kernel and more
+Greenplum® Database, is one advanced and mature open-source Massively Parallel
+Processing (MPP) database, which evolves from the open-source Greenplum®
+Database but features a newer PostgreSQL kernel and more
 advanced enterprise capabilities. It can serve as a data warehouse and can
 also be used for large-scale analytics and AI/ML workloads.
 
@@ -132,6 +132,41 @@ Apache Cloudberry has a sound foundation.
 Cloudberry is licensed under the Apache License, Version 2.0. For details, see
 the [LICENSE](./LICENSE).
 
+## Crypto Export Notice
+
+This distribution includes cryptographic software.  The country in 
+which you currently reside may have restrictions on the import, 
+possession, use, and/or re-export to another country, of 
+encryption software. BEFORE using any encryption software, please 
+check your country's laws, regulations and policies concerning the
+import, possession, or use, and re-export of encryption software, to 
+see if this is permitted. See http://www.wassenaar.org for
+more information.
+
+The Apache Software Foundation has classified this software as Export Commodity 
+Control Number (ECCN) 5D002, which includes information security
+software using or performing cryptographic functions with asymmetric
+algorithms. The form and manner of this Apache Software Foundation
+distribution makes it eligible for export under the "publicly available"
+Section 742.15(b) exemption (see the BIS Export Administration Regulations, 
+Section 742.15(b)) for both object code and source code.
+
+The following provides more details on the included cryptographic
+software:
+
+- Apache Cloudberry uses OpenSSL for TLS support on client and server
+  connections, and for the SCRAM-SHA-256 authentication method.
+- Apache Cloudberry optionally uses GSSAPI (Kerberos) for authentication and
+  connection encryption, and LDAP over TLS for authentication.
+- The bundled `pgcrypto` extension provides cryptographic functions to SQL,
+  using either OpenSSL or its own built-in implementations.
+- The `gpfdist` external table file server can serve over HTTPS, using OpenSSL
+  and X.509 certificates supplied through its `--ssl <certificates_directory>`
+  option.
+
+Apache Cloudberry does not ship OpenSSL itself; it links against the OpenSSL
+provided by the operating system.
+
 ## ASF Incubator disclaimer
 
 Apache Cloudberry is an effort undergoing incubation at The Apache Software

contrib/interconnect/sql/interconnect.sql

@@ -1,6 +1,12 @@
 -- start_ignore
 \! gpconfig -c shared_preload_libraries -v "interconnect"
-\! gpstop -raiq
+-- Restart in fast mode, not immediate: an immediate shutdown skips the
+-- shutdown checkpoint, so the next startup runs crash recovery, and while the
+-- postmaster is in PM_RECOVERY it rejects the connection gpstart makes to read
+-- the segment configuration ("the database system is not accepting
+-- connections").  gpstop -r then fails, psql gives up at the \c below, and the
+-- whole file is skipped with only the coordinator left running.
+\! gpstop -rafq
 \c
 DROP TABLE IF EXISTS test_ic_data;
 CREATE EXTENSION IF NOT EXISTS interconnect;
@@ -83,5 +89,5 @@ DROP EXTENSION interconnect;
 
 -- start_ignore
 \! gpconfig -r shared_preload_libraries
-\! gpstop -raiq
+\! gpstop -rafq
 -- end_ignore

contrib/interconnect/udp/ic_udpifc.c

@@ -7037,8 +7037,14 @@ putIntoUnackQueueRing(UnackQueueRing *uqr, ICBuffer *buf, uint64 expTime, uint64
 		}
 		else if (diff < TIMER_SPAN_LOSS)
 		{
-			diff = diff < TIMER_SPAN_LOSS ? TIMER_SPAN_LOSS : diff;
+			diff = TIMER_SPAN_LOSS;
 		}
+
+		idx = (uqr->idx + diff / TIMER_SPAN_LOSS) % UNACK_QUEUE_RING_SLOTS_NUM;
+
+#ifdef AMS_VERBOSE_LOGGING
+		write_log("PUTTW: curtime " UINT64_FORMAT " now " UINT64_FORMAT " (diff " UINT64_FORMAT ") expTime " UINT64_FORMAT " previdx %d, nowidx %d, nextidx %d", uqr->currentTime, now, diff, expTime, buf->unackQueueRingSlot, uqr->idx, idx);
+#endif
 	}
 	else
 	{
@@ -7065,12 +7071,6 @@ putIntoUnackQueueRing(UnackQueueRing *uqr, ICBuffer *buf, uint64 expTime, uint64
 #endif
 	}
 
-	idx = (uqr->idx + diff / TIMER_SPAN) % UNACK_QUEUE_RING_SLOTS_NUM;
-
-#ifdef AMS_VERBOSE_LOGGING
-	write_log("PUTTW: curtime " UINT64_FORMAT " now " UINT64_FORMAT " (diff " UINT64_FORMAT ") expTime " UINT64_FORMAT " previdx %d, nowidx %d, nextidx %d", uqr->currentTime, now, diff, expTime, buf->unackQueueRingSlot, uqr->idx, idx);
-#endif
-
 	buf->unackQueueRingSlot = idx;
 	icBufferListAppend(&unack_queue_ring.slots[idx], buf);
 }

contrib/pax_storage/CMakeLists.txt

@@ -118,13 +118,30 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/")
 include(FindDependencies)
 add_subdirectory(src/cpp)
 
+# yyjson is only needed by the manifest API implementation, which is off by
+# default.  It is fetched from its upstream release at configure time rather
+# than vendored as a submodule, so that it stays out of the source release
+# for builds that do not use it.
+set(YYJSON_VERSION "0.12.0" CACHE STRING "yyjson release to fetch for the manifest API")
+
 if(USE_MANIFEST_API AND NOT USE_PAX_CATALOG)
+    include(FetchContent)
+
+    message(STATUS "Fetching yyjson ${YYJSON_VERSION} for the manifest API")
+    FetchContent_Declare(
+        yyjson
+        GIT_REPOSITORY https://github.com/ibireme/yyjson.git
+        GIT_TAG ${YYJSON_VERSION}
+        GIT_SHALLOW TRUE
+    )
+
     set(SAVED_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
     set(SAVED_CMAKE_INSTALL_SO_NO_EXE ${CMAKE_INSTALL_SO_NO_EXE})
 
     set(BUILD_SHARED_LIBS ON)
     set(CMAKE_INSTALL_SO_NO_EXE OFF)
-    add_subdirectory(${CMAKE_SOURCE_DIR}/../../dependency/yyjson ${CMAKE_BINARY_DIR}/yyjson_build)
+
+    FetchContent_MakeAvailable(yyjson)
 
     set(BUILD_SHARED_LIBS ${SAVED_BUILD_SHARED_LIBS})
     set(CMAKE_INSTALL_SO_NO_EXE ${SAVED_CMAKE_INSTALL_SO_NO_EXE})

contrib/pax_storage/doc/README.md

@@ -54,12 +54,16 @@ Also, you need to run the following command at the top level of the Cloudberry s
 git submodule update --init --recursive
 ```
 
-The following submodules will be downloaded for building and tesing PAX:
-- yyjson (`dependency/yyjson`)
-- cpp-stub (`contrib/pax_storage/src/cpp/cotnrib`)
-- googlebench (`contrib/pax_storage/src/cpp/cotnrib`)
-- googletest (`contrib/pax_storage/src/cpp/cotnrib`)
-- tabulate (`contrib/pax_storage/src/cpp/cotnrib`)
+The following submodules will be downloaded for building and testing PAX:
+- cpp-stub (`contrib/pax_storage/src/cpp/contrib`)
+- googlebench (`contrib/pax_storage/src/cpp/contrib`)
+- googletest (`contrib/pax_storage/src/cpp/contrib`)
+- tabulate (`contrib/pax_storage/src/cpp/contrib`)
+
+yyjson is not a submodule. It is only needed by the manifest API
+implementation, which is off by default, and CMake fetches it from its
+upstream release during configuration when you build with
+`USE_MANIFEST_API=ON USE_PAX_CATALOG=OFF`.
 
 ### Build debug version
 

contrib/pax_storage/expected/delete_bloom_stats.out

@@ -0,0 +1,60 @@
+-- Regression test for issue #1749:
+-- PAX DELETE crashes with SIGSEGV when bloomfilter_columns are not a
+-- subset of minmax_columns. The stats refresh inside
+-- DeleteWithVisibilityMap must project every column it reads.
+-- Case 1: bloomfilter column (payload) is NOT in minmax_columns.
+-- Pre-fix: segment crashed on DELETE.
+drop table if exists pax_delete_bloom_crash;
+NOTICE:  table "pax_delete_bloom_crash" does not exist, skipping
+create table pax_delete_bloom_crash (id int, k int, payload text)
+using pax
+with (minmax_columns = 'id', bloomfilter_columns = 'payload');
+NOTICE:  Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table.
+HINT:  The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
+insert into pax_delete_bloom_crash
+select i, i % 10, 'payload-' || i::text
+from generate_series(1, 10000) as i;
+delete from pax_delete_bloom_crash where id between 1 and 100;
+select count(*) from pax_delete_bloom_crash;
+ count 
+-------
+  9900
+(1 row)
+
+drop table pax_delete_bloom_crash;
+-- Case 2: bloomfilter only, no minmax columns.
+drop table if exists pax_delete_bf_only;
+NOTICE:  table "pax_delete_bf_only" does not exist, skipping
+create table pax_delete_bf_only (id int, payload text)
+using pax
+with (bloomfilter_columns = 'payload');
+NOTICE:  Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table.
+HINT:  The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
+insert into pax_delete_bf_only
+select i, 'payload-' || i::text from generate_series(1, 5000) as i;
+delete from pax_delete_bf_only where id between 1 and 50;
+select count(*) from pax_delete_bf_only;
+ count 
+-------
+  4950
+(1 row)
+
+drop table pax_delete_bf_only;
+-- Case 3: minmax and bloomfilter columns overlap but neither is a subset.
+drop table if exists pax_delete_mm_bf_mixed;
+NOTICE:  table "pax_delete_mm_bf_mixed" does not exist, skipping
+create table pax_delete_mm_bf_mixed (id int, k int, payload text)
+using pax
+with (minmax_columns = 'id,payload', bloomfilter_columns = 'k,payload');
+NOTICE:  Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table.
+HINT:  The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
+insert into pax_delete_mm_bf_mixed
+select i, i % 7, 'p-' || i::text from generate_series(1, 5000) as i;
+delete from pax_delete_mm_bf_mixed where id between 1 and 50;
+select count(*) from pax_delete_mm_bf_mixed;
+ count 
+-------
+  4950
+(1 row)
+
+drop table pax_delete_mm_bf_mixed;

contrib/pax_storage/pax_schedule

@@ -6,6 +6,7 @@ test: alter_distributed
 test: toast toast_failed detoast ddl numeric types filter
 test: update
 test: statistics_bloom_filter
+test: delete_bloom_stats
 
 test: filter_tree filter_tree_arithmetic
 test: filter_tree_root_quals