diff --git a/contrib/pax_storage/src/test/regress/expected/stats_ext_optimizer.out b/contrib/pax_storage/src/test/regress/expected/stats_ext_optimizer.out index a3818d9bf0c..f1115d738bb 100644 --- a/contrib/pax_storage/src/test/regress/expected/stats_ext_optimizer.out +++ b/contrib/pax_storage/src/test/regress/expected/stats_ext_optimizer.out @@ -1219,19 +1219,19 @@ SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a < ANY (ARRAY[1, 51]) AND b > ''1'''); estimated | actual -----------+-------- - 1815 | 2400 + 2551 | 2400 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a >= ANY (ARRAY[1, 51]) AND b <= ANY (ARRAY[''1'', ''2''])'); estimated | actual -----------+-------- - 4801 | 1250 + 1301 | 1250 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a <= ANY (ARRAY[1, 2, 51, 52]) AND b >= ANY (ARRAY[''1'', ''2''])'); estimated | actual -----------+-------- - 1133 | 2550 + 2651 | 2550 (1 row) -- ALL (should not benefit from functional dependencies) @@ -1385,19 +1385,19 @@ SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a < ANY (ARRAY[1, 51]) AND b > ''1'''); estimated | actual -----------+-------- - 1815 | 2400 + 2551 | 2400 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a >= ANY (ARRAY[1, 51]) AND b <= ANY (ARRAY[''1'', ''2''])'); estimated | actual -----------+-------- - 4801 | 1250 + 1301 | 1250 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a <= ANY (ARRAY[1, 2, 51, 52]) AND b >= ANY (ARRAY[''1'', ''2''])'); estimated | actual -----------+-------- - 1133 | 2550 + 2651 | 2550 (1 row) -- ALL (should not benefit from functional dependencies) @@ -1944,25 +1944,25 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 1 = a AND ''1' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 1 AND b < ''1'''); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 1 > a AND ''1'' > b'); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a <= 0 AND b <= ''0'''); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 0 >= a AND ''0'' >= b'); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 AND b = ''1'' AND c = 1'); @@ -1974,25 +1974,25 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 AND b = SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 5 AND b < ''1'' AND c < 5'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 5 AND ''1'' > b AND 5 > c'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a <= 4 AND b <= ''0'' AND c <= 4'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 4 >= a AND ''0'' >= b AND 4 >= c'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1'); @@ -2085,25 +2085,25 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 1 = a AND ''1' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 1 AND b < ''1'''); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 1 > a AND ''1'' > b'); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a <= 0 AND b <= ''0'''); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 0 >= a AND ''0'' >= b'); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 AND b = ''1'' AND c = 1'); @@ -2115,25 +2115,25 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 AND b = SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 5 AND b < ''1'' AND c < 5'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 5 AND ''1'' > b AND 5 > c'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a <= 4 AND b <= ''0'' AND c <= 4'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 4 >= a AND ''0'' >= b AND 4 >= c'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1'); diff --git a/src/backend/gpopt/gpdbwrappers.cpp b/src/backend/gpopt/gpdbwrappers.cpp index 97b149d6164..7b17d7ba293 100644 --- a/src/backend/gpopt/gpdbwrappers.cpp +++ b/src/backend/gpopt/gpdbwrappers.cpp @@ -43,6 +43,7 @@ extern "C" { #include "catalog/pg_inherits.h" #include "cdb/cdbvars.h" #include "foreign/fdwapi.h" +#include "mb/pg_wchar.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" @@ -56,10 +57,13 @@ extern "C" { #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/partcache.h" +#include "utils/pg_locale.h" extern bool enable_parallel; extern int max_parallel_workers_per_gather; + } + #define GP_WRAP_START \ sigjmp_buf local_sigjmp_buf; \ { \ @@ -2450,6 +2454,133 @@ gpdb::MakeGpPolicy(GpPolicyType ptype, int nattrs, int numsegments) GP_WRAP_END; } +size_t +gpdb::ComputeLocaleSortKey(char *dest, size_t destsize, const char *src, + size_t srclen, Oid collation) +{ + GP_WRAP_START; + { + if (destsize == 0) + { + return 0; + } + + // C/POSIX collation: byte order already matches sort order, just copy. + // Treat InvalidOid the same way (no collation info available). + if (!OidIsValid(collation) || lc_collate_is_c(collation)) + { + size_t n = (srclen < destsize) ? srclen : destsize; + memcpy(dest, src, n); + return n; + } + + pg_locale_t locale = pg_newlocale_from_collation(collation); + + // Non-deterministic collations (case/accent-insensitive ICU) don't + // produce a totally-ordered sort key for our purposes; bail. + if (locale != NULL && !locale->deterministic) + { + return 0; + } + +#ifdef USE_ICU + if (locale != NULL && locale->provider == COLLPROVIDER_ICU) + { + // Best path for UTF-8 databases: ICU iterator gives us exactly + // the prefix we want without computing the full sort key. + if (GetDatabaseEncoding() == PG_UTF8) + { + UCharIterator iter; + uint32_t state[2] = {0, 0}; + UErrorCode status = U_ZERO_ERROR; + uiter_setUTF8(&iter, src, (int32_t) srclen); + int32_t bsize = ucol_nextSortKeyPart( + locale->info.icu.ucol, &iter, state, + (uint8_t *) dest, (int32_t) destsize, &status); + if (U_FAILURE(status)) + { + return 0; + } + return (size_t) bsize; + } + // Non-UTF8 ICU: skip the conversion dance; caller falls back. + return 0; + } +#endif + + // libc path: strxfrm[_l] needs a NUL-terminated input and an output + // buffer big enough to hold the full transform (the C standard + // leaves dest indeterminate if the result didn't fit). Allocate a + // big-enough work buffer, then copy the prefix we actually need. + char nullterm_stack[256]; + char *nullterm = nullterm_stack; + if (srclen + 1 > sizeof(nullterm_stack)) + { + nullterm = (char *) palloc(srclen + 1); + } + memcpy(nullterm, src, srclen); + nullterm[srclen] = '\0'; + + char xfrm_stack[1024]; + char *xfrm = xfrm_stack; + size_t xfrm_size = sizeof(xfrm_stack); + size_t bsize; + for (;;) + { +#ifdef HAVE_LOCALE_T + if (locale != NULL && locale->provider == COLLPROVIDER_LIBC) + { + bsize = + strxfrm_l(xfrm, nullterm, xfrm_size, locale->info.lt); + } + else +#endif + { + bsize = strxfrm(xfrm, nullterm, xfrm_size); + } + if (bsize < xfrm_size) + { + break; + } + // grow and retry; cap to avoid runaway allocations. + if (xfrm_size >= (size_t) 64 * 1024) + { + if (xfrm != xfrm_stack) + { + pfree(xfrm); + } + if (nullterm != nullterm_stack) + { + pfree(nullterm); + } + return 0; + } + if (xfrm != xfrm_stack) + { + pfree(xfrm); + } + xfrm_size = (bsize + 1 > xfrm_size * 2) ? bsize + 1 + : xfrm_size * 2; + xfrm = (char *) palloc(xfrm_size); + } + + size_t out = (bsize < destsize) ? bsize : destsize; + memcpy(dest, xfrm, out); + + if (xfrm != xfrm_stack) + { + pfree(xfrm); + } + if (nullterm != nullterm_stack) + { + pfree(nullterm); + } + return out; + } + GP_WRAP_END; + return 0; +} + uint32 gpdb::HashChar(Datum d) { diff --git a/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp b/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp index aa23d3932a2..96c60c92bcb 100644 --- a/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp +++ b/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp @@ -2055,13 +2055,22 @@ CTranslatorRelcacheToDXL::RetrieveColStats(CMemoryPool *mp, CDouble distinct_remaining(0.0); CDouble freq_remaining(0.0); + // PG's analyze.c writes the same attribute collation into every slot of + // pg_statistic for a given column, so the MCV and histogram slots always + // share one collation. Assert and forward a single value. + GPOS_ASSERT(!OidIsValid(mcv_slot.stacoll) || + !OidIsValid(hist_slot.stacoll) || + mcv_slot.stacoll == hist_slot.stacoll); + Oid att_collation = OidIsValid(mcv_slot.stacoll) ? mcv_slot.stacoll + : hist_slot.stacoll; + // transform all the bits and pieces from pg_statistic // to a single bucket structure CDXLBucketArray *dxl_stats_bucket_array_transformed = TransformStatsToDXLBucketArray( mp, att_type, num_distinct, null_freq, mcv_slot.values, mcv_slot.numbers, ULONG(mcv_slot.nvalues), hist_slot.values, - ULONG(hist_slot.nvalues)); + ULONG(hist_slot.nvalues), att_collation); GPOS_ASSERT(nullptr != dxl_stats_bucket_array_transformed); @@ -2344,15 +2353,17 @@ CDXLBucketArray * CTranslatorRelcacheToDXL::TransformStatsToDXLBucketArray( CMemoryPool *mp, OID att_type, CDouble num_distinct, CDouble null_freq, const Datum *mcv_values, const float4 *mcv_frequencies, - ULONG num_mcv_values, const Datum *hist_values, ULONG num_hist_values) + ULONG num_mcv_values, const Datum *hist_values, ULONG num_hist_values, + Oid collation) { CMDIdGPDB *mdid_atttype = GPOS_NEW(mp) CMDIdGPDB(IMDId::EmdidGeneral, att_type); IMDType *md_type = RetrieveType(mp, mdid_atttype); // translate MCVs to Orca histogram. Create an empty histogram if there are no MCVs. - CHistogram *gpdb_mcv_hist = TransformMcvToOrcaHistogram( - mp, md_type, mcv_values, mcv_frequencies, num_mcv_values); + CHistogram *gpdb_mcv_hist = + TransformMcvToOrcaHistogram(mp, md_type, mcv_values, mcv_frequencies, + num_mcv_values, collation); GPOS_ASSERT(gpdb_mcv_hist->IsValid()); @@ -2365,11 +2376,10 @@ CTranslatorRelcacheToDXL::TransformStatsToDXLBucketArray( hist_freq = CDouble(1.0) - null_freq - mcv_freq; } - BOOL is_text_type = mdid_atttype->Equals(&CMDIdGPDB::m_mdid_varchar) || - mdid_atttype->Equals(&CMDIdGPDB::m_mdid_bpchar) || - mdid_atttype->Equals(&CMDIdGPDB::m_mdid_text); - BOOL has_hist = !is_text_type && 1 < num_hist_values && - CStatistics::Epsilon < hist_freq; + // Text types are no longer excluded: LINT mapping is order-preserving + // (see ExtractLintValueFromDatum); TransformHistToOrcaHistogram drops + // the histogram if any bucket pair fails StatsAreLessThan. + BOOL has_hist = 1 < num_hist_values && CStatistics::Epsilon < hist_freq; CHistogram *histogram = nullptr; @@ -2377,8 +2387,9 @@ CTranslatorRelcacheToDXL::TransformStatsToDXLBucketArray( if (has_hist) { // histogram from gpdb histogram - histogram = TransformHistToOrcaHistogram( - mp, md_type, hist_values, num_hist_values, num_distinct, hist_freq); + histogram = TransformHistToOrcaHistogram(mp, md_type, hist_values, + num_hist_values, num_distinct, + hist_freq, collation); if (0 == histogram->GetNumBuckets()) { has_hist = false; @@ -2439,7 +2450,7 @@ CTranslatorRelcacheToDXL::TransformStatsToDXLBucketArray( CHistogram * CTranslatorRelcacheToDXL::TransformMcvToOrcaHistogram( CMemoryPool *mp, const IMDType *md_type, const Datum *mcv_values, - const float4 *mcv_frequencies, ULONG num_mcv_values) + const float4 *mcv_frequencies, ULONG num_mcv_values, Oid collation) { IDatumArray *datums = GPOS_NEW(mp) IDatumArray(mp); CDoubleArray *freqs = GPOS_NEW(mp) CDoubleArray(mp); @@ -2448,7 +2459,7 @@ CTranslatorRelcacheToDXL::TransformMcvToOrcaHistogram( { Datum datumMCV = mcv_values[ul]; IDatum *datum = CTranslatorScalarToDXL::CreateIDatumFromGpdbDatum( - mp, md_type, false /* is_null */, datumMCV); + mp, md_type, false /* is_null */, datumMCV, collation); datums->Append(datum); freqs->Append(GPOS_NEW(mp) CDouble(mcv_frequencies[ul])); @@ -2481,7 +2492,8 @@ CTranslatorRelcacheToDXL::TransformMcvToOrcaHistogram( CHistogram * CTranslatorRelcacheToDXL::TransformHistToOrcaHistogram( CMemoryPool *mp, const IMDType *md_type, const Datum *hist_values, - ULONG num_hist_values, CDouble num_distinct, CDouble hist_freq) + ULONG num_hist_values, CDouble num_distinct, CDouble hist_freq, + Oid collation) { GPOS_ASSERT(1 < num_hist_values); @@ -2495,9 +2507,9 @@ CTranslatorRelcacheToDXL::TransformHistToOrcaHistogram( for (ULONG ul = 0; ul < num_buckets; ul++) { IDatum *min_datum = CTranslatorScalarToDXL::CreateIDatumFromGpdbDatum( - mp, md_type, false /* is_null */, hist_values[ul]); + mp, md_type, false /* is_null */, hist_values[ul], collation); IDatum *max_datum = CTranslatorScalarToDXL::CreateIDatumFromGpdbDatum( - mp, md_type, false /* is_null */, hist_values[ul + 1]); + mp, md_type, false /* is_null */, hist_values[ul + 1], collation); BOOL is_lower_closed, is_upper_closed; if (min_datum->StatsAreEqual(max_datum)) diff --git a/src/backend/gpopt/translate/CTranslatorScalarToDXL.cpp b/src/backend/gpopt/translate/CTranslatorScalarToDXL.cpp index 7256409eb54..7a1f2479a8e 100644 --- a/src/backend/gpopt/translate/CTranslatorScalarToDXL.cpp +++ b/src/backend/gpopt/translate/CTranslatorScalarToDXL.cpp @@ -756,7 +756,7 @@ CTranslatorScalarToDXL::TranslateConstToDXL(CMemoryPool *mp, CMDAccessor *mda, // translate gpdb datum into a DXL datum CDXLDatum *datum_dxl = CTranslatorScalarToDXL::TranslateDatumToDXL( mp, md_type, constant->consttypmod, constant->constisnull, - constant->constlen, constant->constvalue); + constant->constlen, constant->constvalue, constant->constcollid); return datum_dxl; } @@ -2272,7 +2272,8 @@ CDXLDatum * CTranslatorScalarToDXL::TranslateDatumToDXL(CMemoryPool *mp, const IMDType *md_type, INT type_modifier, BOOL is_null, - ULONG len, Datum datum) + ULONG len, Datum datum, + Oid collation) { switch (md_type->GetDatumType()) { @@ -2280,7 +2281,7 @@ CTranslatorScalarToDXL::TranslateDatumToDXL(CMemoryPool *mp, { // generate a datum of generic type return TranslateGenericDatumToDXL(mp, md_type, type_modifier, - is_null, len, datum); + is_null, len, datum, collation); } case IMDType::EtiInt2: { @@ -2322,7 +2323,7 @@ CTranslatorScalarToDXL::TranslateGenericDatumToDXL(CMemoryPool *mp, const IMDType *md_type, INT type_modifier, BOOL is_null, ULONG len, - Datum datum) + Datum datum, Oid collation) { CMDIdGPDB *mdid_old = CMDIdGPDB::CastMdid(md_type->MDId()); CMDIdGPDB *mdid = GPOS_NEW(mp) CMDIdGPDB(*mdid_old); @@ -2348,7 +2349,7 @@ CTranslatorScalarToDXL::TranslateGenericDatumToDXL(CMemoryPool *mp, CMDIdGPDB(IMDId::EmdidGeneral, gpdb::GetBaseType(mdid->Oid())); // base_mdid is used for text related domain types lint_value = ExtractLintValueFromDatum(md_type, is_null, bytes, length, - base_mdid); + base_mdid, collation); base_mdid->Release(); } @@ -2593,7 +2594,8 @@ LINT CTranslatorScalarToDXL::ExtractLintValueFromDatum(const IMDType *md_type, BOOL is_null, BYTE *bytes, ULONG length, - IMDId *base_mdid) + IMDId *base_mdid, + Oid collation) { IMDId *mdid = md_type->MDId(); GPOS_ASSERT(CMDTypeGenericGPDB::HasByte2IntMapping(md_type)); @@ -2615,43 +2617,83 @@ CTranslatorScalarToDXL::ExtractLintValueFromDatum(const IMDType *md_type, } else { - // use hash value - ULONG hash = 0; - if (is_null) + // We pack the first 7 bytes of an "order-aligned" prefix into a + // non-negative LINT, so that LINT comparison preserves the type's + // native ordering. The prefix bytes come from one of: + // * a libc/ICU locale sort key (pg_strxfrm-equivalent) when the + // column has a non-C collation + // * the raw payload otherwise (C/POSIX collation, uuid, char, + // name -- all byte-orderable) + // + // Trade-off: two values whose first 7 sort-key bytes coincide will + // collide to the same LINT, so StatsAreEqual via LINT mapping treats + // them as equal. This is acceptable for MCV stats and is what + // PG14's convert_string_to_scalar effectively does for intra-bucket + // interpolation as well. + const BYTE *payload = bytes; + ULONG payload_len = length; + + BOOL is_varlena_string = false; + + if (mdid->Equals(&CMDIdGPDB::m_mdid_uuid)) { - hash = gpos::HashValue(&hash); + // uuid: raw bytes are byte-orderable + } + else if (mdid->Equals(&CMDIdGPDB::m_mdid_name) || + (base_mdid->IsValid() && + base_mdid->Equals(&CMDIdGPDB::m_mdid_name))) + { + // name uses C collation in PG, raw bytes are order-correct + payload_len = + (ULONG) strnlen((const char *) bytes, (size_t) length); + } + else if (mdid->Equals(&CMDIdGPDB::m_mdid_char) || + (base_mdid->IsValid() && + base_mdid->Equals(&CMDIdGPDB::m_mdid_char))) + { + // char: byte-orderable } else { - if (mdid->Equals(&CMDIdGPDB::m_mdid_uuid)) - { - hash = gpdb::UUIDHash((Datum) bytes); - } - else if (mdid->Equals(&CMDIdGPDB::m_mdid_bpchar) || - (base_mdid->IsValid() && - base_mdid->Equals(&CMDIdGPDB::m_mdid_bpchar))) - { - hash = gpdb::HashBpChar((Datum) bytes); - } - else if (mdid->Equals(&CMDIdGPDB::m_mdid_char) || - (base_mdid->IsValid() && - base_mdid->Equals(&CMDIdGPDB::m_mdid_char))) - { - hash = gpdb::HashChar((Datum) bytes); - } - else if (mdid->Equals(&CMDIdGPDB::m_mdid_name) || - (base_mdid->IsValid() && - base_mdid->Equals(&CMDIdGPDB::m_mdid_name))) - { - hash = gpdb::HashName((Datum) bytes); - } - else + // varlena types: text, varchar, bpchar. + payload = (const BYTE *) VARDATA_ANY((const void *) bytes); + payload_len = (ULONG) VARSIZE_ANY_EXHDR((const void *) bytes); + is_varlena_string = true; + } + + // For non-C collation on varlena strings, run the payload through + // the locale sort-key transform so the prefix orders by the + // actual comparison semantics rather than raw bytes. uuid/name/ + // char are always byte-orderable and don't need this. + BYTE sortkey[16]; + if (is_varlena_string && OidIsValid(collation)) + { + size_t produced = gpdb::ComputeLocaleSortKey( + (char *) sortkey, sizeof(sortkey), + (const char *) payload, (size_t) payload_len, collation); + if (produced > 0) { - hash = gpdb::HashText((Datum) bytes); + payload = sortkey; + payload_len = (ULONG) produced; } + // produced == 0 means C collation (caller short-circuit) or a + // non-deterministic/unsupported collation; either way keep the + // raw payload. For C collation this is also order-correct. + } + + LINT packed = 0; + const ULONG kPrefix = 7; // leave the sign byte clear + ULONG take = payload_len < kPrefix ? payload_len : kPrefix; + for (ULONG i = 0; i < take; i++) + { + packed = (packed << 8) | (LINT) payload[i]; } + // right-pad short prefixes with zeros so shorter strings sort before + // longer ones that share the prefix. + packed <<= (kPrefix - take) * 8; + packed &= (LINT) 0x7FFFFFFFFFFFFFFFLL; - lint_value = (LINT) hash; + lint_value = packed; } return lint_value; @@ -2669,7 +2711,8 @@ IDatum * CTranslatorScalarToDXL::CreateIDatumFromGpdbDatum(CMemoryPool *mp, const IMDType *md_type, BOOL is_null, - Datum gpdb_datum) + Datum gpdb_datum, + Oid collation) { ULONG length = md_type->Length(); if (!md_type->IsPassedByValue() && !is_null) @@ -2682,7 +2725,8 @@ CTranslatorScalarToDXL::CreateIDatumFromGpdbDatum(CMemoryPool *mp, GPOS_ASSERT(is_null || length > 0); CDXLDatum *datum_dxl = CTranslatorScalarToDXL::TranslateDatumToDXL( - mp, md_type, gpmd::default_type_modifier, is_null, length, gpdb_datum); + mp, md_type, gpmd::default_type_modifier, is_null, length, gpdb_datum, + collation); IDatum *datum = md_type->GetDatumForDXLDatum(mp, datum_dxl); datum_dxl->Release(); return datum; diff --git a/src/backend/gporca/libnaucrates/include/naucrates/statistics/CHistogram.h b/src/backend/gporca/libnaucrates/include/naucrates/statistics/CHistogram.h index a67944ce697..ce4ef46bed7 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/statistics/CHistogram.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/statistics/CHistogram.h @@ -426,10 +426,6 @@ class CHistogram : public gpos::DbgPrintMixin // cap the total number of distinct values (NDVs) in buckets to the number of rows void CapNDVs(CDouble rows); - // is comparison type supported for filters for text columns - static BOOL IsOpSupportedForTextFilter( - CStatsPred::EStatsCmpType stats_cmp_type); - // is comparison type supported for filters static BOOL IsOpSupportedForFilter( CStatsPred::EStatsCmpType stats_cmp_type); diff --git a/src/backend/gporca/libnaucrates/src/statistics/CHistogram.cpp b/src/backend/gporca/libnaucrates/src/statistics/CHistogram.cpp index 14c4f5cb40b..010ad6d29a5 100644 --- a/src/backend/gporca/libnaucrates/src/statistics/CHistogram.cpp +++ b/src/backend/gporca/libnaucrates/src/statistics/CHistogram.cpp @@ -591,27 +591,21 @@ CHistogram::IsValid() const return false; } - if (IsHistogramForTextRelatedTypes()) + // Monotonic bucket-bound check applies to all types since the LINT + // mapping is now order-preserving (see ExtractLintValueFromDatum). + for (ULONG bucket_index = 1; bucket_index < m_histogram_buckets->Size(); + bucket_index++) { - return m_histogram_buckets->Size() == 0 || - this->ContainsOnlySingletonBuckets(); - } - else - { - for (ULONG bucket_index = 1; bucket_index < m_histogram_buckets->Size(); - bucket_index++) + CBucket *bucket = (*m_histogram_buckets)[bucket_index]; + CBucket *previous_bucket = (*m_histogram_buckets)[bucket_index - 1]; + + // the later bucket's lower point must be greater than or equal to + // earlier bucket's upper point. Even if the underlying datum does not + // support ordering, the check is safe. + if (bucket->GetLowerBound()->IsLessThan( + previous_bucket->GetUpperBound())) { - CBucket *bucket = (*m_histogram_buckets)[bucket_index]; - CBucket *previous_bucket = (*m_histogram_buckets)[bucket_index - 1]; - - // the later bucket's lower point must be greater than or equal to - // earlier bucket's upper point. Even if the underlying datum does not - // support ordering, the check is safe. - if (bucket->GetLowerBound()->IsLessThan( - previous_bucket->GetUpperBound())) - { - return false; - } + return false; } } return true; @@ -1045,22 +1039,6 @@ CHistogram::CopyHistogram() const return histogram_copy; } -BOOL -CHistogram::IsOpSupportedForTextFilter(CStatsPred::EStatsCmpType stats_cmp_type) -{ - // is the scalar comparison type one of =, <> - switch (stats_cmp_type) - { - case CStatsPred::EstatscmptEq: - case CStatsPred::EstatscmptNEq: - return true; - default: - return false; - } -} - - - // is statistics comparison type supported for filter? BOOL CHistogram::IsOpSupportedForFilter(CStatsPred::EStatsCmpType stats_cmp_type) diff --git a/src/backend/gporca/libnaucrates/src/statistics/CStatsPredUtils.cpp b/src/backend/gporca/libnaucrates/src/statistics/CStatsPredUtils.cpp index 64bc29d99b7..04dba039257 100644 --- a/src/backend/gporca/libnaucrates/src/statistics/CStatsPredUtils.cpp +++ b/src/backend/gporca/libnaucrates/src/statistics/CStatsPredUtils.cpp @@ -287,19 +287,13 @@ CStatsPredUtils::GetPredStats(CMemoryPool *mp, CExpression *expr) CScalarConst::PopExtractFromConstOrCastConst(expr_right); GPOS_ASSERT(nullptr != scalar_const_op); - CMDAccessor *md_accessor = COptCtxt::PoctxtFromTLS()->Pmda(); IDatum *datum = scalar_const_op->GetDatum(); - const IMDType *datum_type = md_accessor->RetrieveType(datum->MDId()); - - BOOL is_text_related = - datum_type->IsTextRelated() && col_ref->RetrieveType()->IsTextRelated(); - if (is_text_related && - !CHistogram::IsOpSupportedForTextFilter(stats_cmp_type)) - { - return GPOS_NEW(mp) - CStatsPredUnsupported(col_ref->Id(), stats_cmp_type); - } + // Text range predicates (`<`, `<=`, `>`, `>=`) used to be downgraded to + // Unsupported here because the hash-based LINT mapping made ordering + // meaningless. The LINT mapping is now a locale sort-key prefix + // (CTranslatorScalarToDXL::ExtractLintValueFromDatum), so text ranges + // can use the generic filter path uniformly. if (!CHistogram::IsOpSupportedForFilter(stats_cmp_type) || !IMDType::StatsAreComparable(col_ref->RetrieveType(), datum)) diff --git a/src/include/gpopt/gpdbwrappers.h b/src/include/gpopt/gpdbwrappers.h index 01d7eaa8cce..3425f2a2e00 100644 --- a/src/include/gpopt/gpdbwrappers.h +++ b/src/include/gpopt/gpdbwrappers.h @@ -312,6 +312,12 @@ Oid GetArrayType(Oid typid); bool GetAttrStatsSlot(AttStatsSlot *sslot, HeapTuple statstuple, int reqkind, Oid reqop, int flags); +// Compute a sort key into dest such that memcmp on the output preserves +// varstr_cmp(src, collation). Returns bytes written (<= destsize), or 0 +// for non-deterministic collations or any failure (caller falls back). +size_t ComputeLocaleSortKey(char *dest, size_t destsize, const char *src, + size_t srclen, Oid collation); + // free attribute stats slot void FreeAttrStatsSlot(AttStatsSlot *sslot); diff --git a/src/include/gpopt/translate/CTranslatorRelcacheToDXL.h b/src/include/gpopt/translate/CTranslatorRelcacheToDXL.h index b060f4de44e..70aae5bfe27 100644 --- a/src/include/gpopt/translate/CTranslatorRelcacheToDXL.h +++ b/src/include/gpopt/translate/CTranslatorRelcacheToDXL.h @@ -185,12 +185,14 @@ class CTranslatorRelcacheToDXL // transform GPDB's MCV information to optimizer's histogram structure static CHistogram *TransformMcvToOrcaHistogram( CMemoryPool *mp, const IMDType *md_type, const Datum *mcv_values, - const float4 *mcv_frequencies, ULONG num_mcv_values); + const float4 *mcv_frequencies, ULONG num_mcv_values, + Oid collation = InvalidOid); // transform GPDB's hist information to optimizer's histogram structure static CHistogram *TransformHistToOrcaHistogram( CMemoryPool *mp, const IMDType *md_type, const Datum *hist_values, - ULONG num_hist_values, CDouble num_distinct, CDouble hist_freq); + ULONG num_hist_values, CDouble num_distinct, CDouble hist_freq, + Oid collation = InvalidOid); // histogram to array of dxl buckets static CDXLBucketArray *TransformHistogramToDXLBucketArray( @@ -200,7 +202,8 @@ class CTranslatorRelcacheToDXL static CDXLBucketArray *TransformStatsToDXLBucketArray( CMemoryPool *mp, OID att_type, CDouble num_distinct, CDouble null_freq, const Datum *mcv_values, const float4 *mcv_frequencies, - ULONG num_mcv_values, const Datum *hist_values, ULONG num_hist_values); + ULONG num_mcv_values, const Datum *hist_values, ULONG num_hist_values, + Oid collation = InvalidOid); // get partition keys and types for a relation static void RetrievePartKeysAndTypes(CMemoryPool *mp, Relation rel, OID oid, diff --git a/src/include/gpopt/translate/CTranslatorScalarToDXL.h b/src/include/gpopt/translate/CTranslatorScalarToDXL.h index 75bbdf44833..4665d52af0d 100644 --- a/src/include/gpopt/translate/CTranslatorScalarToDXL.h +++ b/src/include/gpopt/translate/CTranslatorScalarToDXL.h @@ -304,12 +304,14 @@ class CTranslatorScalarToDXL static CDXLDatum *TranslateDatumToDXL(CMemoryPool *mp, const IMDType *md_type, INT type_modifier, BOOL is_null, - ULONG len, Datum datum); + ULONG len, Datum datum, + Oid collation = InvalidOid); // translate GPDB datum to IDatum static IDatum *CreateIDatumFromGpdbDatum(CMemoryPool *mp, const IMDType *md_type, - BOOL is_null, Datum datum); + BOOL is_null, Datum datum, + Oid collation = InvalidOid); // extract the byte array value of the datum static BYTE *ExtractByteArrayFromDatum(CMemoryPool *mp, @@ -322,7 +324,8 @@ class CTranslatorScalarToDXL // extract the long int value of a datum static LINT ExtractLintValueFromDatum(const IMDType *md_type, BOOL is_null, BYTE *bytes, ULONG len, - IMDId *base_mdid); + IMDId *base_mdid, + Oid collation = InvalidOid); // datum to oid CDXLDatum static CDXLDatum *TranslateOidDatumToDXL(CMemoryPool *mp, @@ -359,7 +362,8 @@ class CTranslatorScalarToDXL const IMDType *md_type, INT type_modifier, BOOL is_null, ULONG len, - Datum datum); + Datum datum, + Oid collation = InvalidOid); }; } // namespace gpdxl #endif // GPDXL_CTranslatorScalarToDXL_H diff --git a/src/test/regress/expected/bfv_index_optimizer.out b/src/test/regress/expected/bfv_index_optimizer.out index d2d5e812dc0..b6b6da8270d 100644 --- a/src/test/regress/expected/bfv_index_optimizer.out +++ b/src/test/regress/expected/bfv_index_optimizer.out @@ -1317,26 +1317,22 @@ insert into ao_tbl select 'abc' from generate_series(1,20) i; analyze ao_tbl; -- identical plans explain select * from ao_tbl where path_hash = 'ABC'; - QUERY PLAN ------------------------------------------------------------------------------- - Gather Motion 1:1 (slice1; segments: 1) (cost=0.00..387.96 rows=1 width=4) - -> Bitmap Heap Scan on ao_tbl (cost=0.00..387.96 rows=1 width=4) - Recheck Cond: ((path_hash)::text = 'ABC'::text) - -> Bitmap Index Scan on ao_idx (cost=0.00..0.00 rows=0 width=0) - Index Cond: ((path_hash)::text = 'ABC'::text) - Optimizer: Pivotal Optimizer (GPORCA) -(6 rows) + QUERY PLAN +------------------------------------------------------------------------------- + Gather Motion 1:1 (slice1; segments: 1) (cost=0.00..431.00 rows=20 width=4) + -> Seq Scan on ao_tbl (cost=0.00..431.00 rows=7 width=4) + Filter: ((path_hash)::text = 'ABC'::text) + Optimizer: GPORCA +(4 rows) explain select * from ao_tbl where 'ABC' = path_hash; - QUERY PLAN ------------------------------------------------------------------------------- - Gather Motion 1:1 (slice1; segments: 1) (cost=0.00..387.96 rows=1 width=4) - -> Bitmap Heap Scan on ao_tbl (cost=0.00..387.96 rows=1 width=4) - Recheck Cond: ((path_hash)::text = 'ABC'::text) - -> Bitmap Index Scan on ao_idx (cost=0.00..0.00 rows=0 width=0) - Index Cond: ((path_hash)::text = 'ABC'::text) - Optimizer: Pivotal Optimizer (GPORCA) -(6 rows) + QUERY PLAN +------------------------------------------------------------------------------- + Gather Motion 1:1 (slice1; segments: 1) (cost=0.00..431.00 rows=20 width=4) + -> Seq Scan on ao_tbl (cost=0.00..431.00 rows=7 width=4) + Filter: ((path_hash)::text = 'ABC'::text) + Optimizer: GPORCA +(4 rows) -- Test AO partition table -- Dynamic index scan is disabled in AO table, so dynamic bitmap diff --git a/src/test/regress/expected/stats_ext_optimizer.out b/src/test/regress/expected/stats_ext_optimizer.out index dafbf0a28b4..96ffae4fb28 100644 --- a/src/test/regress/expected/stats_ext_optimizer.out +++ b/src/test/regress/expected/stats_ext_optimizer.out @@ -1220,19 +1220,19 @@ SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a < ANY (ARRAY[1, 51]) AND b > ''1'''); estimated | actual -----------+-------- - 1815 | 2400 + 2551 | 2400 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a >= ANY (ARRAY[1, 51]) AND b <= ANY (ARRAY[''1'', ''2''])'); estimated | actual -----------+-------- - 4801 | 1250 + 1301 | 1250 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a <= ANY (ARRAY[1, 2, 51, 52]) AND b >= ANY (ARRAY[''1'', ''2''])'); estimated | actual -----------+-------- - 1133 | 2550 + 2651 | 2550 (1 row) -- ALL (should not benefit from functional dependencies) @@ -1386,19 +1386,19 @@ SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a < ANY (ARRAY[1, 51]) AND b > ''1'''); estimated | actual -----------+-------- - 1815 | 2400 + 2551 | 2400 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a >= ANY (ARRAY[1, 51]) AND b <= ANY (ARRAY[''1'', ''2''])'); estimated | actual -----------+-------- - 4801 | 1250 + 1301 | 1250 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM functional_dependencies WHERE a <= ANY (ARRAY[1, 2, 51, 52]) AND b >= ANY (ARRAY[''1'', ''2''])'); estimated | actual -----------+-------- - 1133 | 2550 + 2651 | 2550 (1 row) -- ALL (should not benefit from functional dependencies) @@ -1947,25 +1947,25 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 1 = a AND ''1' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 1 AND b < ''1'''); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 1 > a AND ''1'' > b'); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a <= 0 AND b <= ''0'''); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 0 >= a AND ''0'' >= b'); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 AND b = ''1'' AND c = 1'); @@ -1977,25 +1977,25 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 AND b = SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 5 AND b < ''1'' AND c < 5'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 5 AND ''1'' > b AND 5 > c'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a <= 4 AND b <= ''0'' AND c <= 4'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 4 >= a AND ''0'' >= b AND 4 >= c'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1'); @@ -2088,25 +2088,25 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 1 = a AND ''1' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 1 AND b < ''1'''); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 1 > a AND ''1'' > b'); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a <= 0 AND b <= ''0'''); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 0 >= a AND ''0'' >= b'); estimated | actual -----------+-------- - 36 | 50 + 2 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 AND b = ''1'' AND c = 1'); @@ -2118,25 +2118,25 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 AND b = SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 5 AND b < ''1'' AND c < 5'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < 5 AND ''1'' > b AND 5 > c'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a <= 4 AND b <= ''0'' AND c <= 4'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE 4 >= a AND ''0'' >= b AND 4 >= c'); estimated | actual -----------+-------- - 85 | 50 + 5 | 50 (1 row) SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1');