From 5a0d2432efb7e44747d11f58170e906663b88679 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Tue, 1 Sep 2026 20:09:44 +1000 Subject: [PATCH] MDEV-40168 [wip] Add multi valued index over fulltext TODOs on top of those in the patch diff: - EXPLAIN output should not say fulltext - check type match to avoid false negative / positive bugs in mysql - transcode the value into the index charset in mvi_encode_key --- libmysqld/CMakeLists.txt | 1 + mysql-test/main/multi_valued_index.opt | 1 + mysql-test/main/multi_valued_index.result | 58 +++ mysql-test/main/multi_valued_index.test | 41 ++ .../suite/perfschema/r/digest_view.result | 50 +-- .../start_server_low_digest_sql_length.result | 4 +- sql/CMakeLists.txt | 3 +- sql/item.h | 5 + sql/item_func.h | 3 +- sql/item_jsonfunc.h | 1 + sql/item_strfunc.h | 24 + sql/lex.h | 1 + sql/opt_multi_valued_index.cc | 410 ++++++++++++++++++ sql/opt_multi_valued_index.h | 17 + sql/sql_class.cc | 3 +- sql/sql_select.cc | 6 + sql/sql_select.h | 1 + sql/sql_table.cc | 4 +- sql/sql_table.h | 2 + sql/sql_yacc.yy | 43 +- 20 files changed, 644 insertions(+), 34 deletions(-) create mode 100644 mysql-test/main/multi_valued_index.opt create mode 100644 mysql-test/main/multi_valued_index.result create mode 100644 mysql-test/main/multi_valued_index.test create mode 100644 sql/opt_multi_valued_index.cc create mode 100644 sql/opt_multi_valued_index.h diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index d38f2a48c737b..6852fbb19a650 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -72,6 +72,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/mf_iocache.cc ../sql/my_decimal.cc ../sql/net_serv.cc ../sql/opt_range.cc ../sql/opt_group_by_cardinality.cc + ../sql/opt_multi_valued_index.cc ../sql/opt_rewrite_date_cmp.cc ../sql/opt_rewrite_remove_casefold.cc ../sql/opt_sargable_left.cc diff --git a/mysql-test/main/multi_valued_index.opt b/mysql-test/main/multi_valued_index.opt new file mode 100644 index 0000000000000..a076db4e5b886 --- /dev/null +++ b/mysql-test/main/multi_valued_index.opt @@ -0,0 +1 @@ +--innodb_ft_index_cache diff --git a/mysql-test/main/multi_valued_index.result b/mysql-test/main/multi_valued_index.result new file mode 100644 index 0000000000000..f69a49d18a264 --- /dev/null +++ b/mysql-test/main/multi_valued_index.result @@ -0,0 +1,58 @@ +# basic test +SET SESSION debug_dbug = '+d,test_invisible_index,test_completely_invisible'; +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +show index from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +t1 1 invisible1 1 invisible1 A 0 NULL NULL YES BTREE NO +t1 1 idx 1 DB_MVI_1 NULL NULL NULL NULL YES FULLTEXT NO +set @old_innodb_ft_aux_table=@@global.innodb_ft_aux_table; +set global innodb_ft_aux_table='test/t1'; +insert into t1 values (1, '{"tags": [1, "abcde", "34567", "", 34567]}'); +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION +31xx 1 1 1 1 0 +3334353637 1 1 1 1 16 +3334353637 1 1 1 1 16 +6162636465 1 1 1 1 5 +xxxx 1 1 1 1 27 +select json_contains(j->'$.tags', '"abcde"') from t1; +json_contains(j->'$.tags', '"abcde"') +1 +explain +select * from t1 where json_contains(j->'$.tags', '"abcde"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 fulltext idx idx 0 1 Using where +select * from t1 where json_contains(j->'$.tags', '"abcde"'); +c j +1 {"tags": [1, "abcde", "34567", "", 34567]} +DROP TABLE t1; +set global innodb_ft_aux_table=@old_innodb_ft_aux_table; +# top level or +create table t1 (c int, j1 json, j2 json, key idx1 ((CAST(j1->'$.tags' AS CHAR(6) ARRAY))), key idx2 ((CAST(j2->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +explain +select * from t1 where json_contains(j1->'$.tags', '"abcde"') or json_contains(j2->'$.tags', '"abcde"'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 1 Using where +DROP TABLE t1; +# direct call of mvi_encode +select mvi_encode('[1, 42, "3"]', int); +mvi_encode('[1, 42, "3"]', int) +8000000000000001 800000000000002a 8000000000000003 +select mvi_encode('[1, 42, "3"]', unsigned); +mvi_encode('[1, 42, "3"]', unsigned) +0000000000000001 000000000000002a 0000000000000003 +select mvi_encode('[1, 42, "3 "]', char(6)); +mvi_encode('[1, 42, "3 "]', char(6)) +31xx 3432 33xx +select mvi_encode('[1, 42, " "]', char(6)); +mvi_encode('[1, 42, " "]', char(6)) +31xx 3432 xxxx +select mvi_encode('[1, 42, "3 "]', binary(6)); +mvi_encode('[1, 42, "3 "]', binary(6)) +31xx 3432 33xx diff --git a/mysql-test/main/multi_valued_index.test b/mysql-test/main/multi_valued_index.test new file mode 100644 index 0000000000000..1da784c0e4fc4 --- /dev/null +++ b/mysql-test/main/multi_valued_index.test @@ -0,0 +1,41 @@ +--source include/have_debug.inc +--source include/have_innodb.inc + +--echo # basic test +SET SESSION debug_dbug = '+d,test_invisible_index,test_completely_invisible'; + +create table t1 (c int, j json, key idx ((CAST(j->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; +SHOW CREATE TABLE t1; +show index from t1; + +set @old_innodb_ft_aux_table=@@global.innodb_ft_aux_table; +set global innodb_ft_aux_table='test/t1'; + +insert into t1 values (1, '{"tags": [1, "abcde", "34567", "", 34567]}'); + +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; + +select json_contains(j->'$.tags', '"abcde"') from t1; +explain +select * from t1 where json_contains(j->'$.tags', '"abcde"'); +select * from t1 where json_contains(j->'$.tags', '"abcde"'); + +DROP TABLE t1; + +set global innodb_ft_aux_table=@old_innodb_ft_aux_table; + +--echo # top level or + +create table t1 (c int, j1 json, j2 json, key idx1 ((CAST(j1->'$.tags' AS CHAR(6) ARRAY))), key idx2 ((CAST(j2->'$.tags' AS CHAR(6) ARRAY))))engine=innodb; + +explain +select * from t1 where json_contains(j1->'$.tags', '"abcde"') or json_contains(j2->'$.tags', '"abcde"'); + +DROP TABLE t1; + +--echo # direct call of mvi_encode +select mvi_encode('[1, 42, "3"]', int); +select mvi_encode('[1, 42, "3"]', unsigned); +select mvi_encode('[1, 42, "3 "]', char(6)); +select mvi_encode('[1, 42, " "]', char(6)); +select mvi_encode('[1, 42, "3 "]', binary(6)); diff --git a/mysql-test/suite/perfschema/r/digest_view.result b/mysql-test/suite/perfschema/r/digest_view.result index 346b5e0c44e60..b7954b50a12ba 100644 --- a/mysql-test/suite/perfschema/r/digest_view.result +++ b/mysql-test/suite/perfschema/r/digest_view.result @@ -191,17 +191,17 @@ SELECT SCHEMA_NAME, DIGEST, DIGEST_TEXT, COUNT_STAR FROM performance_schema.events_statements_summary_by_digest ORDER BY DIGEST_TEXT; SCHEMA_NAME DIGEST DIGEST_TEXT COUNT_STAR -test cc5e38c5a702f49627052e59a2603818 EXPLAIN SELECT * FROM `test` . `v1` 1 -test 264b69debfd30bbfe374cdca018fa4f9 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 1 -test 75f31bf60b75a4f851ff9c9ee4e19d96 EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 1 -test a8f8a85697afacda9f2f3d3b023f9ed0 EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 1 -test 512598992d37826136f3f1292bb32e1e EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 1 -test 54342cc19df16ce54b3ba43d1fd7552d SELECT * FROM `test` . `v1` 1 -test 3750b7d8c33e040b90a2cdccb4c642a3 SELECT * FROM `test` . `v1` WHERE `a` = ? 1 -test 94af70ef76a31364845f534863d99da8 SELECT * FROM `test` . `v1` WHERE `b` > ? 1 -test 0c87d86b62e664a81a23d4f176fbb377 SELECT `a` , `b` FROM `test` . `v1` 1 -test de7e6f1350ff14a952b97b373d76c1a4 SELECT `b` , `a` FROM `test` . `v1` 1 -test 3468091e6d6ce474aded20beaec36b53 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 +test 983c118738efc2378bb2a15495c7e6c9 EXPLAIN SELECT * FROM `test` . `v1` 1 +test f34ab36bba7121e7953b5c6ca8f58e11 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 1 +test ee2356b394246ac8b7fc5faa97b0c1ad EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 1 +test 3e70e03c91932264badade3cb1f2e66e EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 1 +test 021f3cff6701828a7f017af34d29ce1c EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 1 +test d2e11df922514a25243fc04ebf839825 SELECT * FROM `test` . `v1` 1 +test c967f871a7c1bc86a7667b83a17ca6e1 SELECT * FROM `test` . `v1` WHERE `a` = ? 1 +test 7c87fc30bf915208e69006449a702b7a SELECT * FROM `test` . `v1` WHERE `b` > ? 1 +test 258f5bff705a10fe1c43fd25c29c61fd SELECT `a` , `b` FROM `test` . `v1` 1 +test 518f87edc2c96f2b022d6ab3c767df7b SELECT `b` , `a` FROM `test` . `v1` 1 +test d972cde0f9bbec28c84e4dcca8ef3281 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 DROP TABLE test.v1; CREATE VIEW test.v1 AS SELECT * FROM test.t1; EXPLAIN SELECT * from test.v1; @@ -248,19 +248,19 @@ SELECT SCHEMA_NAME, DIGEST, DIGEST_TEXT, COUNT_STAR FROM performance_schema.events_statements_summary_by_digest ORDER BY DIGEST_TEXT; SCHEMA_NAME DIGEST DIGEST_TEXT COUNT_STAR -test ef8b10f452e117fa96af49687a93a10f CREATE VIEW `test` . `v1` AS SELECT * FROM `test` . `t1` 1 -test 6b99f4d2ad410f9fa4ee1d501b0db571 DROP TABLE `test` . `v1` 1 -test cc5e38c5a702f49627052e59a2603818 EXPLAIN SELECT * FROM `test` . `v1` 2 -test 264b69debfd30bbfe374cdca018fa4f9 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 2 -test 75f31bf60b75a4f851ff9c9ee4e19d96 EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 2 -test a8f8a85697afacda9f2f3d3b023f9ed0 EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 2 -test 512598992d37826136f3f1292bb32e1e EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 2 -test 54342cc19df16ce54b3ba43d1fd7552d SELECT * FROM `test` . `v1` 2 -test 3750b7d8c33e040b90a2cdccb4c642a3 SELECT * FROM `test` . `v1` WHERE `a` = ? 2 -test 94af70ef76a31364845f534863d99da8 SELECT * FROM `test` . `v1` WHERE `b` > ? 2 -test 187a846fafe04b746eddaaab80b6a766 SELECT SCHEMA_NAME , `DIGEST` , `DIGEST_TEXT` , `COUNT_STAR` FROM `performance_schema` . `events_statements_summary_by_digest` ORDER BY `DIGEST_TEXT` 1 -test 0c87d86b62e664a81a23d4f176fbb377 SELECT `a` , `b` FROM `test` . `v1` 2 -test de7e6f1350ff14a952b97b373d76c1a4 SELECT `b` , `a` FROM `test` . `v1` 2 -test 3468091e6d6ce474aded20beaec36b53 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 +test bd4716657502ce7922d2ed7e20043b76 CREATE VIEW `test` . `v1` AS SELECT * FROM `test` . `t1` 1 +test 2d14a4845ee825e3a1a21bd5799cba3c DROP TABLE `test` . `v1` 1 +test 983c118738efc2378bb2a15495c7e6c9 EXPLAIN SELECT * FROM `test` . `v1` 2 +test f34ab36bba7121e7953b5c6ca8f58e11 EXPLAIN SELECT * FROM `test` . `v1` WHERE `a` = ? 2 +test ee2356b394246ac8b7fc5faa97b0c1ad EXPLAIN SELECT * FROM `test` . `v1` WHERE `b` > ? 2 +test 3e70e03c91932264badade3cb1f2e66e EXPLAIN SELECT `a` , `b` FROM `test` . `v1` 2 +test 021f3cff6701828a7f017af34d29ce1c EXPLAIN SELECT `b` , `a` FROM `test` . `v1` 2 +test d2e11df922514a25243fc04ebf839825 SELECT * FROM `test` . `v1` 2 +test c967f871a7c1bc86a7667b83a17ca6e1 SELECT * FROM `test` . `v1` WHERE `a` = ? 2 +test 7c87fc30bf915208e69006449a702b7a SELECT * FROM `test` . `v1` WHERE `b` > ? 2 +test aa329bcb29afc39d9793e4fd6e108722 SELECT SCHEMA_NAME , `DIGEST` , `DIGEST_TEXT` , `COUNT_STAR` FROM `performance_schema` . `events_statements_summary_by_digest` ORDER BY `DIGEST_TEXT` 1 +test 258f5bff705a10fe1c43fd25c29c61fd SELECT `a` , `b` FROM `test` . `v1` 2 +test 518f87edc2c96f2b022d6ab3c767df7b SELECT `b` , `a` FROM `test` . `v1` 2 +test d972cde0f9bbec28c84e4dcca8ef3281 TRUNCATE TABLE `performance_schema` . `events_statements_summary_by_digest` 1 DROP VIEW test.v1; DROP TABLE test.t1; diff --git a/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result b/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result index e4e2c8b294a3f..e78e21e19a9ca 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result +++ b/mysql-test/suite/perfschema/r/start_server_low_digest_sql_length.result @@ -8,5 +8,5 @@ SELECT 1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1 #################################### SELECT event_name, digest, digest_text, sql_text FROM events_statements_history_long; event_name digest digest_text sql_text -statement/sql/select c9e142fe40c43498607ca5310e11d2ce SELECT ? + ? + SELECT ... -statement/sql/truncate 506e3496d92689cd2367329dc7725165 TRUNCATE TABLE truncat... +statement/sql/select 3e19ece73d286977122b52fb027c180c SELECT ? + ? + SELECT ... +statement/sql/truncate 7ab36a94986dd98b93370b025166a946 TRUNCATE TABLE truncat... diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 429a85f9cac30..bd01fba5a41dd 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -122,7 +122,6 @@ SET (SQL_SOURCE opt_rewrite_remove_casefold.cc opt_sargable_left.cc opt_sum.cc - opt_vcol_substitution.cc ../sql-common/pack.c parse_file.cc password.c procedure.cc protocol.cc records.cc repl_failsafe.cc rpl_filter.cc session_tracker.cc @@ -199,6 +198,8 @@ SET (SQL_SOURCE json_table.cc proxy_protocol.cc backup.cc xa.cc socketpair.c socketpair.h + opt_multi_valued_index.h + opt_multi_valued_index.cc opt_vcol_substitution.h opt_vcol_substitution.cc opt_hints_parser.cc opt_hints_parser.h scan_char.h diff --git a/sql/item.h b/sql/item.h index dabe4ca1c8801..5d588a58d6911 100644 --- a/sql/item.h +++ b/sql/item.h @@ -2859,6 +2859,11 @@ class Item :public Value_source, DBUG_ASSERT(fixed()); return false; } + virtual Item *create_ft_for_mvi(THD *thd, List *vcol_fields) + { + return NULL; + } + protected: /* diff --git a/sql/item_func.h b/sql/item_func.h index cc0985c8d53b7..59d02828ff8c5 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -109,7 +109,8 @@ class Item_func :public Item_func_or_sum JSON_EXTRACT_FUNC, JSON_VALID_FUNC, ROWNUM_FUNC, CASE_SEARCHED_FUNC, // Used by ColumnStore/Spider CASE_SIMPLE_FUNC, // Used by ColumnStore/spider, - DATE_FUNC, YEAR_FUNC, SUBSTR_FUNC, LEFT_FUNC + DATE_FUNC, YEAR_FUNC, SUBSTR_FUNC, LEFT_FUNC, + MVI_ENCODE_FUNC }; /* diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 3ec85d8c1a980..95e8938a1557d 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -376,6 +376,7 @@ class Item_func_json_contains: public Item_bool_func } bool fix_length_and_dec(THD *thd) override; bool val_bool() override; + Item *create_ft_for_mvi(THD *thd, List *vcol_fields) override; protected: Item *shallow_copy(THD *thd) const override diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index a35749fc4dc1b..6c3f67ce9c670 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -2643,6 +2643,30 @@ class Item_temptable_rowid :public Item_str_func { return get_item_copy(thd, this); } }; +class Item_func_mvi_encode : public Item_str_ascii_func +{ + Lex_cast_type_st m_cast_type; + String tmp_js; + json_engine_t je; +public: + void print(String *str, enum_query_type query_type) override; + Item_func_mvi_encode(THD* thd, Item *expr, const Lex_cast_type_st &cast_type): + Item_str_ascii_func(thd, expr), m_cast_type(cast_type) {} + String *val_str_ascii(String *buf) override; + enum Functype functype() const override { return MVI_ENCODE_FUNC; } + LEX_CSTRING func_name_cstring() const override + { + static LEX_CSTRING name= {STRING_WITH_LEN("mvi_encode")}; + return name; + } + bool fix_length_and_dec(THD *thd) override; + Item *shallow_copy(THD *thd) const override + { + return get_item_copy(thd, this); + } + Lex_cast_type_st &cast_type() { return m_cast_type; } +}; + class Item_func_format_pico_time : public Item_str_ascii_func { diff --git a/sql/lex.h b/sql/lex.h index 61774df019f0d..d14a4dd7c636b 100644 --- a/sql/lex.h +++ b/sql/lex.h @@ -419,6 +419,7 @@ SYMBOL symbols[] = { { "MONITOR", SYM(MONITOR_SYM)}, { "MONTH", SYM(MONTH_SYM)}, { "MUTEX", SYM(MUTEX_SYM)}, + { "MVI_ENCODE", SYM(MVI_ENCODE_SYM)}, { "MYSQL", SYM(MYSQL_SYM)}, { "MYSQL_ERRNO", SYM(MYSQL_ERRNO_SYM)}, { "NAME", SYM(NAME_SYM)}, diff --git a/sql/opt_multi_valued_index.cc b/sql/opt_multi_valued_index.cc new file mode 100644 index 0000000000000..12875ca59eeb2 --- /dev/null +++ b/sql/opt_multi_valued_index.cc @@ -0,0 +1,410 @@ +/* + Copyright (c) 2026, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "mariadb.h" +#include "sql_select.h" +#include "item_func.h" + +void Item_func_mvi_encode::print(String *str, enum_query_type query_type) +{ + char buf[32]; + size_t length; + str->append(func_name_cstring()); + str->append('('); + args[0]->print(str, query_type); + str->append(','); + const Name name= m_cast_type.type_handler()->name(); + switch (m_cast_type.type_handler()->field_type()) + { + case MYSQL_TYPE_LONG_BLOB: + str->append(STRING_WITH_LEN("char")); + str->append('('); + length= (size_t) (longlong10_to_str(m_cast_type.length(), buf, -10) - buf); + str->append(buf, length); + str->append(')'); + break; + default: + str->append(name.ptr(), name.length()); + break; + } + /* TODO: this is copied from another print() implementation */ + if (decimals && decimals != NOT_FIXED_DEC) + { + str->append('('); + length= (size_t) (longlong10_to_str(decimals, buf, -10) - buf); + str->append(buf, length); + str->append(')'); + } + str->append(')'); +} + +/* TODO: this duplicates logic in Item_func_json_extract::val_int */ +static longlong json_value_to_longlong(enum json_value_types type, + CHARSET_INFO *cs, + char* value, int value_len) +{ + switch (type) + { + case JSON_VALUE_NUMBER: + case JSON_VALUE_STRING: + { + char *end; + int err; + return cs->strntoll(value, value_len, 10, &end, &err); + } + case JSON_VALUE_TRUE: + return 1; + default: + return 0; + }; +} + +/* Lifted from Type_handler method of the same name */ +static void store_sort_key_longlong(uchar *to, bool unsigned_flag, + longlong value) +{ + to[7]= (uchar) value; + to[6]= (uchar) (value >> 8); + to[5]= (uchar) (value >> 16); + to[4]= (uchar) (value >> 24); + to[3]= (uchar) (value >> 32); + to[2]= (uchar) (value >> 40); + to[1]= (uchar) (value >> 48); + to[0]= (uchar) (value >> 56) ^ (unsigned_flag ? 0 : 128); +} + +static void encode_mvi_key(json_engine_t *je, const Type_handler *cast_th, + CHARSET_INFO *cs, String *buf) +{ + enum_field_types cast_ftype= cast_th->field_type(); + bool is_unsigned= cast_th->is_unsigned(); + StringBuffer<42> sorted; + /* 1. sort_string */ + sorted.length(0); + /* TODO: handle temporal types and decimal */ + switch(cast_ftype) + { + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_LONGLONG: + store_sort_key_longlong( + (uchar *) sorted.c_ptr(), is_unsigned, + json_value_to_longlong(je->value_type, cs, + (char *) je->value, je->value_len)); + sorted.length(8); + break; + /* TODO: unquote? */ + /* CHAR(n) => LONG BLOB */ + case MYSQL_TYPE_LONG_BLOB: + { + /* Trim trailing whitespaces if possible */ + if (!(cs->state & MY_CS_NOPAD)) + je->value_len= (int) cs->lengthsp((const char *) je->value, + je->value_len); + if (my_binary_compare(cs)) + sorted.set((char *) je->value, je->value_len, + &my_charset_latin1_bin); + else + { + my_strnxfrm_ret_t rc= cs->strnxfrm( + (uchar *) sorted.c_ptr(), 42, 42, je->value, je->value_len, 0); + sorted.length(rc.m_result_length); + } + break; + } + default: + break; + } + + /* 2. hex */ + buf->append_hex(sorted.c_ptr(), sorted.length()); + + /* 3. pad */ + if (sorted.length() == 0) + buf->append(STRING_WITH_LEN("xxxx")); + else if (sorted.length() == 1) + buf->append(STRING_WITH_LEN("xx")); + + /* 4. space */ + buf->append(STRING_WITH_LEN(" ")); +} + +String *Item_func_mvi_encode::val_str_ascii(String *buf) +{ + String *value= args[0]->val_json(&tmp_js); + CHARSET_INFO *cs= value->charset(); + const Type_handler *cast_th= m_cast_type.type_handler(); + bool end_ok= false; + const uchar *start= reinterpret_cast(value->ptr()); + const uchar *end= start + value->length(); + DBUG_ASSERT(fixed()); + buf->length(0); + buf->set_charset(&my_charset_latin1_bin); + + if (json_scan_start(&je, cs, start, end) || + json_read_value(&je)) + goto json_error; + + if (je.value_type != JSON_VALUE_ARRAY) + goto error_format; + + /* TODO: deduplicate, so that ["34567", 34567] yield only one token */ + do { + switch (je.state) + { + case JST_ARRAY_START: + continue; + case JST_ARRAY_END: + buf->length(buf->length() - 1); + end_ok = true; + break; + case JST_VALUE: + { + if (json_read_value(&je)) + goto json_error; + + encode_mvi_key(&je, cast_th, cs, buf); + break; + } + default: + goto error_format; + } + } while (json_scan_next(&je) == 0); + + if (end_ok) + return buf; + +error_format: + { + int position= (int) ((const char *) je.s.c_str - value->ptr()); + /* TODO: fix error */ + push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, + ER_VECTOR_FORMAT_INVALID, ER(ER_VECTOR_FORMAT_INVALID), + position, value->c_ptr_safe()); + null_value= true; + return nullptr; + } + +json_error: + report_json_error_ex(value->ptr(), &je, func_name(), + 0, Sql_condition::WARN_LEVEL_WARN); + null_value= true; + return nullptr; +} + +bool Item_func_mvi_encode::fix_length_and_dec(THD *thd) +{ + /* TODO: validate args[0] is a json array */ + mem_root_dynamic_array_init(thd->mem_root, PSI_INSTRUMENT_MEM, + &je.stack, sizeof(int), NULL, + JSON_DEPTH_DEFAULT, JSON_DEPTH_INC, MYF(0)); + decimals= 0; + fix_length_and_charset(args[0]->max_char_length() * 2, + &my_charset_latin1_bin); + set_maybe_null(); + return false; +} + +static +bool collect_mvi_vcols_for_join(JOIN *join, List *vcol_fields) +{ + List_iterator ti(join->select_lex->leaf_tables); + TABLE_LIST *tl; + TABLE *table; + while ((tl= ti++)) + { + if (!(table= tl->table)) // non-merged semi-join or something like that + continue; + // TODO: Make use of iterator to loop through + // keys_in_use_for_query, instead. + for (uint i=0; i < table->s->keys; i++) + { + // note: we could also support histograms here + if (!table->keys_in_use_for_query.is_set(i)) + continue; + + KEY *key= &table->key_info[i]; + for (uint kp=0; kp < key->user_defined_key_parts; kp++) + { + Field *field= key->key_part[kp].field; + if (field->invisible == INVISIBLE_FULL && + field->vcol_info && + field->vcol_info->expr->type() == Item::FUNC_ITEM && + ((Item_func *) field->vcol_info->expr)->functype() == + Item_func::MVI_ENCODE_FUNC && + vcol_fields->push_back(field)) + return TRUE; // Out of memory + } + } + } + return FALSE; // Ok +} + +class Mvi_context +{ + public: + THD *thd; + /* Virtual columns with fulltext index that we can try substituting */ + List vcol_fields; + + Mvi_context(THD *thd_arg) : thd(thd_arg) {} +}; + +Item *Item_func_json_contains::create_ft_for_mvi(THD *thd, + List *vcol_fields) +{ + List_iterator it(*vcol_fields); + Field *vcol_field; + CHARSET_INFO *cs; + Item_func_mvi_encode *mvitem; + DBUG_ASSERT(fixed()); + if (arg_count > 2 || !a2_constant) + return NULL; + while ((vcol_field= it++)) + { + DBUG_ASSERT(vcol_field->vcol_info->expr->type() == FUNC_ITEM); + DBUG_ASSERT(((Item_func *) vcol_field->vcol_info->expr)->functype() == + MVI_ENCODE_FUNC); + mvitem= (Item_func_mvi_encode *) vcol_field->vcol_info->expr; + if (mvitem->arguments()[0]->eq(args[0], true)) + { + cs= mvitem->arguments()[0]->collation.collation; + break; + } + } + if (!vcol_field) + return NULL; + + const Type_handler *cast_th= mvitem->cast_type().type_handler(); + StringBuffer<42> sorted; + StringBuffer<256> buf; + const uchar *start, *end; + List ifm_args; + Item_field *ivcol; + Item_string *ift_query; + DBUG_ASSERT(fixed()); + buf.length(0); + buf.set_charset(&my_charset_latin1_bin); + if (!a2_parsed) + { + val= args[1]->val_json(&tmp_val); + a2_parsed= true; + } + if (!val) + return NULL; + start= reinterpret_cast(val->ptr()); + end= start + val->length(); + + if (json_scan_start(&je, cs, start, end) || json_read_value(&je)) + return NULL; + + if (je.value_type == JSON_VALUE_UNINITIALIZED || + je.value_type == JSON_VALUE_OBJECT) + return NULL; + if (je.value_type != JSON_VALUE_ARRAY) + { + /* scalar */ + encode_mvi_key(&je, cast_th, cs, &buf); + buf.length(buf.length() - 1); + goto ok; + } + + /* TODO: deduplicate? */ + do { + switch (je.state) + { + /* TODO: nested array? */ + case JST_ARRAY_START: + continue; + case JST_ARRAY_END: + buf.length(buf.length() - 1); + break; + case JST_VALUE: + { + if (json_read_value(&je)) + return NULL; + + buf.append('+'); + encode_mvi_key(&je, cast_th, cs, &buf); + break; + } + default: + return NULL; + } + } while (json_scan_next(&je) == 0); + +ok: + ift_query= new (thd->mem_root) Item_string(thd, &my_charset_latin1_bin, + buf.c_ptr(), buf.length()); + ifm_args.push_back(ift_query); + ivcol= new (thd->mem_root) Item_field(thd, vcol_field); + ifm_args.push_back(ivcol); + return new (thd->mem_root) Item_func_match(thd, ifm_args, FT_BOOL); +} + +static bool add_ft_for_mvi(Mvi_context *ctx, Item **conds_ref, + List *ftfunc_list) +{ + Item *conds= *conds_ref; + Item *cond, *match; + List matches; + THD *thd= ctx->thd; + if (conds->type() != Item::COND_ITEM) + { + if ((match= conds->create_ft_for_mvi(thd, &ctx->vcol_fields))) + { + matches.push_back(match); + ftfunc_list->push_back((Item_func_match *) match); + } + } + else if (((Item_cond *) conds)->functype() == Item_func::COND_OR_FUNC) + return false; + else + { + List_iterator it(*((Item_cond *) conds)->argument_list()); + while ((cond= it++)) + { + if ((match= cond->create_ft_for_mvi(thd, &ctx->vcol_fields))) + { + matches.push_back(match); + ftfunc_list->push_back((Item_func_match *) match); + } + } + } + if (matches.elements == 1) + cond= matches.pop(); + else + cond= new (thd->mem_root) Item_cond_and(thd, matches); + if (cond && + ((cond->fix_fields(thd, &cond) || + !(conds= and_items(thd, conds, cond)) || + conds->fix_fields(thd, &conds)))) + return true; + *conds_ref= conds; + return false; +} + +bool setup_mvi_for_join(JOIN *join) +{ + Mvi_context ctx(join->thd); + if (collect_mvi_vcols_for_join(join, &ctx.vcol_fields)) + return true; + if (!ctx.vcol_fields.is_empty() && join->conds) + return add_ft_for_mvi(&ctx, &join->conds, join->select_lex->ftfunc_list); + return false; +} diff --git a/sql/opt_multi_valued_index.h b/sql/opt_multi_valued_index.h new file mode 100644 index 0000000000000..76a2c4a275b52 --- /dev/null +++ b/sql/opt_multi_valued_index.h @@ -0,0 +1,17 @@ +/* + Copyright (c) 2026, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +bool setup_mvi_for_join(JOIN *join); diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 47de84e48d131..7d488ff341ebf 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -170,7 +170,8 @@ Key::Key(const Key &rhs, MEM_ROOT *mem_root) columns(rhs.columns, mem_root), name(rhs.name), option_list(rhs.option_list), - generated(rhs.generated), invisible(false), + generated(rhs.generated), + invisible(rhs.invisible), without_overlaps(rhs.without_overlaps), old(rhs.old), length(rhs.length), period(rhs.period) { diff --git a/sql/sql_select.cc b/sql/sql_select.cc index dc9e7b6113333..8f7c0987c7c7a 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -2309,6 +2309,12 @@ JOIN::optimize_inner() optimize_schema_tables_memory_usage(select_lex->leaf_tables)) DBUG_RETURN(1); + if (setup_mvi_for_join(this)) + { + error= 1; + DBUG_RETURN(1); + } + if (setup_ftfuncs(select_lex)) /* should be after having->fix_fields */ DBUG_RETURN(-1); diff --git a/sql/sql_select.h b/sql/sql_select.h index 23a927bfcb14e..d1e97994915ad 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -3035,5 +3035,6 @@ void propagate_new_equalities(THD *thd, Item *cond, bool dbug_user_var_equals_str(THD *thd, const char *name, const char *value); #include "opt_vcol_substitution.h" +#include "opt_multi_valued_index.h" #endif /* SQL_SELECT_INCLUDED */ diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 54a604e404d47..1725697c65243 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -116,8 +116,6 @@ static Lex_ident_column make_unique_key_name(THD *, static bool make_unique_constraint_name(THD *, LEX_CSTRING *, const char *, List *, List *, uint *); -static Lex_ident_column make_internal_field_name(THD *, const char *, - List *); static int copy_data_between_tables(THD *, TABLE *,TABLE *, bool, uint, ORDER *, ha_rows *, ha_rows *, @@ -2775,7 +2773,7 @@ static int mysql_add_invisible_field(THD *thd, List * field_list, #define INTERNAL_FIELD_NAME_LENGTH 30 -static Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, +Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, List *create_list) { char buf[INTERNAL_FIELD_NAME_LENGTH]= {0}; diff --git a/sql/sql_table.h b/sql/sql_table.h index 3a27204f2c45b..171a7aa075ec9 100644 --- a/sql/sql_table.h +++ b/sql/sql_table.h @@ -226,4 +226,6 @@ extern MYSQL_PLUGIN_IMPORT const Lex_ident_column primary_key_name; bool check_engine(THD *, const char *, const char *, HA_CREATE_INFO *); +Lex_ident_column make_internal_field_name(THD *thd, const char *prefix, + List *create_list); #endif /* SQL_TABLE_INCLUDED */ diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index c2fe9974c42df..78e7905e9d897 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1011,6 +1011,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %token MONITOR_SYM /* MariaDB privilege */ %token MONTH_SYM /* SQL-2003-R */ %token MUTEX_SYM +%token MVI_ENCODE_SYM %token MYSQL_SYM %token MYSQL_ERRNO_SYM %token NAMES_SYM /* SQL-2003-N */ @@ -1761,7 +1762,7 @@ rule: using_list opt_use_partition use_partition %type - key_part key_part_simple + key_part key_part_simple multi_valued_key_part %type join_table_list join_table @@ -7636,6 +7637,7 @@ opt_without_overlaps: key_part: key_part_simple + | multi_valued_key_part | ident '(' NUM ')' { int key_part_len= atoi($3.str); @@ -7647,6 +7649,38 @@ key_part: } ; +multi_valued_key_part: + '(' CAST_SYM '(' expr AS cast_type ARRAY_SYM ')' ')' + { + /* TODO: check fts_min_token_size is 4, warn if not */ + /* Create a Create_field */ + Create_field *f= new (thd->mem_root) Create_field(); + LEX_CSTRING fname= make_internal_field_name(thd, "DB_MVI_", &Lex->alter_info.create_list); + Item *vcol_expr= + new (thd->mem_root) Item_func_mvi_encode(thd, $4, $6); + + if (unlikely(!f)) + MYSQL_YYABORT; + + f->invisible= INVISIBLE_FULL; + Lex->last_key->invisible= true; + f->set_handler(&type_handler_blob); + f->charset= &my_charset_latin1_bin; + Lex->last_key->type= Key::FULLTEXT; + Lex->init_last_field(f, &fname); + Lex->alter_info.create_list.push_back(f, thd->mem_root); + + /* Create a vcol */ + Virtual_column_info *v= add_virtual_expression(thd, vcol_expr); + if (unlikely(!v)) + MYSQL_YYABORT; + Lex->last_field->vcol_info= v; + Lex->last_field->vcol_info->set_vcol_type(VCOL_GENERATED_STORED); + + $$= new (thd->mem_root) Key_part_spec(&fname, 0, /*gen=*/true); + } + ; + key_part_simple: ident { @@ -11146,6 +11180,12 @@ function_call_nonkeyword: MYSQL_YYABORT; Lex->safe_to_cache_query= false; } + | MVI_ENCODE_SYM '(' expr ',' cast_type ')' + { + $$= new (thd->mem_root) Item_func_mvi_encode(thd, $3, $5); + if (unlikely($$ == NULL)) + MYSQL_YYABORT; + } | NOW_SYM opt_time_precision { $$= new (thd->mem_root) Item_func_current_timestamp(thd, $2); @@ -16907,6 +16947,7 @@ keyword_sp_var_not_label: | HELP_SYM | HOST_SYM | INSTALL_SYM + | MVI_ENCODE_SYM | OPTION | OPTIONS_SYM | OTHERS_MARIADB_SYM