diff --git a/sql/handler.cc b/sql/handler.cc index 269115bc0da86..05bd7459e7a27 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -3379,6 +3379,40 @@ int handler::ha_close(void) DBUG_RETURN(close()); } +int handler::ha_sample_next(uchar *buf) +{ + int result; + DBUG_ENTER("handler::ha_sample_next"); + DBUG_ASSERT(table_share->tmp_table != NO_TMP_TABLE || + m_lock_type != F_UNLCK); + DBUG_ASSERT(inited == SAMPLING); + + do + { + TABLE_IO_WAIT(tracker, PSI_TABLE_FETCH_ROW, MAX_KEY, result, + { result= sample_next(buf); }) + if (result != HA_ERR_RECORD_DELETED && result != HA_ERR_KEY_NOT_FOUND) + break; + //status_var_increment(table->in_use->status_var.ha_read_rnd_deleted_count); + } while (!table->in_use->check_killed(1)); + + if (result == HA_ERR_RECORD_DELETED) + result= HA_ERR_ABORTED_BY_USER; + else + { + if (!result) + { + update_rows_read(); + if (table->vfield && buf == table->record[0]) + table->update_virtual_fields(this, VCOL_UPDATE_FOR_READ); + } + //increment_statistics(&SSV::ha_read_rnd_next_count); + } + + table->status=result ? STATUS_NOT_FOUND: 0; + DBUG_RETURN(result); +} + int handler::ha_rnd_next(uchar *buf) { diff --git a/sql/handler.h b/sql/handler.h index 8ad521e189a4f..c3f2908daf803 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -366,7 +366,10 @@ enum chf_create_flags { /* Implements SELECT ... FOR UPDATE SKIP LOCKED */ #define HA_CAN_SKIP_LOCKED (1ULL << 61) -#define HA_LAST_TABLE_FLAG HA_CAN_SKIP_LOCKED +/* Implements native sampling (MDEV-19556) */ +#define HA_NATIVE_SAMPLING (1ULL << 62) + +#define HA_LAST_TABLE_FLAG HA_NATIVE_SAMPLING /* bits in index_flags(index_number) for what you can do with index */ @@ -3245,7 +3248,7 @@ class handler :public Sql_alloc /** Length of ref (1-8 or the clustered key length) */ uint ref_length; FT_INFO *ft_handler; - enum init_stat { NONE=0, INDEX, RND }; + enum init_stat { NONE=0, INDEX, RND, SAMPLING }; init_stat inited, pre_inited; const COND *pushed_cond; @@ -3506,6 +3509,25 @@ class handler :public Sql_alloc } int ha_rnd_init_with_error(bool scan) __attribute__ ((warn_unused_result)); int ha_reset(); + int ha_sample_init() + { + int result; + DBUG_ENTER("ha_sample_init"); + DBUG_ASSERT(inited == NONE); + result= sample_init(); + inited= result ? NONE: SAMPLING; + end_range= NULL; + DBUG_RETURN(result); + } + int ha_sample_next(uchar* buf); + int ha_sample_end() + { + DBUG_ENTER("ha_sample_end"); + DBUG_ASSERT(inited==SAMPLING); + inited= NONE; + end_range= NULL; + DBUG_RETURN(sample_end()); + } /* this is necessary in many places, e.g. in HANDLER command */ int ha_index_or_rnd_end() { @@ -4030,6 +4052,11 @@ class handler :public Sql_alloc virtual int ft_read(uchar *buf) { return HA_ERR_WRONG_COMMAND; } virtual int rnd_next(uchar *buf)=0; virtual int rnd_pos(uchar * buf, uchar *pos)=0; + + virtual int sample_init() {return 0;} + virtual int sample_next(uchar *buf) { return 0;} + virtual int sample_end() {return 0;} + /** This function only works for handlers having HA_PRIMARY_KEY_REQUIRED_FOR_POSITION set. diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index 1f034f490c8e2..2bfa022e65abb 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -2650,29 +2650,28 @@ int collect_statistics_for_table(THD *thd, TABLE *table) for (field_ptr= table->field; *field_ptr; field_ptr++) { - table_field= *field_ptr; + table_field= *field_ptr; if (!table_field->collected_stats) - continue; + continue; table_field->collected_stats->init(thd, table_field); } restore_record(table, s->default_values); - /* Perform a full table scan to collect statistics on 'table's columns */ - if (!(rc= file->ha_rnd_init(TRUE))) + if(file->ha_table_flags() & HA_NATIVE_SAMPLING) { - DEBUG_SYNC(table->in_use, "statistics_collection_start"); - - while ((rc= file->ha_rnd_next(table->record[0])) != HA_ERR_END_OF_FILE) - { - if (thd->killed) - break; - - if (rc) - break; + if(!(rc= file->ha_sample_init())) { + DEBUG_SYNC(table->in_use, "statistics_collection_start"); + rows = file->records() * (thd->variables.sample_percentage / 100); - if (thd_rnd(thd) <= sample_fraction) + for (ulonglong i= 0; i < rows; ++i) { + rc = file->ha_sample_next(table->record[0]); + if (thd->killed) + break; + + if (rc) + break; for (field_ptr= table->field; *field_ptr; field_ptr++) { table_field= *field_ptr; @@ -2683,12 +2682,47 @@ int collect_statistics_for_table(THD *thd, TABLE *table) } if (rc) break; - rows++; } + file->ha_sample_end(); } - file->ha_rnd_end(); + rc= (rc == 0 && !thd->killed) ? 0 : 1; } - rc= (rc == HA_ERR_END_OF_FILE && !thd->killed) ? 0 : 1; + else + { + + /* Perform a full table scan to collect statistics on 'table's columns */ + if (!(rc= file->ha_rnd_init(TRUE))) + { + DEBUG_SYNC(table->in_use, "statistics_collection_start"); + + while ((rc= file->ha_rnd_next(table->record[0])) != HA_ERR_END_OF_FILE) + { + if (thd->killed) + break; + + if (rc) + break; + + if (thd_rnd(thd) <= sample_fraction) + { + for (field_ptr= table->field; *field_ptr; field_ptr++) + { + table_field= *field_ptr; + if (!table_field->collected_stats) + continue; + if ((rc= table_field->collected_stats->add())) + break; + } + if (rc) + break; + rows++; + } + } + file->ha_rnd_end(); + } + rc= (rc == HA_ERR_END_OF_FILE && !thd->killed) ? 0 : 1; + } + /* Calculate values for all statistical characteristics on columns and diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 03862caa62790..128afeb9ff709 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -2849,14 +2849,18 @@ btr_cur_open_at_index_side( /**********************************************************************//** Positions a cursor at a randomly chosen position within a B-tree. -@return true if the index is available and we have put the cursor, false -if the index is unavailable */ -bool +@return DB_SUCCESS if the index is available and we have put the cursor, +error code if the index is unavailable. If simulate_uniform=true, could be +DB_RECORD_NOT_FOUND returned, which means that no record is chosen in the +generated tree path. The caller should retry a call, that will +try a new tree path */ +dberr_t btr_cur_open_at_rnd_pos( - dict_index_t* index, /*!< in: index */ - ulint latch_mode, /*!< in: BTR_SEARCH_LEAF, ... */ - btr_cur_t* cursor, /*!< in/out: B-tree cursor */ - mtr_t* mtr) /*!< in: mtr */ + dict_index_t* index, /*!< in: index */ + ulint latch_mode, /*!< in: BTR_SEARCH_LEAF, ... */ + btr_cur_t* cursor, /*!< in/out: B-tree cursor */ + mtr_t* mtr, /*!< in: mtr */ + bool sim_uniform_dist) /*!< in: uniform distribution simulation */ { page_cur_t* page_cursor; ulint node_ptr_max_size = srv_page_size / 2; @@ -2868,6 +2872,11 @@ btr_cur_open_at_rnd_pos( ulint n_blocks = 0; ulint n_releases = 0; mem_heap_t* heap = NULL; + /* For uniform distribution simulation is used naive A/R sampling + algorithm, check link below for details + https://courses.cs.washington.edu/courses/cse590q/05au/papers/Olken-Sampling.pdf + */ + double p = 1.0; rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; rec_offs* offsets = offsets_; rec_offs_init(offsets_); @@ -2917,7 +2926,7 @@ btr_cur_open_at_rnd_pos( } DBUG_EXECUTE_IF("test_index_is_unavailable", - return(false);); + return(DB_ERROR);); if (index->page == FIL_NULL) { /* Since we don't hold index lock until just now, the index @@ -2925,7 +2934,7 @@ btr_cur_open_at_rnd_pos( statistics updater for referenced table, it could be marked as unavailable by 'DROP TABLE' in the mean time, since we don't hold lock for statistics updater */ - return(false); + return(DB_ERROR); } const rw_lock_type_t root_leaf_rw_latch = btr_cur_latch_for_root_leaf( @@ -3003,8 +3012,18 @@ btr_cur_open_at_rnd_pos( /* We are in the root node */ height = btr_page_get_level(page); + } else { + if(sim_uniform_dist) { + ulint n_recs = page_get_n_recs(block->page.frame); + int extra_bytes = dict_table_is_comp(index->table) ? + REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES; + // Get max fanout from n_recs / (srv_page_size / + // extra_bytes +1), but exchange two divisons by + // one multiplication and one division + p *= (double)n_recs * (extra_bytes + 1) / + (double)srv_page_size; + } } - if (height == 0) { if (rw_latch == RW_NO_LATCH || srv_read_only_mode) { @@ -3038,7 +3057,7 @@ btr_cur_open_at_rnd_pos( } } - page_cur_open_on_rnd_user_rec(block, page_cursor); + page_cur_open_on_rnd_user_rec(block, page_cursor); if (height == 0) { @@ -3133,7 +3152,14 @@ btr_cur_open_at_rnd_pos( mem_heap_free(heap); } - return err == DB_SUCCESS; + // We get max value of type by using ~(type)0 for + // getting 0..1 pseudo random number from ut_rnd_gen() + // and exchange division by multiplication like + // (b / c) < a <=> b < (a * c) + if(sim_uniform_dist && (ut_rnd_gen() < (p * ~(uint32_t)0))) + err = DB_RECORD_NOT_FOUND; + + return err; } /*==================== B-TREE INSERT =========================*/ diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index fd744e6e2f52f..b720d9a422b79 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1224,12 +1224,12 @@ btr_estimate_number_of_different_key_vals(dict_index_t* index, for (i = 0; i < n_sample_pages; i++) { mtr.start(); - bool available; + dberr_t available; available = btr_cur_open_at_rnd_pos(index, BTR_SEARCH_LEAF, - &cursor, &mtr); + &cursor, &mtr, false); - if (!available || index->table->bulk_trx_id != bulk_trx_id) { + if (available != DB_SUCCESS || index->table->bulk_trx_id != bulk_trx_id) { mtr.commit(); mem_heap_free(heap); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 17a92a242fd41..aedb35dad7549 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -113,6 +113,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include "fil0pagecompress.h" #include "ut0mem.h" #include "row0ext.h" +#include "row0sel.h" #include "lz4.h" #include "lzo/lzo1x.h" @@ -3020,7 +3021,8 @@ ha_innobase::ha_innobase( | HA_CAN_ONLINE_BACKUPS | HA_CONCURRENT_OPTIMIZE | HA_CAN_SKIP_LOCKED - | (srv_force_primary_key ? HA_REQUIRE_PRIMARY_KEY : 0) + | (srv_force_primary_key ? HA_REQUIRE_PRIMARY_KEY : 0 + | HA_NATIVE_SAMPLING) ), m_start_of_scan(), m_mysql_has_locked() @@ -9489,6 +9491,66 @@ ha_innobase::rnd_next( DBUG_RETURN(error); } +int +ha_innobase::sample_init() +{ + int err; + + /* Store the active index value so that we can restore the original + value after a scan */ + + if (m_prebuilt->clust_index_was_generated) { + err = change_active_index(MAX_KEY); + } else { + err = change_active_index(m_primary_key); + } + + return(err); +} + +int +ha_innobase::sample_end() +{ + return(index_end()); +} +int +ha_innobase::sample_next( +/*=====================*/ + uchar *buf) +{ + mtr_t mtr; + btr_pcur_t* pcur= m_prebuilt->pcur; + rec_t* rec; + bool res; + dberr_t err; + rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; + rec_offs* offsets= offsets_; + rec_offs_init(offsets_); + + dict_index_t* index= innobase_get_index(MAX_KEY); + mtr.start(); + err= btr_pcur_open_at_rnd_pos(index, BTR_SEARCH_LEAF, pcur, &mtr, true); + mtr.commit(); + if(err != DB_SUCCESS) + { + return HA_ERR_KEY_NOT_FOUND; + } + rec= btr_pcur_get_rec(pcur); + mem_heap_t* heap= NULL; + auto _ = make_scope_exit([heap]() { if(heap) mem_heap_free(heap); }); + + offsets= rec_get_offsets(rec, index, offsets, index->n_core_fields, ULINT_UNDEFINED, &heap); + ut_ad(offsets); + if (!offsets) + return HA_ERR_INTERNAL_ERROR; + + res= row_sel_store_mysql_rec( + buf, m_prebuilt, rec, NULL, true, + index, offsets); + + return res ? 0 : HA_ERR_INTERNAL_ERROR; +} + /**********************************************************************//** Fetches a row from the table based on a row reference. @return 0, HA_ERR_KEY_NOT_FOUND, or error code */ diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 08501859ec908..cd2df33b30ea1 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -155,6 +155,12 @@ class ha_innobase final : public handler int rnd_next(uchar *buf) override; + int sample_init() override; + + int sample_end() override; + + int sample_next(uchar *buf) override; + int rnd_pos(uchar * buf, uchar *pos) override; int ft_init() override; diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 2540d180fb02c..b08fc6a989307 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -2386,12 +2386,12 @@ ibuf_merge_pages( /* Open a cursor to a randomly chosen leaf of the tree, at a random position within the leaf */ - bool available; + dberr_t err; - available = btr_pcur_open_at_rnd_pos(ibuf.index, BTR_SEARCH_LEAF, - &pcur, &mtr); + err = btr_pcur_open_at_rnd_pos(ibuf.index, BTR_SEARCH_LEAF, + &pcur, &mtr, false); /* No one should make this index unavailable when server is running */ - ut_a(available); + ut_a(err == DB_SUCCESS); ut_ad(page_validate(btr_pcur_get_page(&pcur), ibuf.index)); diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index f5f1c97295712..45875f7a6e5f2 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -207,14 +207,18 @@ btr_cur_open_at_index_side( /**********************************************************************//** Positions a cursor at a randomly chosen position within a B-tree. -@return true if the index is available and we have put the cursor, false -if the index is unavailable */ -bool +@return DB_SUCCESS if the index is available and we have put the cursor, +error code if the index is unavailable. If simulate_uniform=true, could be +DB_RECORD_NOT_FOUND returned, which means that no record is chosen in the +generated tree path. The caller should retry a call, that will +try a new tree path */ +dberr_t btr_cur_open_at_rnd_pos( - dict_index_t* index, /*!< in: index */ - ulint latch_mode, /*!< in: BTR_SEARCH_LEAF, ... */ - btr_cur_t* cursor, /*!< in/out: B-tree cursor */ - mtr_t* mtr); /*!< in: mtr */ + dict_index_t* index, /*!< in: index */ + ulint latch_mode, /*!< in: BTR_SEARCH_LEAF, ... */ + btr_cur_t* cursor, /*!< in/out: B-tree cursor */ + mtr_t* mtr, /*!< in: mtr */ + bool sim_uniform_dist); /*!< in: uniform distribution simulation */ /*************************************************************//** Tries to perform an insert to a page in an index tree, next to cursor. It is assumed that mtr holds an x-latch on the page. The operation does diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h index f6636816e7290..eed0cb86deceb 100644 --- a/storage/innobase/include/btr0pcur.h +++ b/storage/innobase/include/btr0pcur.h @@ -200,15 +200,19 @@ btr_pcur_open_on_user_rec( mtr_t* mtr); /*!< in: mtr */ /**********************************************************************//** Positions a cursor at a randomly chosen position within a B-tree. -@return true if the index is available and we have put the cursor, false -if the index is unavailable */ +@return DB_SUCCESS if the index is available and we have put the cursor, +error code if the index is unavailable. If simulate_uniform=true, could be +DB_RECORD_NOT_FOUND returned, which means that no record is chosen in the +generated tree path. The caller should retry a call, that will +try a new tree path */ UNIV_INLINE -bool +dberr_t btr_pcur_open_at_rnd_pos( - dict_index_t* index, /*!< in: index */ - ulint latch_mode, /*!< in: BTR_SEARCH_LEAF, ... */ - btr_pcur_t* cursor, /*!< in/out: B-tree pcur */ - mtr_t* mtr); /*!< in: mtr */ + dict_index_t* index, /*!< in: index */ + ulint latch_mode, /*!< in: BTR_SEARCH_LEAF, ... */ + btr_pcur_t* cursor, /*!< in/out: B-tree pcur */ + mtr_t* mtr, /*!< in: mtr */ + bool sim_uniform_dist); /*!< in: uniform distribution simulation */ /**************************************************************//** Frees the possible memory heap of a persistent cursor and sets the latch mode of the persistent cursor to BTR_NO_LATCHES. diff --git a/storage/innobase/include/btr0pcur.inl b/storage/innobase/include/btr0pcur.inl index f5e59c7268e50..9e8ac32113763 100644 --- a/storage/innobase/include/btr0pcur.inl +++ b/storage/innobase/include/btr0pcur.inl @@ -479,15 +479,19 @@ btr_pcur_open_at_index_side( /**********************************************************************//** Positions a cursor at a randomly chosen position within a B-tree. -@return true if the index is available and we have put the cursor, false -if the index is unavailable */ +@return DB_SUCCESS if the index is available and we have put the cursor, +error code if the index is unavailable. If simulate_uniform=true, could be +DB_RECORD_NOT_FOUND returned, which means that no record is chosen in the +generated tree path. The caller should retry a call, that will +try a new tree path */ UNIV_INLINE -bool +dberr_t btr_pcur_open_at_rnd_pos( - dict_index_t* index, /*!< in: index */ - ulint latch_mode, /*!< in: BTR_SEARCH_LEAF, ... */ - btr_pcur_t* cursor, /*!< in/out: B-tree pcur */ - mtr_t* mtr) /*!< in: mtr */ + dict_index_t* index, /*!< in: index */ + ulint latch_mode, /*!< in: BTR_SEARCH_LEAF, ... */ + btr_pcur_t* cursor, /*!< in/out: B-tree pcur */ + mtr_t* mtr, /*!< in: mtr */ + bool sim_uniform_dist) /*!< in: uniform distribution simulation */ { /* Initialize the cursor */ @@ -496,17 +500,17 @@ btr_pcur_open_at_rnd_pos( btr_pcur_init(cursor); - bool available; + dberr_t err; - available = btr_cur_open_at_rnd_pos(index, latch_mode, + err = btr_cur_open_at_rnd_pos(index, latch_mode, btr_pcur_get_btr_cur(cursor), - mtr); + mtr, sim_uniform_dist); cursor->pos_state = BTR_PCUR_IS_POSITIONED; cursor->old_stored = false; cursor->trx_if_known = NULL; - return(available); + return err; } /**************************************************************//** diff --git a/storage/innobase/include/row0sel.h b/storage/innobase/include/row0sel.h index eb83a4bcad656..ddff736a83fed 100644 --- a/storage/innobase/include/row0sel.h +++ b/storage/innobase/include/row0sel.h @@ -480,3 +480,29 @@ row_sel_field_store_in_mysql_format_func( #include "row0sel.inl" #endif + +/** Convert a row in the Innobase format to a row in the MySQL format. +Note that the template in prebuilt may advise us to copy only a few +columns to mysql_rec, other columns are left blank. All columns may not +be needed in the query. +@param[out] mysql_rec row in the MySQL format +@param[in] prebuilt cursor +@param[in] rec Innobase record in the index + which was described in prebuilt's + template, or in the clustered index; + must be protected by a page latch +@param[in] vrow virtual columns +@param[in] rec_clust whether index must be the clustered index +@param[in] index index of rec +@param[in] offsets array returned by rec_get_offsets(rec) +@retval true on success +@retval false if not all columns could be retrieved */ +MY_ATTRIBUTE((warn_unused_result)) +bool row_sel_store_mysql_rec( + byte* mysql_rec, + row_prebuilt_t* prebuilt, + const rec_t* rec, + const dtuple_t* vrow, + bool rec_clust, + const dict_index_t* index, + const rec_offs* offsets); diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index f208ca51d69b3..19d30bd3f3bf0 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -3092,7 +3092,7 @@ be needed in the query. @retval true on success @retval false if not all columns could be retrieved */ MY_ATTRIBUTE((warn_unused_result)) -static bool row_sel_store_mysql_rec( +bool row_sel_store_mysql_rec( byte* mysql_rec, row_prebuilt_t* prebuilt, const rec_t* rec,