diff --git a/be/src/storage/data_dir.cpp b/be/src/storage/data_dir.cpp index ab7b56f610a724..490af91954b019 100644 --- a/be/src/storage/data_dir.cpp +++ b/be/src/storage/data_dir.cpp @@ -66,6 +66,7 @@ #include "storage/tablet/tablet_meta_manager.h" #include "storage/txn/txn_manager.h" #include "storage/utils.h" // for check_dir_existed +#include "util/debug_points.h" #include "util/string_util.h" #include "util/uid_util.h" @@ -108,6 +109,34 @@ Status _write_cluster_id_to_path(const std::string& path, int32_t cluster_id) { } // namespace +const char* tablet_path_gc_mode_name(TabletPathGcMode mode) { + switch (mode) { + case TabletPathGcMode::MOVE_TO_TRASH: + return "MOVE_TO_TRASH"; + case TabletPathGcMode::DELETE_DIRECTLY: + return "DELETE_DIRECTLY"; + } + LOG(FATAL) << "invalid tablet path gc mode=" << static_cast(mode); + __builtin_unreachable(); +} + +const char* tablet_path_gc_reason_name(TabletPathGcReason reason) { + switch (reason) { + case TabletPathGcReason::NORMAL_RETENTION: + return "NORMAL_RETENTION"; + case TabletPathGcReason::TRASH_RETENTION_DISABLED: + return "TRASH_RETENTION_DISABLED"; + case TabletPathGcReason::MANUAL_CLEAN_TRASH: + return "MANUAL_CLEAN_TRASH"; + case TabletPathGcReason::HIGH_DISK_WATERMARK: + return "HIGH_DISK_WATERMARK"; + case TabletPathGcReason::UNUSED_DATA_DIR: + return "UNUSED_DATA_DIR"; + } + LOG(FATAL) << "invalid tablet path gc reason=" << static_cast(reason); + __builtin_unreachable(); +} + DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(disks_total_capacity, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(disks_avail_capacity, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(disks_local_used_capacity, MetricUnit::BYTES); @@ -1061,14 +1090,29 @@ void DataDir::disks_compaction_num_increment(int64_t delta) { disks_compaction_num->increment(delta); } -Status DataDir::move_to_trash(const std::string& tablet_path) { - if (config::trash_file_expire_time_sec <= 0) { - LOG(INFO) << "delete tablet dir " << tablet_path - << " directly due to trash_file_expire_time_sec is 0"; +Status DataDir::gc_tablet_path(const std::string& tablet_path, TabletPathGcMode mode) { + switch (mode) { + case TabletPathGcMode::DELETE_DIRECTLY: + LOG(INFO) << "delete tablet dir directly. path=" << tablet_path; + DBUG_EXECUTE_IF("DataDir.gc_tablet_path.delete_directly_failed", { + return Status::InternalError("injected direct delete failure. path={}", tablet_path); + }); RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(tablet_path)); return delete_tablet_parent_path_if_empty(tablet_path); + case TabletPathGcMode::MOVE_TO_TRASH: + return _move_tablet_path_to_trash(tablet_path); } + LOG(FATAL) << "invalid tablet path gc mode=" << static_cast(mode); + __builtin_unreachable(); +} +Status DataDir::move_to_trash(const std::string& tablet_path) { + const auto mode = config::trash_file_expire_time_sec <= 0 ? TabletPathGcMode::DELETE_DIRECTLY + : TabletPathGcMode::MOVE_TO_TRASH; + return gc_tablet_path(tablet_path, mode); +} + +Status DataDir::_move_tablet_path_to_trash(const std::string& tablet_path) { Status res = Status::OK(); // 1. get timestamp string std::string time_str; @@ -1112,8 +1156,13 @@ Status DataDir::move_to_trash(const std::string& tablet_path) { Status DataDir::delete_tablet_parent_path_if_empty(const std::string& tablet_path) { auto fs_tablet_path = io::Path(tablet_path); std::string source_parent_dir = fs_tablet_path.parent_path(); // tablet_id level - std::vector sub_files; bool exists = true; + RETURN_IF_ERROR(io::global_local_filesystem()->exists(source_parent_dir, &exists)); + if (!exists) { + return Status::OK(); + } + + std::vector sub_files; RETURN_IF_ERROR( io::global_local_filesystem()->list(source_parent_dir, false, &sub_files, &exists)); if (sub_files.empty()) { diff --git a/be/src/storage/data_dir.h b/be/src/storage/data_dir.h index 4598f3d87719fe..965c35cf4d42a2 100644 --- a/be/src/storage/data_dir.h +++ b/be/src/storage/data_dir.h @@ -32,6 +32,7 @@ #include "common/metrics/metrics.h" #include "common/status.h" +#include "storage/data_dir_sweep_policy.h" #include "storage/olap_common.h" namespace doris { @@ -141,7 +142,10 @@ class DataDir { (double)_disk_capacity_bytes; } - // Move tablet to trash. + // Apply the explicit sweep policy to a tablet path. + Status gc_tablet_path(const std::string& tablet_path, TabletPathGcMode mode); + + // Move the tablet path according to the configured trash retention policy. Status move_to_trash(const std::string& tablet_path); static Status delete_tablet_parent_path_if_empty(const std::string& tablet_path); @@ -158,6 +162,8 @@ class DataDir { // process will log fatal. Status _check_incompatible_old_format_tablet(); + Status _move_tablet_path_to_trash(const std::string& tablet_path); + int _path_gc_step {0}; void _perform_tablet_gc(const std::string& tablet_schema_hash_path, int16_t shard_name); diff --git a/be/src/storage/data_dir_sweep_policy.h b/be/src/storage/data_dir_sweep_policy.h new file mode 100644 index 00000000000000..7eea6d612fbf8b --- /dev/null +++ b/be/src/storage/data_dir_sweep_policy.h @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace doris { + +class DataDir; + +enum class TabletPathGcMode : uint8_t { + MOVE_TO_TRASH, + DELETE_DIRECTLY, +}; + +enum class TabletPathGcReason : uint8_t { + NORMAL_RETENTION, + TRASH_RETENTION_DISABLED, + MANUAL_CLEAN_TRASH, + HIGH_DISK_WATERMARK, + UNUSED_DATA_DIR, +}; + +struct ShutdownTabletGcPolicy { + // Ineligible shutdown tablets remain in the global queue and are not dispatched. + bool eligible = true; + TabletPathGcMode mode = TabletPathGcMode::MOVE_TO_TRASH; + TabletPathGcReason reason = TabletPathGcReason::NORMAL_RETENTION; +}; + +struct DataDirSweepPolicy { + bool is_used = false; + int32_t effective_trash_expire_seconds = 0; + ShutdownTabletGcPolicy shutdown_tablet_gc; +}; + +using DataDirSweepPolicies = std::unordered_map; + +inline DataDirSweepPolicy build_data_dir_sweep_policy(bool is_used, bool ignore_guard, + int32_t configured_trash_expire, + double current_usage, double guard_space) { + DataDirSweepPolicy policy; + policy.is_used = is_used; + if (!is_used) { + policy.effective_trash_expire_seconds = + configured_trash_expire <= 0 ? 0 : configured_trash_expire; + policy.shutdown_tablet_gc.eligible = false; + policy.shutdown_tablet_gc.reason = TabletPathGcReason::UNUSED_DATA_DIR; + return policy; + } + + const bool force_delete = + ignore_guard || configured_trash_expire <= 0 || current_usage > guard_space; + policy.effective_trash_expire_seconds = force_delete ? 0 : configured_trash_expire; + policy.shutdown_tablet_gc.mode = + force_delete ? TabletPathGcMode::DELETE_DIRECTLY : TabletPathGcMode::MOVE_TO_TRASH; + + if (configured_trash_expire <= 0) { + policy.shutdown_tablet_gc.reason = TabletPathGcReason::TRASH_RETENTION_DISABLED; + } else if (ignore_guard) { + policy.shutdown_tablet_gc.reason = TabletPathGcReason::MANUAL_CLEAN_TRASH; + } else if (current_usage > guard_space) { + policy.shutdown_tablet_gc.reason = TabletPathGcReason::HIGH_DISK_WATERMARK; + } + return policy; +} + +const char* tablet_path_gc_mode_name(TabletPathGcMode mode); +const char* tablet_path_gc_reason_name(TabletPathGcReason reason); + +} // namespace doris diff --git a/be/src/storage/storage_engine.cpp b/be/src/storage/storage_engine.cpp index 2bc9e46b101378..2aa62bd5c88573 100644 --- a/be/src/storage/storage_engine.cpp +++ b/be/src/storage/storage_engine.cpp @@ -856,13 +856,13 @@ Status StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) { const int32_t trash_expire = config::trash_file_expire_time_sec; // the guard space should be lower than storage_flood_stage_usage_percent, // so here we multiply 0.9 - // if ignore_guard is true, set guard_space to 0. - const double guard_space = - ignore_guard ? 0 : config::storage_flood_stage_usage_percent / 100.0 * 0.9; + const double guard_space = config::storage_flood_stage_usage_percent / 100.0 * 0.9; std::vector data_dir_infos; RETURN_NOT_OK_STATUS_WITH_WARN(get_all_data_dir_info(&data_dir_infos, false), "failed to get root path stat info when sweep trash.") std::sort(data_dir_infos.begin(), data_dir_infos.end(), DataDirInfoLessAvailability()); + DataDirSweepPolicies data_dir_sweep_policies; + data_dir_sweep_policies.reserve(data_dir_infos.size()); time_t now = time(nullptr); //获取UTC时间 tm local_tm_now; @@ -874,14 +874,32 @@ Status StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) { double tmp_usage = 0.0; for (DataDirInfo& info : data_dir_infos) { - LOG(INFO) << "Start to sweep path " << info.path; - if (!info.is_used) { - continue; + DataDir* data_dir = get_store(info.path); + CHECK(data_dir != nullptr) << "data dir is missing from store map. path=" << info.path; + + double curr_usage = 0.0; + if (info.is_used) { + curr_usage = + static_cast(info.disk_capacity - info.available) / info.disk_capacity; + tmp_usage = std::max(tmp_usage, curr_usage); } - double curr_usage = - (double)(info.disk_capacity - info.available) / (double)info.disk_capacity; - tmp_usage = std::max(tmp_usage, curr_usage); + auto policy = build_data_dir_sweep_policy(info.is_used, ignore_guard, trash_expire, + curr_usage, guard_space); + auto [_, inserted] = data_dir_sweep_policies.emplace(data_dir, policy); + CHECK(inserted) << "duplicated data dir sweep policy. path=" << info.path; + + LOG(INFO) << "Start to sweep path " << info.path << ", is_used=" << policy.is_used + << ", usage=" << curr_usage << ", guard_space=" << guard_space + << ", configured_trash_expire=" << trash_expire + << ", effective_trash_expire=" << policy.effective_trash_expire_seconds + << ", shutdown_tablet_gc_mode=" + << tablet_path_gc_mode_name(policy.shutdown_tablet_gc.mode) + << ", shutdown_tablet_gc_reason=" + << tablet_path_gc_reason_name(policy.shutdown_tablet_gc.reason); + if (!policy.is_used) { + continue; + } Status curr_res = Status::OK(); auto snapshot_path = fmt::format("{}/{}", info.path, SNAPSHOT_PREFIX); @@ -893,7 +911,7 @@ Status StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) { } auto trash_path = fmt::format("{}/{}", info.path, TRASH_PREFIX); - curr_res = _do_sweep(trash_path, local_now, curr_usage > guard_space ? 0 : trash_expire); + curr_res = _do_sweep(trash_path, local_now, policy.effective_trash_expire_seconds); if (!curr_res.ok()) { LOG(WARNING) << "failed to sweep trash. path=" << trash_path << ", err_code=" << curr_res; @@ -905,8 +923,8 @@ Status StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) { *usage = tmp_usage; // update usage } - // clear expire incremental rowset, move deleted tablet to trash - RETURN_IF_ERROR(_tablet_manager->start_trash_sweep()); + // Clear expired incremental rowsets and resolve shutdown tablet paths. + RETURN_IF_ERROR(_tablet_manager->start_trash_sweep(data_dir_sweep_policies)); // clean rubbish transactions _clean_unused_txns(); diff --git a/be/src/storage/tablet/tablet_manager.cpp b/be/src/storage/tablet/tablet_manager.cpp index c27215d3b914b8..8b638b45571459 100644 --- a/be/src/storage/tablet/tablet_manager.cpp +++ b/be/src/storage/tablet/tablet_manager.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include "absl/strings/substitute.h" #include "bvar/bvar.h" @@ -78,6 +79,23 @@ using std::vector; namespace doris { using namespace ErrorCode; +namespace { +bvar::Adder g_shutdown_tablet_direct_delete_attempts_total( + "shutdown_tablet_direct_delete_attempts_total"); +bvar::Adder g_shutdown_tablet_direct_delete_success_total( + "shutdown_tablet_direct_delete_success_total"); +bvar::Adder g_shutdown_tablet_direct_delete_failed_attempts_total( + "shutdown_tablet_direct_delete_failed_attempts_total"); +bvar::Adder g_shutdown_tablet_direct_delete_ms_total( + "shutdown_tablet_direct_delete_ms_total"); +bvar::Adder g_shutdown_tablet_direct_delete_clean_trash_total( + "shutdown_tablet_direct_delete_clean_trash_total"); +bvar::Adder g_shutdown_tablet_direct_delete_high_watermark_total( + "shutdown_tablet_direct_delete_high_watermark_total"); +bvar::Status g_shutdown_tablet_last_sweep_deferred_unused_data_dir( + "shutdown_tablet_last_sweep_deferred_unused_data_dir", 0); +} // namespace + bvar::Adder g_tablet_meta_schema_columns_count("tablet_meta_schema_columns_count"); TabletManager::TabletManager(StorageEngine& engine, int32_t tablet_map_lock_shard_size) @@ -1136,7 +1154,7 @@ void TabletManager::build_all_report_tablets_info(std::map* LOG(INFO) << "success to build all report tablets info. tablet_count=" << tablets_info->size(); } -Status TabletManager::start_trash_sweep() { +Status TabletManager::start_trash_sweep(const DataDirSweepPolicies& data_dir_sweep_policies) { DBUG_EXECUTE_IF("TabletManager.start_trash_sweep.sleep", DBUG_BLOCK); std::unique_lock lock(_gc_tablets_lock, std::defer_lock); if (!lock.try_lock()) { @@ -1179,6 +1197,8 @@ Status TabletManager::start_trash_sweep() { << ", tablet_id=" << tablet_id_with_max_useless_rowset_version_count; } + g_shutdown_tablet_last_sweep_deferred_unused_data_dir.set_value(0); + int64_t deferred_unused_data_dir_count = 0; std::list::iterator last_it; { std::shared_lock rdlock(_shutdown_tablets_lock); @@ -1188,17 +1208,26 @@ Status TabletManager::start_trash_sweep() { } } - auto get_batch_tablets = [this, &last_it](int limit) { - std::vector batch_tablets; + auto get_batch_tablets = [this, &data_dir_sweep_policies, &deferred_unused_data_dir_count, + &last_it](int limit) { + std::vector> batch_tablets; std::lock_guard wrdlock(_shutdown_tablets_lock); while (last_it != _shutdown_tablets.end() && batch_tablets.size() < limit) { + const auto& policy = _get_shutdown_tablet_gc_policy(data_dir_sweep_policies, *last_it); + if (!policy.eligible) { + ++deferred_unused_data_dir_count; + ++last_it; + continue; + } + // it means current tablet is referenced by other thread if (last_it->use_count() > 1) { last_it++; - } else { - batch_tablets.push_back(*last_it); - last_it = _shutdown_tablets.erase(last_it); + continue; } + + batch_tablets.emplace_back(*last_it, policy); + last_it = _shutdown_tablets.erase(last_it); } return batch_tablets; @@ -1210,8 +1239,8 @@ Status TabletManager::start_trash_sweep() { int limit = 200; for (;;) { auto batch_tablets = get_batch_tablets(limit); - for (const auto& tablet : batch_tablets) { - if (_move_tablet_to_trash(tablet)) { + for (const auto& [tablet, policy] : batch_tablets) { + if (_resolve_shutdown_tablet(tablet, policy)) { limit--; } else { failed_tablets.push_back(tablet); @@ -1239,12 +1268,49 @@ Status TabletManager::start_trash_sweep() { _shutdown_tablets.splice(_shutdown_tablets.end(), failed_tablets); } + g_shutdown_tablet_last_sweep_deferred_unused_data_dir.set_value(deferred_unused_data_dir_count); return Status::OK(); } -bool TabletManager::_move_tablet_to_trash(const TabletSharedPtr& tablet) { - RETURN_IF_ERROR(register_transition_tablet(tablet->tablet_id(), "move to trash")); - Defer defer {[&]() { unregister_transition_tablet(tablet->tablet_id(), "move to trash"); }}; +const ShutdownTabletGcPolicy& TabletManager::_get_shutdown_tablet_gc_policy( + const DataDirSweepPolicies& data_dir_sweep_policies, const TabletSharedPtr& tablet) const { + auto policy_it = data_dir_sweep_policies.find(tablet->data_dir()); + CHECK(policy_it != data_dir_sweep_policies.end()) + << "missing shutdown tablet gc policy. tablet_id=" << tablet->tablet_id() + << ", data_dir=" << tablet->data_dir()->path(); + return policy_it->second.shutdown_tablet_gc; +} + +Status TabletManager::_gc_shutdown_tablet_path(const TabletSharedPtr& tablet, + const ShutdownTabletGcPolicy& policy) { + if (policy.mode != TabletPathGcMode::DELETE_DIRECTLY) { + return tablet->data_dir()->gc_tablet_path(tablet->tablet_path(), policy.mode); + } + + g_shutdown_tablet_direct_delete_attempts_total << 1; + const int64_t start_ms = MonotonicMillis(); + Status status = tablet->data_dir()->gc_tablet_path(tablet->tablet_path(), policy.mode); + g_shutdown_tablet_direct_delete_ms_total << MonotonicMillis() - start_ms; + if (!status.ok()) { + g_shutdown_tablet_direct_delete_failed_attempts_total << 1; + return status; + } + + g_shutdown_tablet_direct_delete_success_total << 1; + if (policy.reason == TabletPathGcReason::MANUAL_CLEAN_TRASH) { + g_shutdown_tablet_direct_delete_clean_trash_total << 1; + } else if (policy.reason == TabletPathGcReason::HIGH_DISK_WATERMARK) { + g_shutdown_tablet_direct_delete_high_watermark_total << 1; + } + return status; +} + +bool TabletManager::_resolve_shutdown_tablet(const TabletSharedPtr& tablet, + const ShutdownTabletGcPolicy& policy) { + RETURN_IF_ERROR(register_transition_tablet(tablet->tablet_id(), "resolve shutdown tablet")); + Defer defer {[&]() { + unregister_transition_tablet(tablet->tablet_id(), "resolve shutdown tablet"); + }}; TabletSharedPtr tablet_in_not_shutdown = get_tablet(tablet->tablet_id()); if (tablet_in_not_shutdown) { @@ -1255,13 +1321,24 @@ bool TabletManager::_move_tablet_to_trash(const TabletSharedPtr& tablet) { tablet->clear_cache(); // shard_id in memory not eq shard_id in shutdown if (tablet_in_not_shutdown->tablet_path() != tablet->tablet_path()) { - LOG(INFO) << "tablet path not eq shutdown tablet path, move it to trash, tablet_id=" + LOG(INFO) << "tablet path not eq shutdown tablet path, resolve old path, tablet_id=" << tablet_in_not_shutdown->tablet_id() << ", mem manager tablet path=" << tablet_in_not_shutdown->tablet_path() - << ", shutdown tablet path=" << tablet->tablet_path(); - return tablet->data_dir()->move_to_trash(tablet->tablet_path()); + << ", shutdown tablet path=" << tablet->tablet_path() + << ", mode=" << tablet_path_gc_mode_name(policy.mode) + << ", reason=" << tablet_path_gc_reason_name(policy.reason); + Status gc_status = _gc_shutdown_tablet_path(tablet, policy); + if (!gc_status.ok()) { + LOG(WARNING) << "failed to resolve shutdown tablet path. tablet_id=" + << tablet->tablet_id() << ", tablet_path=" << tablet->tablet_path() + << ", mode=" << tablet_path_gc_mode_name(policy.mode) + << ", reason=" << tablet_path_gc_reason_name(policy.reason) + << ", error=" << gc_status; + return false; + } + return true; } else { - LOG(INFO) << "tablet path eq shutdown tablet path, not move to trash, tablet_id=" + LOG(INFO) << "tablet path eq shutdown tablet path, skip path gc, tablet_id=" << tablet_in_not_shutdown->tablet_id() << ", mem manager tablet path=" << tablet_in_not_shutdown->tablet_path() << ", shutdown tablet path=" << tablet->tablet_path(); @@ -1287,7 +1364,7 @@ bool TabletManager::_move_tablet_to_trash(const TabletSharedPtr& tablet) { tablet->clear_cache(); - // move data to trash + // Resolve the shutdown tablet path according to this sweep epoch's DataDir policy. const auto& tablet_path = tablet->tablet_path(); bool exists = false; Status exists_st = io::global_local_filesystem()->exists(tablet_path, &exists); @@ -1306,26 +1383,41 @@ bool TabletManager::_move_tablet_to_trash(const TabletSharedPtr& tablet) { return false; } int64_t now = MonotonicMicros(); - LOG(INFO) << "start to move tablet to trash. " << tablet_path + LOG(INFO) << "start to resolve shutdown tablet path. " << tablet_path + << ", mode=" << tablet_path_gc_mode_name(policy.mode) + << ", reason=" << tablet_path_gc_reason_name(policy.reason) << ". rocksdb get meta cost " << (save_meta_ts - get_meta_ts) << " us, rocksdb save meta cost " << (now - save_meta_ts) << " us"; - Status rm_st = tablet->data_dir()->move_to_trash(tablet_path); + Status rm_st = _gc_shutdown_tablet_path(tablet, policy); if (!rm_st.ok()) { - LOG(WARNING) << "fail to move dir to trash. " << tablet_path; + LOG(WARNING) << "failed to resolve shutdown tablet path. tablet_id=" + << tablet->tablet_id() << ", tablet_path=" << tablet_path + << ", mode=" << tablet_path_gc_mode_name(policy.mode) + << ", reason=" << tablet_path_gc_reason_name(policy.reason) + << ", error=" << rm_st; return false; } } // remove tablet meta - auto remove_st = TabletMetaManager::remove(tablet->data_dir(), tablet->tablet_id(), - tablet->schema_hash()); + auto remove_st = [&]() -> Status { + DBUG_EXECUTE_IF("TabletManager._resolve_shutdown_tablet.remove_meta_failed", { + return Status::InternalError( + "injected shutdown tablet meta delete failure. tablet_id={}, " + "schema_hash={}", + tablet->tablet_id(), tablet->schema_hash()); + }); + return TabletMetaManager::remove(tablet->data_dir(), tablet->tablet_id(), + tablet->schema_hash()); + }(); if (!remove_st.ok()) { LOG(WARNING) << "failed to remove meta, tablet_id=" << tablet_meta->tablet_id() << ", tablet_uid=" << tablet_meta->tablet_uid() << ", error=" << remove_st; return false; } - LOG(INFO) << "successfully move tablet to trash. " - << "tablet_id=" << tablet->tablet_id() - << ", schema_hash=" << tablet->schema_hash() << ", tablet_path=" << tablet_path; + LOG(INFO) << "successfully resolved shutdown tablet. tablet_id=" << tablet->tablet_id() + << ", schema_hash=" << tablet->schema_hash() << ", tablet_path=" << tablet_path + << ", mode=" << tablet_path_gc_mode_name(policy.mode) + << ", reason=" << tablet_path_gc_reason_name(policy.reason); return true; } else { tablet->clear_cache(); @@ -1342,8 +1434,8 @@ bool TabletManager::_move_tablet_to_trash(const TabletSharedPtr& tablet) { << "tablet_id=" << tablet->tablet_id() << ", schema_hash=" << tablet->schema_hash() << ", delete tablet_path=" << tablet_path; - RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(tablet_path)); - RETURN_IF_ERROR(DataDir::delete_tablet_parent_path_if_empty(tablet_path)); + RETURN_IF_ERROR(tablet->data_dir()->gc_tablet_path( + tablet_path, TabletPathGcMode::DELETE_DIRECTLY)); return true; } LOG(WARNING) << "errors while load meta from store, skip this tablet. " diff --git a/be/src/storage/tablet/tablet_manager.h b/be/src/storage/tablet/tablet_manager.h index 89b7c3fc96e24f..4a96e0e3f495cb 100644 --- a/be/src/storage/tablet/tablet_manager.h +++ b/be/src/storage/tablet/tablet_manager.h @@ -37,6 +37,7 @@ #include #include "common/status.h" +#include "storage/data_dir_sweep_policy.h" #include "storage/olap_common.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_meta.h" @@ -141,7 +142,7 @@ class TabletManager { void build_all_report_tablets_info(std::map* tablets_info); - Status start_trash_sweep(); + Status start_trash_sweep(const DataDirSweepPolicies& data_dir_sweep_policies); void try_delete_unused_tablet_path(DataDir* data_dir, TTabletId tablet_id, SchemaHash schema_hash, const std::string& schema_hash_path, @@ -225,7 +226,15 @@ class TabletManager { std::shared_mutex& _get_tablets_shard_lock(TTabletId tabletId); - bool _move_tablet_to_trash(const TabletSharedPtr& tablet); + const ShutdownTabletGcPolicy& _get_shutdown_tablet_gc_policy( + const DataDirSweepPolicies& data_dir_sweep_policies, + const TabletSharedPtr& tablet) const; + + Status _gc_shutdown_tablet_path(const TabletSharedPtr& tablet, + const ShutdownTabletGcPolicy& policy); + + bool _resolve_shutdown_tablet(const TabletSharedPtr& tablet, + const ShutdownTabletGcPolicy& policy); private: DISALLOW_COPY_AND_ASSIGN(TabletManager); diff --git a/be/test/storage/storage_engine_test.cpp b/be/test/storage/storage_engine_test.cpp index 9099601d434620..4b3ddaac2af171 100644 --- a/be/test/storage/storage_engine_test.cpp +++ b/be/test/storage/storage_engine_test.cpp @@ -33,6 +33,7 @@ #include "gtest/gtest_pred_impl.h" #include "io/fs/local_file_system.h" #include "storage/data_dir.h" +#include "storage/data_dir_sweep_policy.h" #include "storage/tablet/tablet_manager.h" #include "storage/tablet/tablet_meta_manager.h" #include "util/threadpool.h" @@ -40,6 +41,52 @@ namespace doris { using namespace config; +TEST(DataDirSweepPolicyTest, BuildsConsistentTrashAndShutdownPolicies) { + struct TestCase { + const char* name; + bool is_used; + bool ignore_guard; + int32_t configured_trash_expire; + double current_usage; + double guard_space; + int32_t expected_effective_expire; + bool expected_eligible; + TabletPathGcMode expected_mode; + TabletPathGcReason expected_reason; + }; + + const std::vector test_cases { + {"retention_disabled", true, false, 0, 0.1, 0.8, 0, true, + TabletPathGcMode::DELETE_DIRECTLY, TabletPathGcReason::TRASH_RETENTION_DISABLED}, + {"manual_clean_at_zero_usage", true, true, 3600, 0.0, 0.8, 0, true, + TabletPathGcMode::DELETE_DIRECTLY, TabletPathGcReason::MANUAL_CLEAN_TRASH}, + {"high_watermark", true, false, 3600, 0.81, 0.8, 0, true, + TabletPathGcMode::DELETE_DIRECTLY, TabletPathGcReason::HIGH_DISK_WATERMARK}, + {"below_watermark", true, false, 3600, 0.79, 0.8, 3600, true, + TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::NORMAL_RETENTION}, + {"at_watermark", true, false, 3600, 0.8, 0.8, 3600, true, + TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::NORMAL_RETENTION}, + {"unused_data_dir", false, true, 3600, 0.9, 0.8, 3600, false, + TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::UNUSED_DATA_DIR}, + }; + + for (const auto& test_case : test_cases) { + SCOPED_TRACE(test_case.name); + auto policy = build_data_dir_sweep_policy(test_case.is_used, test_case.ignore_guard, + test_case.configured_trash_expire, + test_case.current_usage, test_case.guard_space); + EXPECT_EQ(policy.is_used, test_case.is_used); + EXPECT_EQ(policy.effective_trash_expire_seconds, test_case.expected_effective_expire); + EXPECT_EQ(policy.shutdown_tablet_gc.eligible, test_case.expected_eligible); + EXPECT_EQ(policy.shutdown_tablet_gc.mode, test_case.expected_mode); + EXPECT_EQ(policy.shutdown_tablet_gc.reason, test_case.expected_reason); + if (policy.is_used) { + EXPECT_EQ(policy.effective_trash_expire_seconds <= 0, + policy.shutdown_tablet_gc.mode == TabletPathGcMode::DELETE_DIRECTLY); + } + } +} + class StorageEngineTest : public testing::Test { public: virtual void SetUp() { @@ -68,6 +115,33 @@ class StorageEngineTest : public testing::Test { std::unique_ptr _data_dir; }; +TEST_F(StorageEngineTest, GcTabletPathUsesExplicitMode) { + const std::string move_path = _engine_data_path + "/data/0/301/3333"; + ASSERT_TRUE(io::global_local_filesystem()->create_directory(move_path).ok()); + Status status = _data_dir->gc_tablet_path(move_path, TabletPathGcMode::MOVE_TO_TRASH); + ASSERT_TRUE(status.ok()) << status; + + bool exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(move_path, &exists).ok()); + EXPECT_FALSE(exists); + std::vector trash_paths; + _data_dir->find_tablet_in_trash(301, &trash_paths); + EXPECT_EQ(trash_paths.size(), 1); + + const std::string direct_path = _engine_data_path + "/data/0/302/3333"; + ASSERT_TRUE(io::global_local_filesystem()->create_directory(direct_path).ok()); + status = _data_dir->gc_tablet_path(direct_path, TabletPathGcMode::DELETE_DIRECTLY); + ASSERT_TRUE(status.ok()) << status; + ASSERT_TRUE(io::global_local_filesystem()->exists(direct_path, &exists).ok()); + EXPECT_FALSE(exists); + trash_paths.clear(); + _data_dir->find_tablet_in_trash(302, &trash_paths); + EXPECT_TRUE(trash_paths.empty()); + + status = _data_dir->gc_tablet_path(direct_path, TabletPathGcMode::DELETE_DIRECTLY); + EXPECT_TRUE(status.ok()) << status; +} + TEST_F(StorageEngineTest, TestBrokenDisk) { std::string path = config::custom_config_dir + "/be_custom.conf"; diff --git a/be/test/storage/tablet/tablet_mgr_test.cpp b/be/test/storage/tablet/tablet_mgr_test.cpp index b3e15265fc97f6..1fd3890a5fe7a4 100644 --- a/be/test/storage/tablet/tablet_mgr_test.cpp +++ b/be/test/storage/tablet/tablet_mgr_test.cpp @@ -24,10 +24,13 @@ #include #include +#include #include #include +#include #include +#include "bvar/variable.h" #include "common/config.h" #include "common/status.h" #include "gtest/gtest_pred_impl.h" @@ -47,6 +50,7 @@ #include "storage/tablet/tablet_manager.h" #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_meta_manager.h" +#include "util/debug_points.h" #include "util/uid_util.h" using ::testing::_; @@ -59,6 +63,11 @@ namespace doris { class TabletMgrTest : public testing::Test { public: virtual void SetUp() { + _original_enable_debug_points = config::enable_debug_points; + DebugPoints::instance()->remove("DataDir.gc_tablet_path.delete_directly_failed"); + DebugPoints::instance()->remove( + "TabletManager._resolve_shutdown_tablet.remove_meta_failed"); + _engine_data_path = "./be/test/storage/test_data/converter_test_data/tmp"; auto st = io::global_local_filesystem()->delete_directory(_engine_data_path); ASSERT_TRUE(st.ok()) << st; @@ -82,18 +91,105 @@ class TabletMgrTest : public testing::Test { } virtual void TearDown() { + _secondary_data_dir.reset(); + if (!_secondary_engine_data_path.empty()) { + EXPECT_TRUE(io::global_local_filesystem() + ->delete_directory(_secondary_engine_data_path) + .ok()); + } SAFE_DELETE(_data_dir); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_engine_data_path).ok()); ExecEnv::GetInstance()->set_storage_engine(nullptr); _tablet_mgr = nullptr; config::compaction_num_per_round = 1; + DebugPoints::instance()->remove("DataDir.gc_tablet_path.delete_directly_failed"); + DebugPoints::instance()->remove( + "TabletManager._resolve_shutdown_tablet.remove_meta_failed"); + config::enable_debug_points = _original_enable_debug_points; + } + + DataDirSweepPolicy sweep_policy( + TabletPathGcMode mode = TabletPathGcMode::DELETE_DIRECTLY, + TabletPathGcReason reason = TabletPathGcReason::TRASH_RETENTION_DISABLED, + bool eligible = true) const { + DataDirSweepPolicy policy; + policy.is_used = eligible; + policy.effective_trash_expire_seconds = + mode == TabletPathGcMode::DELETE_DIRECTLY ? 0 : 3600; + policy.shutdown_tablet_gc.eligible = eligible; + policy.shutdown_tablet_gc.mode = mode; + policy.shutdown_tablet_gc.reason = reason; + return policy; + } + + DataDirSweepPolicies sweep_policies( + TabletPathGcMode mode = TabletPathGcMode::DELETE_DIRECTLY, + TabletPathGcReason reason = TabletPathGcReason::TRASH_RETENTION_DISABLED, + bool eligible = true) const { + DataDirSweepPolicies policies; + policies.emplace(_data_dir, sweep_policy(mode, reason, eligible)); + return policies; + } + + TabletSharedPtr create_test_tablet(int64_t tablet_id, int32_t schema_hash = 3333) { + return create_test_tablet_on_data_dir(_data_dir, tablet_id, schema_hash); + } + + TabletSharedPtr create_test_tablet_on_data_dir(DataDir* data_dir, int64_t tablet_id, + int32_t schema_hash = 3333) { + TColumnType col_type; + col_type.__set_type(TPrimitiveType::SMALLINT); + TColumn column; + column.__set_column_name("col1"); + column.__set_column_type(col_type); + column.__set_is_key(true); + + TTabletSchema tablet_schema; + tablet_schema.__set_short_key_column_count(1); + tablet_schema.__set_schema_hash(schema_hash); + tablet_schema.__set_keys_type(TKeysType::AGG_KEYS); + tablet_schema.__set_storage_type(TStorageType::COLUMN); + tablet_schema.__set_columns({column}); + + TCreateTabletReq request; + request.__set_tablet_schema(tablet_schema); + request.__set_tablet_id(tablet_id); + request.__set_version(2); + + RuntimeProfile profile("CreateTablet"); + Status status = _tablet_mgr->create_tablet(request, {data_dir}, &profile); + EXPECT_TRUE(status.ok()) << status; + return _tablet_mgr->get_tablet(tablet_id); } + + Status create_secondary_data_dir() { + _secondary_engine_data_path = _engine_data_path + "_secondary"; + RETURN_IF_ERROR( + io::global_local_filesystem()->delete_directory(_secondary_engine_data_path)); + RETURN_IF_ERROR( + io::global_local_filesystem()->create_directory(_secondary_engine_data_path)); + RETURN_IF_ERROR(io::global_local_filesystem()->create_directory( + _secondary_engine_data_path + "/meta")); + _secondary_data_dir = + std::make_unique(*k_engine, _secondary_engine_data_path, 1000000000); + return _secondary_data_dir->init(); + } + + DataDir* secondary_data_dir() const { return _secondary_data_dir.get(); } + + int64_t metric_value(const std::string& name) const { + return std::stoll(bvar::Variable::describe_exposed(name)); + } + StorageEngine* k_engine; private: DataDir* _data_dir = nullptr; + std::unique_ptr _secondary_data_dir; std::string _engine_data_path; + std::string _secondary_engine_data_path; TabletManager* _tablet_mgr = nullptr; + bool _original_enable_debug_points = false; }; TEST_F(TabletMgrTest, CreateTablet) { @@ -138,7 +234,7 @@ TEST_F(TabletMgrTest, CreateTablet) { Status drop_st = _tablet_mgr->drop_tablet(111, create_tablet_req.replica_id, false); EXPECT_TRUE(drop_st == Status::OK()); tablet.reset(); - Status trash_st = _tablet_mgr->start_trash_sweep(); + Status trash_st = _tablet_mgr->start_trash_sweep(sweep_policies()); EXPECT_TRUE(trash_st == Status::OK()); } @@ -195,7 +291,7 @@ TEST_F(TabletMgrTest, CreateTabletWithSequence) { Status drop_st = _tablet_mgr->drop_tablet(111, create_tablet_req.replica_id, false); EXPECT_TRUE(drop_st == Status::OK()); tablet.reset(); - Status trash_st = _tablet_mgr->start_trash_sweep(); + Status trash_st = _tablet_mgr->start_trash_sweep(sweep_policies()); EXPECT_TRUE(trash_st == Status::OK()); } @@ -248,7 +344,7 @@ TEST_F(TabletMgrTest, DropTablet) { // do trash sweep, tablet will not be garbage collected // because tablet ptr referenced it - Status trash_st = _tablet_mgr->start_trash_sweep(); + Status trash_st = _tablet_mgr->start_trash_sweep(sweep_policies()); EXPECT_TRUE(trash_st == Status::OK()); tablet = _tablet_mgr->get_tablet(111, true); EXPECT_TRUE(tablet != nullptr); @@ -257,7 +353,7 @@ TEST_F(TabletMgrTest, DropTablet) { // reset tablet ptr tablet.reset(); - trash_st = _tablet_mgr->start_trash_sweep(); + trash_st = _tablet_mgr->start_trash_sweep(sweep_policies()); EXPECT_TRUE(trash_st == Status::OK()); tablet = _tablet_mgr->get_tablet(111, true); EXPECT_TRUE(tablet == nullptr); @@ -265,6 +361,289 @@ TEST_F(TabletMgrTest, DropTablet) { EXPECT_FALSE(dir_exist); } +TEST_F(TabletMgrTest, ShutdownTabletMovesToTrashWithRetentionPolicy) { + constexpr int64_t tablet_id = 201; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + status = _tablet_mgr->start_trash_sweep( + sweep_policies(TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::NORMAL_RETENTION)); + ASSERT_TRUE(status.ok()) << status; + + bool exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_FALSE(exists); + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); + + std::vector trash_paths; + _data_dir->find_tablet_in_trash(tablet_id, &trash_paths); + EXPECT_EQ(trash_paths.size(), 1); +} + +TEST_F(TabletMgrTest, ShutdownTabletDeletesDirectlyWithSweepPolicy) { + constexpr int64_t tablet_id = 202; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + status = _tablet_mgr->start_trash_sweep(sweep_policies( + TabletPathGcMode::DELETE_DIRECTLY, TabletPathGcReason::HIGH_DISK_WATERMARK)); + ASSERT_TRUE(status.ok()) << status; + + bool exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_FALSE(exists); + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); + + std::vector trash_paths; + _data_dir->find_tablet_in_trash(tablet_id, &trash_paths); + EXPECT_TRUE(trash_paths.empty()); +} + +TEST_F(TabletMgrTest, ShutdownTabletsUseDifferentDataDirPoliciesInSameSweep) { + Status status = create_secondary_data_dir(); + ASSERT_TRUE(status.ok()) << status; + DataDir* secondary_data_dir = this->secondary_data_dir(); + ASSERT_NE(secondary_data_dir, nullptr); + + constexpr int64_t direct_delete_tablet_id = 203; + auto direct_delete_tablet = create_test_tablet(direct_delete_tablet_id); + ASSERT_NE(direct_delete_tablet, nullptr); + const std::string direct_delete_path = direct_delete_tablet->tablet_path(); + + constexpr int64_t move_to_trash_tablet_id = 204; + auto move_to_trash_tablet = + create_test_tablet_on_data_dir(secondary_data_dir, move_to_trash_tablet_id); + ASSERT_NE(move_to_trash_tablet, nullptr); + const std::string move_to_trash_path = move_to_trash_tablet->tablet_path(); + + status = _tablet_mgr->drop_tablet(direct_delete_tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + status = _tablet_mgr->drop_tablet(move_to_trash_tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + direct_delete_tablet.reset(); + move_to_trash_tablet.reset(); + + auto policies = sweep_policies(TabletPathGcMode::DELETE_DIRECTLY, + TabletPathGcReason::HIGH_DISK_WATERMARK); + auto [_, inserted] = policies.emplace( + secondary_data_dir, + sweep_policy(TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::NORMAL_RETENTION)); + ASSERT_TRUE(inserted); + + status = _tablet_mgr->start_trash_sweep(policies); + ASSERT_TRUE(status.ok()) << status; + + bool exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(direct_delete_path, &exists).ok()); + EXPECT_FALSE(exists); + ASSERT_TRUE(io::global_local_filesystem()->exists(move_to_trash_path, &exists).ok()); + EXPECT_FALSE(exists); + EXPECT_EQ(_tablet_mgr->get_tablet(direct_delete_tablet_id, true), nullptr); + EXPECT_EQ(_tablet_mgr->get_tablet(move_to_trash_tablet_id, true), nullptr); + + std::vector trash_paths; + _data_dir->find_tablet_in_trash(direct_delete_tablet_id, &trash_paths); + EXPECT_TRUE(trash_paths.empty()); + secondary_data_dir->find_tablet_in_trash(move_to_trash_tablet_id, &trash_paths); + EXPECT_EQ(trash_paths.size(), 1); +} + +TEST_F(TabletMgrTest, ShutdownTabletOnUnusedDataDirRemainsQueued) { + constexpr int64_t tablet_id = 205; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + status = _tablet_mgr->start_trash_sweep(sweep_policies( + TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::UNUSED_DATA_DIR, false)); + ASSERT_TRUE(status.ok()) << status; + + tablet = _tablet_mgr->get_tablet(tablet_id, true); + ASSERT_NE(tablet, nullptr); + bool exists = false; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_TRUE(exists); + + tablet.reset(); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); +} + +TEST_F(TabletMgrTest, ShutdownTabletFailureIsRequeued) { + constexpr int64_t tablet_id = 206; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + Status transition_status; + std::promise transition_registered; + std::promise release_transition; + auto transition_registered_future = transition_registered.get_future(); + auto release_future = release_transition.get_future(); + std::thread transition_holder([&] { + transition_status = _tablet_mgr->register_transition_tablet(tablet_id, "test transition"); + transition_registered.set_value(); + release_future.wait(); + if (transition_status.ok()) { + _tablet_mgr->unregister_transition_tablet(tablet_id, "test transition"); + } + }); + + transition_registered_future.wait(); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + release_transition.set_value(); + transition_holder.join(); + ASSERT_TRUE(transition_status.ok()) << transition_status; + ASSERT_TRUE(status.ok()) << status; + + tablet = _tablet_mgr->get_tablet(tablet_id, true); + ASSERT_NE(tablet, nullptr); + bool exists = false; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_TRUE(exists); + + tablet.reset(); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); +} + +TEST_F(TabletMgrTest, ShutdownTabletDirectDeleteFailureIsRequeued) { + constexpr int64_t tablet_id = 207; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + const int64_t attempts_before = metric_value("shutdown_tablet_direct_delete_attempts_total"); + const int64_t success_before = metric_value("shutdown_tablet_direct_delete_success_total"); + const int64_t failed_attempts_before = + metric_value("shutdown_tablet_direct_delete_failed_attempts_total"); + config::enable_debug_points = true; + DebugPoints::instance()->add("DataDir.gc_tablet_path.delete_directly_failed"); + + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_attempts_total"), attempts_before + 1); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_success_total"), success_before); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_failed_attempts_total"), + failed_attempts_before + 1); + + tablet = _tablet_mgr->get_tablet(tablet_id, true); + ASSERT_NE(tablet, nullptr); + bool exists = false; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_TRUE(exists); + TabletMetaSharedPtr tablet_meta(new TabletMeta()); + status = TabletMetaManager::get_meta(_data_dir, tablet_id, tablet->schema_hash(), tablet_meta); + EXPECT_TRUE(status.ok()) << status; + + DebugPoints::instance()->remove("DataDir.gc_tablet_path.delete_directly_failed"); + tablet.reset(); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_FALSE(exists); +} + +TEST_F(TabletMgrTest, ShutdownTabletMetaDeleteFailureIsRetriedAfterPathDeletion) { + constexpr int64_t tablet_id = 208; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const int32_t schema_hash = tablet->schema_hash(); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + const int64_t attempts_before = metric_value("shutdown_tablet_direct_delete_attempts_total"); + const int64_t success_before = metric_value("shutdown_tablet_direct_delete_success_total"); + config::enable_debug_points = true; + DebugPoints::instance()->add("TabletManager._resolve_shutdown_tablet.remove_meta_failed"); + + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_attempts_total"), attempts_before + 1); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_success_total"), success_before + 1); + bool exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_FALSE(exists); + + tablet = _tablet_mgr->get_tablet(tablet_id, true); + ASSERT_NE(tablet, nullptr); + TabletMetaSharedPtr tablet_meta(new TabletMeta()); + status = TabletMetaManager::get_meta(_data_dir, tablet_id, schema_hash, tablet_meta); + EXPECT_TRUE(status.ok()) << status; + + DebugPoints::instance()->remove("TabletManager._resolve_shutdown_tablet.remove_meta_failed"); + tablet.reset(); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_attempts_total"), attempts_before + 1); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_success_total"), success_before + 1); + + status = TabletMetaManager::get_meta(_data_dir, tablet_id, schema_hash, tablet_meta); + EXPECT_TRUE(status.is()) << status; +} + +TEST_F(TabletMgrTest, ShutdownTabletIntentionalSkipDoesNotCountDirectDeleteSuccess) { + constexpr int64_t tablet_id = 209; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const int32_t schema_hash = tablet->schema_hash(); + const std::string tablet_path = tablet->tablet_path(); + std::string running_tablet_meta; + tablet->tablet_meta()->serialize(&running_tablet_meta); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + status = _tablet_mgr->load_tablet_from_meta(_data_dir, tablet_id, schema_hash, + running_tablet_meta, true, true, false, true); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + const int64_t attempts_before = metric_value("shutdown_tablet_direct_delete_attempts_total"); + const int64_t success_before = metric_value("shutdown_tablet_direct_delete_success_total"); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_attempts_total"), attempts_before); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_success_total"), success_before); + tablet = _tablet_mgr->get_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + EXPECT_EQ(tablet->tablet_path(), tablet_path); + bool exists = false; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_TRUE(exists); +} + TEST_F(TabletMgrTest, GetRowsetId) { // normal case { @@ -497,7 +876,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { } } - Status trash_st = _tablet_mgr->start_trash_sweep(); + Status trash_st = _tablet_mgr->start_trash_sweep(sweep_policies()); ASSERT_TRUE(trash_st.ok()) << trash_st; }