From d46dbd637604b7f25eb633d36e5370bbee82ccc2 Mon Sep 17 00:00:00 2001 From: wenzhenghu Date: Mon, 3 Aug 2026 21:55:31 +0800 Subject: [PATCH] [improvement](be) Apply trash sweep policy to shutdown tablets ### What problem does this PR solve? Issue Number: None Related PR: HYDCP/hy-doris#75 Problem Summary: Shutdown tablets were moved into trash even when the current sweep was deleting trash immediately for manual cleanup, disabled retention, or high disk usage. This delayed disk-space reclamation and could recreate trash entries during an urgent cleanup. Build one immutable policy per DataDir and use it for both trash expiration and shutdown-tablet path resolution. Eligible shutdown tablet paths are deleted directly during immediate cleanup conditions and continue to move to trash during normal retention sweeps. Shutdown tablets on unused DataDirs remain deferred, and existing transition, UID, path, metadata, reference-count, and failure-requeue safeguards are preserved. ### Release note Shutdown tablet paths now follow the current per-DataDir trash sweep policy. During immediate cleanup conditions they are deleted directly instead of being moved into trash again. ### Check List (For Author) - Test: Unit Test coverage added but not executed per requested validation scope - Static checks: clang-format 16, build-support/check-format.sh, and git diff --check passed - BE build: Not completed; interrupted during third-party dependency preparation before source compilation - Behavior changed: Yes. Shutdown tablet paths may be deleted directly during immediate trash cleanup conditions. - Does this need documentation: No --- be/src/storage/data_dir.cpp | 59 +++- be/src/storage/data_dir.h | 8 +- be/src/storage/data_dir_sweep_policy.h | 87 +++++ be/src/storage/storage_engine.cpp | 42 ++- be/src/storage/tablet/tablet_manager.cpp | 144 ++++++-- be/src/storage/tablet/tablet_manager.h | 13 +- be/test/storage/storage_engine_test.cpp | 74 ++++ be/test/storage/tablet/tablet_mgr_test.cpp | 389 ++++++++++++++++++++- 8 files changed, 765 insertions(+), 51 deletions(-) create mode 100644 be/src/storage/data_dir_sweep_policy.h diff --git a/be/src/storage/data_dir.cpp b/be/src/storage/data_dir.cpp index ab7b56f610a724..490af91954b019 100644 --- a/be/src/storage/data_dir.cpp +++ b/be/src/storage/data_dir.cpp @@ -66,6 +66,7 @@ #include "storage/tablet/tablet_meta_manager.h" #include "storage/txn/txn_manager.h" #include "storage/utils.h" // for check_dir_existed +#include "util/debug_points.h" #include "util/string_util.h" #include "util/uid_util.h" @@ -108,6 +109,34 @@ Status _write_cluster_id_to_path(const std::string& path, int32_t cluster_id) { } // namespace +const char* tablet_path_gc_mode_name(TabletPathGcMode mode) { + switch (mode) { + case TabletPathGcMode::MOVE_TO_TRASH: + return "MOVE_TO_TRASH"; + case TabletPathGcMode::DELETE_DIRECTLY: + return "DELETE_DIRECTLY"; + } + LOG(FATAL) << "invalid tablet path gc mode=" << static_cast(mode); + __builtin_unreachable(); +} + +const char* tablet_path_gc_reason_name(TabletPathGcReason reason) { + switch (reason) { + case TabletPathGcReason::NORMAL_RETENTION: + return "NORMAL_RETENTION"; + case TabletPathGcReason::TRASH_RETENTION_DISABLED: + return "TRASH_RETENTION_DISABLED"; + case TabletPathGcReason::MANUAL_CLEAN_TRASH: + return "MANUAL_CLEAN_TRASH"; + case TabletPathGcReason::HIGH_DISK_WATERMARK: + return "HIGH_DISK_WATERMARK"; + case TabletPathGcReason::UNUSED_DATA_DIR: + return "UNUSED_DATA_DIR"; + } + LOG(FATAL) << "invalid tablet path gc reason=" << static_cast(reason); + __builtin_unreachable(); +} + DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(disks_total_capacity, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(disks_avail_capacity, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(disks_local_used_capacity, MetricUnit::BYTES); @@ -1061,14 +1090,29 @@ void DataDir::disks_compaction_num_increment(int64_t delta) { disks_compaction_num->increment(delta); } -Status DataDir::move_to_trash(const std::string& tablet_path) { - if (config::trash_file_expire_time_sec <= 0) { - LOG(INFO) << "delete tablet dir " << tablet_path - << " directly due to trash_file_expire_time_sec is 0"; +Status DataDir::gc_tablet_path(const std::string& tablet_path, TabletPathGcMode mode) { + switch (mode) { + case TabletPathGcMode::DELETE_DIRECTLY: + LOG(INFO) << "delete tablet dir directly. path=" << tablet_path; + DBUG_EXECUTE_IF("DataDir.gc_tablet_path.delete_directly_failed", { + return Status::InternalError("injected direct delete failure. path={}", tablet_path); + }); RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(tablet_path)); return delete_tablet_parent_path_if_empty(tablet_path); + case TabletPathGcMode::MOVE_TO_TRASH: + return _move_tablet_path_to_trash(tablet_path); } + LOG(FATAL) << "invalid tablet path gc mode=" << static_cast(mode); + __builtin_unreachable(); +} +Status DataDir::move_to_trash(const std::string& tablet_path) { + const auto mode = config::trash_file_expire_time_sec <= 0 ? TabletPathGcMode::DELETE_DIRECTLY + : TabletPathGcMode::MOVE_TO_TRASH; + return gc_tablet_path(tablet_path, mode); +} + +Status DataDir::_move_tablet_path_to_trash(const std::string& tablet_path) { Status res = Status::OK(); // 1. get timestamp string std::string time_str; @@ -1112,8 +1156,13 @@ Status DataDir::move_to_trash(const std::string& tablet_path) { Status DataDir::delete_tablet_parent_path_if_empty(const std::string& tablet_path) { auto fs_tablet_path = io::Path(tablet_path); std::string source_parent_dir = fs_tablet_path.parent_path(); // tablet_id level - std::vector sub_files; bool exists = true; + RETURN_IF_ERROR(io::global_local_filesystem()->exists(source_parent_dir, &exists)); + if (!exists) { + return Status::OK(); + } + + std::vector sub_files; RETURN_IF_ERROR( io::global_local_filesystem()->list(source_parent_dir, false, &sub_files, &exists)); if (sub_files.empty()) { diff --git a/be/src/storage/data_dir.h b/be/src/storage/data_dir.h index 4598f3d87719fe..965c35cf4d42a2 100644 --- a/be/src/storage/data_dir.h +++ b/be/src/storage/data_dir.h @@ -32,6 +32,7 @@ #include "common/metrics/metrics.h" #include "common/status.h" +#include "storage/data_dir_sweep_policy.h" #include "storage/olap_common.h" namespace doris { @@ -141,7 +142,10 @@ class DataDir { (double)_disk_capacity_bytes; } - // Move tablet to trash. + // Apply the explicit sweep policy to a tablet path. + Status gc_tablet_path(const std::string& tablet_path, TabletPathGcMode mode); + + // Move the tablet path according to the configured trash retention policy. Status move_to_trash(const std::string& tablet_path); static Status delete_tablet_parent_path_if_empty(const std::string& tablet_path); @@ -158,6 +162,8 @@ class DataDir { // process will log fatal. Status _check_incompatible_old_format_tablet(); + Status _move_tablet_path_to_trash(const std::string& tablet_path); + int _path_gc_step {0}; void _perform_tablet_gc(const std::string& tablet_schema_hash_path, int16_t shard_name); diff --git a/be/src/storage/data_dir_sweep_policy.h b/be/src/storage/data_dir_sweep_policy.h new file mode 100644 index 00000000000000..7eea6d612fbf8b --- /dev/null +++ b/be/src/storage/data_dir_sweep_policy.h @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace doris { + +class DataDir; + +enum class TabletPathGcMode : uint8_t { + MOVE_TO_TRASH, + DELETE_DIRECTLY, +}; + +enum class TabletPathGcReason : uint8_t { + NORMAL_RETENTION, + TRASH_RETENTION_DISABLED, + MANUAL_CLEAN_TRASH, + HIGH_DISK_WATERMARK, + UNUSED_DATA_DIR, +}; + +struct ShutdownTabletGcPolicy { + // Ineligible shutdown tablets remain in the global queue and are not dispatched. + bool eligible = true; + TabletPathGcMode mode = TabletPathGcMode::MOVE_TO_TRASH; + TabletPathGcReason reason = TabletPathGcReason::NORMAL_RETENTION; +}; + +struct DataDirSweepPolicy { + bool is_used = false; + int32_t effective_trash_expire_seconds = 0; + ShutdownTabletGcPolicy shutdown_tablet_gc; +}; + +using DataDirSweepPolicies = std::unordered_map; + +inline DataDirSweepPolicy build_data_dir_sweep_policy(bool is_used, bool ignore_guard, + int32_t configured_trash_expire, + double current_usage, double guard_space) { + DataDirSweepPolicy policy; + policy.is_used = is_used; + if (!is_used) { + policy.effective_trash_expire_seconds = + configured_trash_expire <= 0 ? 0 : configured_trash_expire; + policy.shutdown_tablet_gc.eligible = false; + policy.shutdown_tablet_gc.reason = TabletPathGcReason::UNUSED_DATA_DIR; + return policy; + } + + const bool force_delete = + ignore_guard || configured_trash_expire <= 0 || current_usage > guard_space; + policy.effective_trash_expire_seconds = force_delete ? 0 : configured_trash_expire; + policy.shutdown_tablet_gc.mode = + force_delete ? TabletPathGcMode::DELETE_DIRECTLY : TabletPathGcMode::MOVE_TO_TRASH; + + if (configured_trash_expire <= 0) { + policy.shutdown_tablet_gc.reason = TabletPathGcReason::TRASH_RETENTION_DISABLED; + } else if (ignore_guard) { + policy.shutdown_tablet_gc.reason = TabletPathGcReason::MANUAL_CLEAN_TRASH; + } else if (current_usage > guard_space) { + policy.shutdown_tablet_gc.reason = TabletPathGcReason::HIGH_DISK_WATERMARK; + } + return policy; +} + +const char* tablet_path_gc_mode_name(TabletPathGcMode mode); +const char* tablet_path_gc_reason_name(TabletPathGcReason reason); + +} // namespace doris diff --git a/be/src/storage/storage_engine.cpp b/be/src/storage/storage_engine.cpp index 2bc9e46b101378..2aa62bd5c88573 100644 --- a/be/src/storage/storage_engine.cpp +++ b/be/src/storage/storage_engine.cpp @@ -856,13 +856,13 @@ Status StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) { const int32_t trash_expire = config::trash_file_expire_time_sec; // the guard space should be lower than storage_flood_stage_usage_percent, // so here we multiply 0.9 - // if ignore_guard is true, set guard_space to 0. - const double guard_space = - ignore_guard ? 0 : config::storage_flood_stage_usage_percent / 100.0 * 0.9; + const double guard_space = config::storage_flood_stage_usage_percent / 100.0 * 0.9; std::vector data_dir_infos; RETURN_NOT_OK_STATUS_WITH_WARN(get_all_data_dir_info(&data_dir_infos, false), "failed to get root path stat info when sweep trash.") std::sort(data_dir_infos.begin(), data_dir_infos.end(), DataDirInfoLessAvailability()); + DataDirSweepPolicies data_dir_sweep_policies; + data_dir_sweep_policies.reserve(data_dir_infos.size()); time_t now = time(nullptr); //获取UTC时间 tm local_tm_now; @@ -874,14 +874,32 @@ Status StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) { double tmp_usage = 0.0; for (DataDirInfo& info : data_dir_infos) { - LOG(INFO) << "Start to sweep path " << info.path; - if (!info.is_used) { - continue; + DataDir* data_dir = get_store(info.path); + CHECK(data_dir != nullptr) << "data dir is missing from store map. path=" << info.path; + + double curr_usage = 0.0; + if (info.is_used) { + curr_usage = + static_cast(info.disk_capacity - info.available) / info.disk_capacity; + tmp_usage = std::max(tmp_usage, curr_usage); } - double curr_usage = - (double)(info.disk_capacity - info.available) / (double)info.disk_capacity; - tmp_usage = std::max(tmp_usage, curr_usage); + auto policy = build_data_dir_sweep_policy(info.is_used, ignore_guard, trash_expire, + curr_usage, guard_space); + auto [_, inserted] = data_dir_sweep_policies.emplace(data_dir, policy); + CHECK(inserted) << "duplicated data dir sweep policy. path=" << info.path; + + LOG(INFO) << "Start to sweep path " << info.path << ", is_used=" << policy.is_used + << ", usage=" << curr_usage << ", guard_space=" << guard_space + << ", configured_trash_expire=" << trash_expire + << ", effective_trash_expire=" << policy.effective_trash_expire_seconds + << ", shutdown_tablet_gc_mode=" + << tablet_path_gc_mode_name(policy.shutdown_tablet_gc.mode) + << ", shutdown_tablet_gc_reason=" + << tablet_path_gc_reason_name(policy.shutdown_tablet_gc.reason); + if (!policy.is_used) { + continue; + } Status curr_res = Status::OK(); auto snapshot_path = fmt::format("{}/{}", info.path, SNAPSHOT_PREFIX); @@ -893,7 +911,7 @@ Status StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) { } auto trash_path = fmt::format("{}/{}", info.path, TRASH_PREFIX); - curr_res = _do_sweep(trash_path, local_now, curr_usage > guard_space ? 0 : trash_expire); + curr_res = _do_sweep(trash_path, local_now, policy.effective_trash_expire_seconds); if (!curr_res.ok()) { LOG(WARNING) << "failed to sweep trash. path=" << trash_path << ", err_code=" << curr_res; @@ -905,8 +923,8 @@ Status StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) { *usage = tmp_usage; // update usage } - // clear expire incremental rowset, move deleted tablet to trash - RETURN_IF_ERROR(_tablet_manager->start_trash_sweep()); + // Clear expired incremental rowsets and resolve shutdown tablet paths. + RETURN_IF_ERROR(_tablet_manager->start_trash_sweep(data_dir_sweep_policies)); // clean rubbish transactions _clean_unused_txns(); diff --git a/be/src/storage/tablet/tablet_manager.cpp b/be/src/storage/tablet/tablet_manager.cpp index c27215d3b914b8..8b638b45571459 100644 --- a/be/src/storage/tablet/tablet_manager.cpp +++ b/be/src/storage/tablet/tablet_manager.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include "absl/strings/substitute.h" #include "bvar/bvar.h" @@ -78,6 +79,23 @@ using std::vector; namespace doris { using namespace ErrorCode; +namespace { +bvar::Adder g_shutdown_tablet_direct_delete_attempts_total( + "shutdown_tablet_direct_delete_attempts_total"); +bvar::Adder g_shutdown_tablet_direct_delete_success_total( + "shutdown_tablet_direct_delete_success_total"); +bvar::Adder g_shutdown_tablet_direct_delete_failed_attempts_total( + "shutdown_tablet_direct_delete_failed_attempts_total"); +bvar::Adder g_shutdown_tablet_direct_delete_ms_total( + "shutdown_tablet_direct_delete_ms_total"); +bvar::Adder g_shutdown_tablet_direct_delete_clean_trash_total( + "shutdown_tablet_direct_delete_clean_trash_total"); +bvar::Adder g_shutdown_tablet_direct_delete_high_watermark_total( + "shutdown_tablet_direct_delete_high_watermark_total"); +bvar::Status g_shutdown_tablet_last_sweep_deferred_unused_data_dir( + "shutdown_tablet_last_sweep_deferred_unused_data_dir", 0); +} // namespace + bvar::Adder g_tablet_meta_schema_columns_count("tablet_meta_schema_columns_count"); TabletManager::TabletManager(StorageEngine& engine, int32_t tablet_map_lock_shard_size) @@ -1136,7 +1154,7 @@ void TabletManager::build_all_report_tablets_info(std::map* LOG(INFO) << "success to build all report tablets info. tablet_count=" << tablets_info->size(); } -Status TabletManager::start_trash_sweep() { +Status TabletManager::start_trash_sweep(const DataDirSweepPolicies& data_dir_sweep_policies) { DBUG_EXECUTE_IF("TabletManager.start_trash_sweep.sleep", DBUG_BLOCK); std::unique_lock lock(_gc_tablets_lock, std::defer_lock); if (!lock.try_lock()) { @@ -1179,6 +1197,8 @@ Status TabletManager::start_trash_sweep() { << ", tablet_id=" << tablet_id_with_max_useless_rowset_version_count; } + g_shutdown_tablet_last_sweep_deferred_unused_data_dir.set_value(0); + int64_t deferred_unused_data_dir_count = 0; std::list::iterator last_it; { std::shared_lock rdlock(_shutdown_tablets_lock); @@ -1188,17 +1208,26 @@ Status TabletManager::start_trash_sweep() { } } - auto get_batch_tablets = [this, &last_it](int limit) { - std::vector batch_tablets; + auto get_batch_tablets = [this, &data_dir_sweep_policies, &deferred_unused_data_dir_count, + &last_it](int limit) { + std::vector> batch_tablets; std::lock_guard wrdlock(_shutdown_tablets_lock); while (last_it != _shutdown_tablets.end() && batch_tablets.size() < limit) { + const auto& policy = _get_shutdown_tablet_gc_policy(data_dir_sweep_policies, *last_it); + if (!policy.eligible) { + ++deferred_unused_data_dir_count; + ++last_it; + continue; + } + // it means current tablet is referenced by other thread if (last_it->use_count() > 1) { last_it++; - } else { - batch_tablets.push_back(*last_it); - last_it = _shutdown_tablets.erase(last_it); + continue; } + + batch_tablets.emplace_back(*last_it, policy); + last_it = _shutdown_tablets.erase(last_it); } return batch_tablets; @@ -1210,8 +1239,8 @@ Status TabletManager::start_trash_sweep() { int limit = 200; for (;;) { auto batch_tablets = get_batch_tablets(limit); - for (const auto& tablet : batch_tablets) { - if (_move_tablet_to_trash(tablet)) { + for (const auto& [tablet, policy] : batch_tablets) { + if (_resolve_shutdown_tablet(tablet, policy)) { limit--; } else { failed_tablets.push_back(tablet); @@ -1239,12 +1268,49 @@ Status TabletManager::start_trash_sweep() { _shutdown_tablets.splice(_shutdown_tablets.end(), failed_tablets); } + g_shutdown_tablet_last_sweep_deferred_unused_data_dir.set_value(deferred_unused_data_dir_count); return Status::OK(); } -bool TabletManager::_move_tablet_to_trash(const TabletSharedPtr& tablet) { - RETURN_IF_ERROR(register_transition_tablet(tablet->tablet_id(), "move to trash")); - Defer defer {[&]() { unregister_transition_tablet(tablet->tablet_id(), "move to trash"); }}; +const ShutdownTabletGcPolicy& TabletManager::_get_shutdown_tablet_gc_policy( + const DataDirSweepPolicies& data_dir_sweep_policies, const TabletSharedPtr& tablet) const { + auto policy_it = data_dir_sweep_policies.find(tablet->data_dir()); + CHECK(policy_it != data_dir_sweep_policies.end()) + << "missing shutdown tablet gc policy. tablet_id=" << tablet->tablet_id() + << ", data_dir=" << tablet->data_dir()->path(); + return policy_it->second.shutdown_tablet_gc; +} + +Status TabletManager::_gc_shutdown_tablet_path(const TabletSharedPtr& tablet, + const ShutdownTabletGcPolicy& policy) { + if (policy.mode != TabletPathGcMode::DELETE_DIRECTLY) { + return tablet->data_dir()->gc_tablet_path(tablet->tablet_path(), policy.mode); + } + + g_shutdown_tablet_direct_delete_attempts_total << 1; + const int64_t start_ms = MonotonicMillis(); + Status status = tablet->data_dir()->gc_tablet_path(tablet->tablet_path(), policy.mode); + g_shutdown_tablet_direct_delete_ms_total << MonotonicMillis() - start_ms; + if (!status.ok()) { + g_shutdown_tablet_direct_delete_failed_attempts_total << 1; + return status; + } + + g_shutdown_tablet_direct_delete_success_total << 1; + if (policy.reason == TabletPathGcReason::MANUAL_CLEAN_TRASH) { + g_shutdown_tablet_direct_delete_clean_trash_total << 1; + } else if (policy.reason == TabletPathGcReason::HIGH_DISK_WATERMARK) { + g_shutdown_tablet_direct_delete_high_watermark_total << 1; + } + return status; +} + +bool TabletManager::_resolve_shutdown_tablet(const TabletSharedPtr& tablet, + const ShutdownTabletGcPolicy& policy) { + RETURN_IF_ERROR(register_transition_tablet(tablet->tablet_id(), "resolve shutdown tablet")); + Defer defer {[&]() { + unregister_transition_tablet(tablet->tablet_id(), "resolve shutdown tablet"); + }}; TabletSharedPtr tablet_in_not_shutdown = get_tablet(tablet->tablet_id()); if (tablet_in_not_shutdown) { @@ -1255,13 +1321,24 @@ bool TabletManager::_move_tablet_to_trash(const TabletSharedPtr& tablet) { tablet->clear_cache(); // shard_id in memory not eq shard_id in shutdown if (tablet_in_not_shutdown->tablet_path() != tablet->tablet_path()) { - LOG(INFO) << "tablet path not eq shutdown tablet path, move it to trash, tablet_id=" + LOG(INFO) << "tablet path not eq shutdown tablet path, resolve old path, tablet_id=" << tablet_in_not_shutdown->tablet_id() << ", mem manager tablet path=" << tablet_in_not_shutdown->tablet_path() - << ", shutdown tablet path=" << tablet->tablet_path(); - return tablet->data_dir()->move_to_trash(tablet->tablet_path()); + << ", shutdown tablet path=" << tablet->tablet_path() + << ", mode=" << tablet_path_gc_mode_name(policy.mode) + << ", reason=" << tablet_path_gc_reason_name(policy.reason); + Status gc_status = _gc_shutdown_tablet_path(tablet, policy); + if (!gc_status.ok()) { + LOG(WARNING) << "failed to resolve shutdown tablet path. tablet_id=" + << tablet->tablet_id() << ", tablet_path=" << tablet->tablet_path() + << ", mode=" << tablet_path_gc_mode_name(policy.mode) + << ", reason=" << tablet_path_gc_reason_name(policy.reason) + << ", error=" << gc_status; + return false; + } + return true; } else { - LOG(INFO) << "tablet path eq shutdown tablet path, not move to trash, tablet_id=" + LOG(INFO) << "tablet path eq shutdown tablet path, skip path gc, tablet_id=" << tablet_in_not_shutdown->tablet_id() << ", mem manager tablet path=" << tablet_in_not_shutdown->tablet_path() << ", shutdown tablet path=" << tablet->tablet_path(); @@ -1287,7 +1364,7 @@ bool TabletManager::_move_tablet_to_trash(const TabletSharedPtr& tablet) { tablet->clear_cache(); - // move data to trash + // Resolve the shutdown tablet path according to this sweep epoch's DataDir policy. const auto& tablet_path = tablet->tablet_path(); bool exists = false; Status exists_st = io::global_local_filesystem()->exists(tablet_path, &exists); @@ -1306,26 +1383,41 @@ bool TabletManager::_move_tablet_to_trash(const TabletSharedPtr& tablet) { return false; } int64_t now = MonotonicMicros(); - LOG(INFO) << "start to move tablet to trash. " << tablet_path + LOG(INFO) << "start to resolve shutdown tablet path. " << tablet_path + << ", mode=" << tablet_path_gc_mode_name(policy.mode) + << ", reason=" << tablet_path_gc_reason_name(policy.reason) << ". rocksdb get meta cost " << (save_meta_ts - get_meta_ts) << " us, rocksdb save meta cost " << (now - save_meta_ts) << " us"; - Status rm_st = tablet->data_dir()->move_to_trash(tablet_path); + Status rm_st = _gc_shutdown_tablet_path(tablet, policy); if (!rm_st.ok()) { - LOG(WARNING) << "fail to move dir to trash. " << tablet_path; + LOG(WARNING) << "failed to resolve shutdown tablet path. tablet_id=" + << tablet->tablet_id() << ", tablet_path=" << tablet_path + << ", mode=" << tablet_path_gc_mode_name(policy.mode) + << ", reason=" << tablet_path_gc_reason_name(policy.reason) + << ", error=" << rm_st; return false; } } // remove tablet meta - auto remove_st = TabletMetaManager::remove(tablet->data_dir(), tablet->tablet_id(), - tablet->schema_hash()); + auto remove_st = [&]() -> Status { + DBUG_EXECUTE_IF("TabletManager._resolve_shutdown_tablet.remove_meta_failed", { + return Status::InternalError( + "injected shutdown tablet meta delete failure. tablet_id={}, " + "schema_hash={}", + tablet->tablet_id(), tablet->schema_hash()); + }); + return TabletMetaManager::remove(tablet->data_dir(), tablet->tablet_id(), + tablet->schema_hash()); + }(); if (!remove_st.ok()) { LOG(WARNING) << "failed to remove meta, tablet_id=" << tablet_meta->tablet_id() << ", tablet_uid=" << tablet_meta->tablet_uid() << ", error=" << remove_st; return false; } - LOG(INFO) << "successfully move tablet to trash. " - << "tablet_id=" << tablet->tablet_id() - << ", schema_hash=" << tablet->schema_hash() << ", tablet_path=" << tablet_path; + LOG(INFO) << "successfully resolved shutdown tablet. tablet_id=" << tablet->tablet_id() + << ", schema_hash=" << tablet->schema_hash() << ", tablet_path=" << tablet_path + << ", mode=" << tablet_path_gc_mode_name(policy.mode) + << ", reason=" << tablet_path_gc_reason_name(policy.reason); return true; } else { tablet->clear_cache(); @@ -1342,8 +1434,8 @@ bool TabletManager::_move_tablet_to_trash(const TabletSharedPtr& tablet) { << "tablet_id=" << tablet->tablet_id() << ", schema_hash=" << tablet->schema_hash() << ", delete tablet_path=" << tablet_path; - RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(tablet_path)); - RETURN_IF_ERROR(DataDir::delete_tablet_parent_path_if_empty(tablet_path)); + RETURN_IF_ERROR(tablet->data_dir()->gc_tablet_path( + tablet_path, TabletPathGcMode::DELETE_DIRECTLY)); return true; } LOG(WARNING) << "errors while load meta from store, skip this tablet. " diff --git a/be/src/storage/tablet/tablet_manager.h b/be/src/storage/tablet/tablet_manager.h index 89b7c3fc96e24f..4a96e0e3f495cb 100644 --- a/be/src/storage/tablet/tablet_manager.h +++ b/be/src/storage/tablet/tablet_manager.h @@ -37,6 +37,7 @@ #include #include "common/status.h" +#include "storage/data_dir_sweep_policy.h" #include "storage/olap_common.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_meta.h" @@ -141,7 +142,7 @@ class TabletManager { void build_all_report_tablets_info(std::map* tablets_info); - Status start_trash_sweep(); + Status start_trash_sweep(const DataDirSweepPolicies& data_dir_sweep_policies); void try_delete_unused_tablet_path(DataDir* data_dir, TTabletId tablet_id, SchemaHash schema_hash, const std::string& schema_hash_path, @@ -225,7 +226,15 @@ class TabletManager { std::shared_mutex& _get_tablets_shard_lock(TTabletId tabletId); - bool _move_tablet_to_trash(const TabletSharedPtr& tablet); + const ShutdownTabletGcPolicy& _get_shutdown_tablet_gc_policy( + const DataDirSweepPolicies& data_dir_sweep_policies, + const TabletSharedPtr& tablet) const; + + Status _gc_shutdown_tablet_path(const TabletSharedPtr& tablet, + const ShutdownTabletGcPolicy& policy); + + bool _resolve_shutdown_tablet(const TabletSharedPtr& tablet, + const ShutdownTabletGcPolicy& policy); private: DISALLOW_COPY_AND_ASSIGN(TabletManager); diff --git a/be/test/storage/storage_engine_test.cpp b/be/test/storage/storage_engine_test.cpp index 9099601d434620..4b3ddaac2af171 100644 --- a/be/test/storage/storage_engine_test.cpp +++ b/be/test/storage/storage_engine_test.cpp @@ -33,6 +33,7 @@ #include "gtest/gtest_pred_impl.h" #include "io/fs/local_file_system.h" #include "storage/data_dir.h" +#include "storage/data_dir_sweep_policy.h" #include "storage/tablet/tablet_manager.h" #include "storage/tablet/tablet_meta_manager.h" #include "util/threadpool.h" @@ -40,6 +41,52 @@ namespace doris { using namespace config; +TEST(DataDirSweepPolicyTest, BuildsConsistentTrashAndShutdownPolicies) { + struct TestCase { + const char* name; + bool is_used; + bool ignore_guard; + int32_t configured_trash_expire; + double current_usage; + double guard_space; + int32_t expected_effective_expire; + bool expected_eligible; + TabletPathGcMode expected_mode; + TabletPathGcReason expected_reason; + }; + + const std::vector test_cases { + {"retention_disabled", true, false, 0, 0.1, 0.8, 0, true, + TabletPathGcMode::DELETE_DIRECTLY, TabletPathGcReason::TRASH_RETENTION_DISABLED}, + {"manual_clean_at_zero_usage", true, true, 3600, 0.0, 0.8, 0, true, + TabletPathGcMode::DELETE_DIRECTLY, TabletPathGcReason::MANUAL_CLEAN_TRASH}, + {"high_watermark", true, false, 3600, 0.81, 0.8, 0, true, + TabletPathGcMode::DELETE_DIRECTLY, TabletPathGcReason::HIGH_DISK_WATERMARK}, + {"below_watermark", true, false, 3600, 0.79, 0.8, 3600, true, + TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::NORMAL_RETENTION}, + {"at_watermark", true, false, 3600, 0.8, 0.8, 3600, true, + TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::NORMAL_RETENTION}, + {"unused_data_dir", false, true, 3600, 0.9, 0.8, 3600, false, + TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::UNUSED_DATA_DIR}, + }; + + for (const auto& test_case : test_cases) { + SCOPED_TRACE(test_case.name); + auto policy = build_data_dir_sweep_policy(test_case.is_used, test_case.ignore_guard, + test_case.configured_trash_expire, + test_case.current_usage, test_case.guard_space); + EXPECT_EQ(policy.is_used, test_case.is_used); + EXPECT_EQ(policy.effective_trash_expire_seconds, test_case.expected_effective_expire); + EXPECT_EQ(policy.shutdown_tablet_gc.eligible, test_case.expected_eligible); + EXPECT_EQ(policy.shutdown_tablet_gc.mode, test_case.expected_mode); + EXPECT_EQ(policy.shutdown_tablet_gc.reason, test_case.expected_reason); + if (policy.is_used) { + EXPECT_EQ(policy.effective_trash_expire_seconds <= 0, + policy.shutdown_tablet_gc.mode == TabletPathGcMode::DELETE_DIRECTLY); + } + } +} + class StorageEngineTest : public testing::Test { public: virtual void SetUp() { @@ -68,6 +115,33 @@ class StorageEngineTest : public testing::Test { std::unique_ptr _data_dir; }; +TEST_F(StorageEngineTest, GcTabletPathUsesExplicitMode) { + const std::string move_path = _engine_data_path + "/data/0/301/3333"; + ASSERT_TRUE(io::global_local_filesystem()->create_directory(move_path).ok()); + Status status = _data_dir->gc_tablet_path(move_path, TabletPathGcMode::MOVE_TO_TRASH); + ASSERT_TRUE(status.ok()) << status; + + bool exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(move_path, &exists).ok()); + EXPECT_FALSE(exists); + std::vector trash_paths; + _data_dir->find_tablet_in_trash(301, &trash_paths); + EXPECT_EQ(trash_paths.size(), 1); + + const std::string direct_path = _engine_data_path + "/data/0/302/3333"; + ASSERT_TRUE(io::global_local_filesystem()->create_directory(direct_path).ok()); + status = _data_dir->gc_tablet_path(direct_path, TabletPathGcMode::DELETE_DIRECTLY); + ASSERT_TRUE(status.ok()) << status; + ASSERT_TRUE(io::global_local_filesystem()->exists(direct_path, &exists).ok()); + EXPECT_FALSE(exists); + trash_paths.clear(); + _data_dir->find_tablet_in_trash(302, &trash_paths); + EXPECT_TRUE(trash_paths.empty()); + + status = _data_dir->gc_tablet_path(direct_path, TabletPathGcMode::DELETE_DIRECTLY); + EXPECT_TRUE(status.ok()) << status; +} + TEST_F(StorageEngineTest, TestBrokenDisk) { std::string path = config::custom_config_dir + "/be_custom.conf"; diff --git a/be/test/storage/tablet/tablet_mgr_test.cpp b/be/test/storage/tablet/tablet_mgr_test.cpp index b3e15265fc97f6..1fd3890a5fe7a4 100644 --- a/be/test/storage/tablet/tablet_mgr_test.cpp +++ b/be/test/storage/tablet/tablet_mgr_test.cpp @@ -24,10 +24,13 @@ #include #include +#include #include #include +#include #include +#include "bvar/variable.h" #include "common/config.h" #include "common/status.h" #include "gtest/gtest_pred_impl.h" @@ -47,6 +50,7 @@ #include "storage/tablet/tablet_manager.h" #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_meta_manager.h" +#include "util/debug_points.h" #include "util/uid_util.h" using ::testing::_; @@ -59,6 +63,11 @@ namespace doris { class TabletMgrTest : public testing::Test { public: virtual void SetUp() { + _original_enable_debug_points = config::enable_debug_points; + DebugPoints::instance()->remove("DataDir.gc_tablet_path.delete_directly_failed"); + DebugPoints::instance()->remove( + "TabletManager._resolve_shutdown_tablet.remove_meta_failed"); + _engine_data_path = "./be/test/storage/test_data/converter_test_data/tmp"; auto st = io::global_local_filesystem()->delete_directory(_engine_data_path); ASSERT_TRUE(st.ok()) << st; @@ -82,18 +91,105 @@ class TabletMgrTest : public testing::Test { } virtual void TearDown() { + _secondary_data_dir.reset(); + if (!_secondary_engine_data_path.empty()) { + EXPECT_TRUE(io::global_local_filesystem() + ->delete_directory(_secondary_engine_data_path) + .ok()); + } SAFE_DELETE(_data_dir); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_engine_data_path).ok()); ExecEnv::GetInstance()->set_storage_engine(nullptr); _tablet_mgr = nullptr; config::compaction_num_per_round = 1; + DebugPoints::instance()->remove("DataDir.gc_tablet_path.delete_directly_failed"); + DebugPoints::instance()->remove( + "TabletManager._resolve_shutdown_tablet.remove_meta_failed"); + config::enable_debug_points = _original_enable_debug_points; + } + + DataDirSweepPolicy sweep_policy( + TabletPathGcMode mode = TabletPathGcMode::DELETE_DIRECTLY, + TabletPathGcReason reason = TabletPathGcReason::TRASH_RETENTION_DISABLED, + bool eligible = true) const { + DataDirSweepPolicy policy; + policy.is_used = eligible; + policy.effective_trash_expire_seconds = + mode == TabletPathGcMode::DELETE_DIRECTLY ? 0 : 3600; + policy.shutdown_tablet_gc.eligible = eligible; + policy.shutdown_tablet_gc.mode = mode; + policy.shutdown_tablet_gc.reason = reason; + return policy; + } + + DataDirSweepPolicies sweep_policies( + TabletPathGcMode mode = TabletPathGcMode::DELETE_DIRECTLY, + TabletPathGcReason reason = TabletPathGcReason::TRASH_RETENTION_DISABLED, + bool eligible = true) const { + DataDirSweepPolicies policies; + policies.emplace(_data_dir, sweep_policy(mode, reason, eligible)); + return policies; + } + + TabletSharedPtr create_test_tablet(int64_t tablet_id, int32_t schema_hash = 3333) { + return create_test_tablet_on_data_dir(_data_dir, tablet_id, schema_hash); + } + + TabletSharedPtr create_test_tablet_on_data_dir(DataDir* data_dir, int64_t tablet_id, + int32_t schema_hash = 3333) { + TColumnType col_type; + col_type.__set_type(TPrimitiveType::SMALLINT); + TColumn column; + column.__set_column_name("col1"); + column.__set_column_type(col_type); + column.__set_is_key(true); + + TTabletSchema tablet_schema; + tablet_schema.__set_short_key_column_count(1); + tablet_schema.__set_schema_hash(schema_hash); + tablet_schema.__set_keys_type(TKeysType::AGG_KEYS); + tablet_schema.__set_storage_type(TStorageType::COLUMN); + tablet_schema.__set_columns({column}); + + TCreateTabletReq request; + request.__set_tablet_schema(tablet_schema); + request.__set_tablet_id(tablet_id); + request.__set_version(2); + + RuntimeProfile profile("CreateTablet"); + Status status = _tablet_mgr->create_tablet(request, {data_dir}, &profile); + EXPECT_TRUE(status.ok()) << status; + return _tablet_mgr->get_tablet(tablet_id); } + + Status create_secondary_data_dir() { + _secondary_engine_data_path = _engine_data_path + "_secondary"; + RETURN_IF_ERROR( + io::global_local_filesystem()->delete_directory(_secondary_engine_data_path)); + RETURN_IF_ERROR( + io::global_local_filesystem()->create_directory(_secondary_engine_data_path)); + RETURN_IF_ERROR(io::global_local_filesystem()->create_directory( + _secondary_engine_data_path + "/meta")); + _secondary_data_dir = + std::make_unique(*k_engine, _secondary_engine_data_path, 1000000000); + return _secondary_data_dir->init(); + } + + DataDir* secondary_data_dir() const { return _secondary_data_dir.get(); } + + int64_t metric_value(const std::string& name) const { + return std::stoll(bvar::Variable::describe_exposed(name)); + } + StorageEngine* k_engine; private: DataDir* _data_dir = nullptr; + std::unique_ptr _secondary_data_dir; std::string _engine_data_path; + std::string _secondary_engine_data_path; TabletManager* _tablet_mgr = nullptr; + bool _original_enable_debug_points = false; }; TEST_F(TabletMgrTest, CreateTablet) { @@ -138,7 +234,7 @@ TEST_F(TabletMgrTest, CreateTablet) { Status drop_st = _tablet_mgr->drop_tablet(111, create_tablet_req.replica_id, false); EXPECT_TRUE(drop_st == Status::OK()); tablet.reset(); - Status trash_st = _tablet_mgr->start_trash_sweep(); + Status trash_st = _tablet_mgr->start_trash_sweep(sweep_policies()); EXPECT_TRUE(trash_st == Status::OK()); } @@ -195,7 +291,7 @@ TEST_F(TabletMgrTest, CreateTabletWithSequence) { Status drop_st = _tablet_mgr->drop_tablet(111, create_tablet_req.replica_id, false); EXPECT_TRUE(drop_st == Status::OK()); tablet.reset(); - Status trash_st = _tablet_mgr->start_trash_sweep(); + Status trash_st = _tablet_mgr->start_trash_sweep(sweep_policies()); EXPECT_TRUE(trash_st == Status::OK()); } @@ -248,7 +344,7 @@ TEST_F(TabletMgrTest, DropTablet) { // do trash sweep, tablet will not be garbage collected // because tablet ptr referenced it - Status trash_st = _tablet_mgr->start_trash_sweep(); + Status trash_st = _tablet_mgr->start_trash_sweep(sweep_policies()); EXPECT_TRUE(trash_st == Status::OK()); tablet = _tablet_mgr->get_tablet(111, true); EXPECT_TRUE(tablet != nullptr); @@ -257,7 +353,7 @@ TEST_F(TabletMgrTest, DropTablet) { // reset tablet ptr tablet.reset(); - trash_st = _tablet_mgr->start_trash_sweep(); + trash_st = _tablet_mgr->start_trash_sweep(sweep_policies()); EXPECT_TRUE(trash_st == Status::OK()); tablet = _tablet_mgr->get_tablet(111, true); EXPECT_TRUE(tablet == nullptr); @@ -265,6 +361,289 @@ TEST_F(TabletMgrTest, DropTablet) { EXPECT_FALSE(dir_exist); } +TEST_F(TabletMgrTest, ShutdownTabletMovesToTrashWithRetentionPolicy) { + constexpr int64_t tablet_id = 201; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + status = _tablet_mgr->start_trash_sweep( + sweep_policies(TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::NORMAL_RETENTION)); + ASSERT_TRUE(status.ok()) << status; + + bool exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_FALSE(exists); + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); + + std::vector trash_paths; + _data_dir->find_tablet_in_trash(tablet_id, &trash_paths); + EXPECT_EQ(trash_paths.size(), 1); +} + +TEST_F(TabletMgrTest, ShutdownTabletDeletesDirectlyWithSweepPolicy) { + constexpr int64_t tablet_id = 202; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + status = _tablet_mgr->start_trash_sweep(sweep_policies( + TabletPathGcMode::DELETE_DIRECTLY, TabletPathGcReason::HIGH_DISK_WATERMARK)); + ASSERT_TRUE(status.ok()) << status; + + bool exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_FALSE(exists); + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); + + std::vector trash_paths; + _data_dir->find_tablet_in_trash(tablet_id, &trash_paths); + EXPECT_TRUE(trash_paths.empty()); +} + +TEST_F(TabletMgrTest, ShutdownTabletsUseDifferentDataDirPoliciesInSameSweep) { + Status status = create_secondary_data_dir(); + ASSERT_TRUE(status.ok()) << status; + DataDir* secondary_data_dir = this->secondary_data_dir(); + ASSERT_NE(secondary_data_dir, nullptr); + + constexpr int64_t direct_delete_tablet_id = 203; + auto direct_delete_tablet = create_test_tablet(direct_delete_tablet_id); + ASSERT_NE(direct_delete_tablet, nullptr); + const std::string direct_delete_path = direct_delete_tablet->tablet_path(); + + constexpr int64_t move_to_trash_tablet_id = 204; + auto move_to_trash_tablet = + create_test_tablet_on_data_dir(secondary_data_dir, move_to_trash_tablet_id); + ASSERT_NE(move_to_trash_tablet, nullptr); + const std::string move_to_trash_path = move_to_trash_tablet->tablet_path(); + + status = _tablet_mgr->drop_tablet(direct_delete_tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + status = _tablet_mgr->drop_tablet(move_to_trash_tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + direct_delete_tablet.reset(); + move_to_trash_tablet.reset(); + + auto policies = sweep_policies(TabletPathGcMode::DELETE_DIRECTLY, + TabletPathGcReason::HIGH_DISK_WATERMARK); + auto [_, inserted] = policies.emplace( + secondary_data_dir, + sweep_policy(TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::NORMAL_RETENTION)); + ASSERT_TRUE(inserted); + + status = _tablet_mgr->start_trash_sweep(policies); + ASSERT_TRUE(status.ok()) << status; + + bool exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(direct_delete_path, &exists).ok()); + EXPECT_FALSE(exists); + ASSERT_TRUE(io::global_local_filesystem()->exists(move_to_trash_path, &exists).ok()); + EXPECT_FALSE(exists); + EXPECT_EQ(_tablet_mgr->get_tablet(direct_delete_tablet_id, true), nullptr); + EXPECT_EQ(_tablet_mgr->get_tablet(move_to_trash_tablet_id, true), nullptr); + + std::vector trash_paths; + _data_dir->find_tablet_in_trash(direct_delete_tablet_id, &trash_paths); + EXPECT_TRUE(trash_paths.empty()); + secondary_data_dir->find_tablet_in_trash(move_to_trash_tablet_id, &trash_paths); + EXPECT_EQ(trash_paths.size(), 1); +} + +TEST_F(TabletMgrTest, ShutdownTabletOnUnusedDataDirRemainsQueued) { + constexpr int64_t tablet_id = 205; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + status = _tablet_mgr->start_trash_sweep(sweep_policies( + TabletPathGcMode::MOVE_TO_TRASH, TabletPathGcReason::UNUSED_DATA_DIR, false)); + ASSERT_TRUE(status.ok()) << status; + + tablet = _tablet_mgr->get_tablet(tablet_id, true); + ASSERT_NE(tablet, nullptr); + bool exists = false; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_TRUE(exists); + + tablet.reset(); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); +} + +TEST_F(TabletMgrTest, ShutdownTabletFailureIsRequeued) { + constexpr int64_t tablet_id = 206; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + Status transition_status; + std::promise transition_registered; + std::promise release_transition; + auto transition_registered_future = transition_registered.get_future(); + auto release_future = release_transition.get_future(); + std::thread transition_holder([&] { + transition_status = _tablet_mgr->register_transition_tablet(tablet_id, "test transition"); + transition_registered.set_value(); + release_future.wait(); + if (transition_status.ok()) { + _tablet_mgr->unregister_transition_tablet(tablet_id, "test transition"); + } + }); + + transition_registered_future.wait(); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + release_transition.set_value(); + transition_holder.join(); + ASSERT_TRUE(transition_status.ok()) << transition_status; + ASSERT_TRUE(status.ok()) << status; + + tablet = _tablet_mgr->get_tablet(tablet_id, true); + ASSERT_NE(tablet, nullptr); + bool exists = false; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_TRUE(exists); + + tablet.reset(); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); +} + +TEST_F(TabletMgrTest, ShutdownTabletDirectDeleteFailureIsRequeued) { + constexpr int64_t tablet_id = 207; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + const int64_t attempts_before = metric_value("shutdown_tablet_direct_delete_attempts_total"); + const int64_t success_before = metric_value("shutdown_tablet_direct_delete_success_total"); + const int64_t failed_attempts_before = + metric_value("shutdown_tablet_direct_delete_failed_attempts_total"); + config::enable_debug_points = true; + DebugPoints::instance()->add("DataDir.gc_tablet_path.delete_directly_failed"); + + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_attempts_total"), attempts_before + 1); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_success_total"), success_before); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_failed_attempts_total"), + failed_attempts_before + 1); + + tablet = _tablet_mgr->get_tablet(tablet_id, true); + ASSERT_NE(tablet, nullptr); + bool exists = false; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_TRUE(exists); + TabletMetaSharedPtr tablet_meta(new TabletMeta()); + status = TabletMetaManager::get_meta(_data_dir, tablet_id, tablet->schema_hash(), tablet_meta); + EXPECT_TRUE(status.ok()) << status; + + DebugPoints::instance()->remove("DataDir.gc_tablet_path.delete_directly_failed"); + tablet.reset(); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_FALSE(exists); +} + +TEST_F(TabletMgrTest, ShutdownTabletMetaDeleteFailureIsRetriedAfterPathDeletion) { + constexpr int64_t tablet_id = 208; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const int32_t schema_hash = tablet->schema_hash(); + const std::string tablet_path = tablet->tablet_path(); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + const int64_t attempts_before = metric_value("shutdown_tablet_direct_delete_attempts_total"); + const int64_t success_before = metric_value("shutdown_tablet_direct_delete_success_total"); + config::enable_debug_points = true; + DebugPoints::instance()->add("TabletManager._resolve_shutdown_tablet.remove_meta_failed"); + + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_attempts_total"), attempts_before + 1); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_success_total"), success_before + 1); + bool exists = true; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_FALSE(exists); + + tablet = _tablet_mgr->get_tablet(tablet_id, true); + ASSERT_NE(tablet, nullptr); + TabletMetaSharedPtr tablet_meta(new TabletMeta()); + status = TabletMetaManager::get_meta(_data_dir, tablet_id, schema_hash, tablet_meta); + EXPECT_TRUE(status.ok()) << status; + + DebugPoints::instance()->remove("TabletManager._resolve_shutdown_tablet.remove_meta_failed"); + tablet.reset(); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(_tablet_mgr->get_tablet(tablet_id, true), nullptr); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_attempts_total"), attempts_before + 1); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_success_total"), success_before + 1); + + status = TabletMetaManager::get_meta(_data_dir, tablet_id, schema_hash, tablet_meta); + EXPECT_TRUE(status.is()) << status; +} + +TEST_F(TabletMgrTest, ShutdownTabletIntentionalSkipDoesNotCountDirectDeleteSuccess) { + constexpr int64_t tablet_id = 209; + auto tablet = create_test_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + const int32_t schema_hash = tablet->schema_hash(); + const std::string tablet_path = tablet->tablet_path(); + std::string running_tablet_meta; + tablet->tablet_meta()->serialize(&running_tablet_meta); + + Status status = _tablet_mgr->drop_tablet(tablet_id, 0, false); + ASSERT_TRUE(status.ok()) << status; + status = _tablet_mgr->load_tablet_from_meta(_data_dir, tablet_id, schema_hash, + running_tablet_meta, true, true, false, true); + ASSERT_TRUE(status.ok()) << status; + tablet.reset(); + + const int64_t attempts_before = metric_value("shutdown_tablet_direct_delete_attempts_total"); + const int64_t success_before = metric_value("shutdown_tablet_direct_delete_success_total"); + status = _tablet_mgr->start_trash_sweep(sweep_policies()); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_attempts_total"), attempts_before); + EXPECT_EQ(metric_value("shutdown_tablet_direct_delete_success_total"), success_before); + tablet = _tablet_mgr->get_tablet(tablet_id); + ASSERT_NE(tablet, nullptr); + EXPECT_EQ(tablet->tablet_path(), tablet_path); + bool exists = false; + ASSERT_TRUE(io::global_local_filesystem()->exists(tablet_path, &exists).ok()); + EXPECT_TRUE(exists); +} + TEST_F(TabletMgrTest, GetRowsetId) { // normal case { @@ -497,7 +876,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { } } - Status trash_st = _tablet_mgr->start_trash_sweep(); + Status trash_st = _tablet_mgr->start_trash_sweep(sweep_policies()); ASSERT_TRUE(trash_st.ok()) << trash_st; }