-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Add ImportWAL tool with WAL replay controls #18516
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.apache.iotdb.db.i18n; | ||
|
|
||
| /** Compile-time i18n constants for the WAL import tool (English). */ | ||
| public final class ImportWALMessages { | ||
|
|
||
| public static final String MESSAGE_IMPORT_WAL_5E42804E = "import-wal"; | ||
| public static final String | ||
| MESSAGE_PATH_OF_A_WAL_FILE_OR_A_DIRECTORY_CONTAINING_WAL_FILES_473D0554 = | ||
| "Path of a WAL file or a directory containing WAL files."; | ||
| public static final String MESSAGE_TARGET_IOTDB_HOST_DEFAULT_127_0_0_1_3729156F = | ||
| "Target IoTDB host. Default: 127.0.0.1."; | ||
| public static final String MESSAGE_TARGET_IOTDB_RPC_PORT_DEFAULT_6667_FC0D345D = | ||
| "Target IoTDB RPC port. Default: 6667."; | ||
| public static final String MESSAGE_TARGET_IOTDB_USERNAME_DEFAULT_ROOT_EB91453B = | ||
| "Target IoTDB username. Default: root."; | ||
| public static final String | ||
| MESSAGE_TARGET_IOTDB_PASSWORD_PROMPTED_INTERACTIVELY_IF_OMITTED_29681961 = | ||
| "Target IoTDB password. Prompted interactively if omitted."; | ||
| public static final String MESSAGE_PASSWORD_PROMPT_F2D0E794 = "Password: "; | ||
| public static final String MESSAGE_TARGET_DATABASE_FOR_TABLE_MODEL_WAL_ENTRIES_27BACD1C = | ||
| "Target database for table-model WAL entries."; | ||
| public static final String | ||
| MESSAGE_WHEN_ALL_WAL_FILES_ARE_REPLAYED_SUCCESSFULLY_DO_OPERATION_ON_SOURCE_WAL_FILES_OPTIONAL_PARAMETERS_ARE_NONE_DEFAULT_AND_DELETE_41963A66 = | ||
| "When all WAL files are replayed successfully, do operation on source WAL files. Optional parameters are none (default) and delete."; | ||
| public static final String | ||
| MESSAGE_NUMBER_OF_THREADS_USED_TO_REPLAY_WAL_DIRECTORIES_IN_PARALLEL_DEFAULT_1_6AEF4F50 = | ||
| "Number of threads used to replay WAL directories in parallel. Default: 1."; | ||
| public static final String MESSAGE_PRINT_THIS_HELP_MESSAGE_E800AF7A = | ||
| "Print this help message."; | ||
| public static final String MESSAGE_ARGUMENT_ERROR_ARG_A9767F62 = "Argument error: %s"; | ||
| public static final String MESSAGE_WAL_IMPORT_FAILED_ARG_55C014BA = "WAL import failed: %s"; | ||
| public static final String EXCEPTION_SOURCE_PATH_DOES_NOT_EXIST_ARG_7C806CA2 = | ||
| "Source path does not exist: %s"; | ||
| public static final String EXCEPTION_SOURCE_FILE_IS_NOT_A_WAL_FILE_ARG_14A43F76 = | ||
| "Source file is not a WAL file: %s"; | ||
| public static final String EXCEPTION_NO_WAL_FILES_FOUND_UNDER_ARG_45F7FA22 = | ||
| "No WAL files found under: %s"; | ||
| public static final String EXCEPTION_INVALID_PORT_ARG_A7CDD5AC = "Invalid port: %s"; | ||
| public static final String | ||
| EXCEPTION_INVALID_THREAD_COUNT_ARG_EXPECTED_A_POSITIVE_INTEGER_F3AE2CFD = | ||
| "Invalid thread count: %s. Expected a positive integer."; | ||
| public static final String EXCEPTION_WAL_REPLAY_WAS_INTERRUPTED_770BA8AD = | ||
| "WAL replay was interrupted."; | ||
| public static final String | ||
| MESSAGE_REPLAYED_ARG_OPERATIONS_FROM_ARG_WAL_FILES_SKIPPED_ARG_ENTRIES_F0D37E3A = | ||
| "Replayed %d operations from %d WAL files; skipped %d entries."; | ||
| public static final String | ||
| MESSAGE_PROGRESS_ARG_COMPLETED_FILES_ARG_TOTAL_FILES_ARG_PROCESSED_BYTES_ARG_TOTAL_BYTES_ARG_PERCENT_ARG_ELAPSED_SECONDS_ARG_RATE_ARG_MB_PER_SECOND_F1C1356F = | ||
| "Progress: %d/%d WAL files completed, %d/%d bytes (%.1f%%), elapsed %.1f s, rate %.1f MB/s."; | ||
| public static final String | ||
| MESSAGE_IMPORT_DURATION_ARG_SECONDS_TOTAL_SIZE_ARG_BYTES_AVERAGE_RATE_ARG_MB_PER_SECOND_4B4EA58D = | ||
| "Import duration: %.1f s; total size: %d bytes; average rate: %.1f MB/s."; | ||
| public static final String MESSAGE_DELETED_ARG_SOURCE_WAL_FILES_C7A5AA1B = | ||
| "Deleted %d source WAL files."; | ||
| public static final String EXCEPTION_FAILED_TO_REPLAY_WAL_FILE_ARG_AT_OFFSET_ARG_ARG_FCFAF7F9 = | ||
| "Failed to replay WAL file %s at offset %d: %s"; | ||
| public static final String EXCEPTION_FAILED_TO_DELETE_SOURCE_WAL_FILE_ARG_ARG_236AF580 = | ||
| "Failed to delete source WAL file %s: %s"; | ||
| public static final String | ||
| EXCEPTION_UNSUPPORTED_ON_SUCCESS_VALUE_ARG_EXPECTED_NONE_OR_DELETE_F1C8EACE = | ||
| "Unsupported on_success value: %s. Expected none or delete."; | ||
| public static final String EXCEPTION_TABLE_MODEL_WAL_ENTRIES_REQUIRE_DB_DATABASE_F7597726 = | ||
| "Table-model WAL entries require -db/--database."; | ||
| public static final String EXCEPTION_UNSUPPORTED_WAL_OPERATION_ARG_ABD227A0 = | ||
| "Unsupported WAL operation: %s"; | ||
| public static final String | ||
| MESSAGE_TREE_MODEL_DELETE_OPERATION_DETECTED_ARG_CHOOSE_E_EXECUTE_S_SKIP_A_EXECUTE_ALL_L_SKIP_ALL_Q_QUIT_11E39FD7 = | ||
| "Tree-model delete operation detected: %s. Choose e=execute, s=skip, a=execute all, l=skip all, q=quit: "; | ||
| public static final String | ||
| MESSAGE_UNSUPPORTED_WAL_OPERATION_ARG_CHOOSE_S_SKIP_L_SKIP_ALL_Q_QUIT_0A734E52 = | ||
| "Unsupported WAL operation: %s. Choose s=skip, l=skip all, q=quit: "; | ||
| public static final String EXCEPTION_WAL_REPLAY_WAS_TERMINATED_BY_THE_USER_E0BD6197 = | ||
| "WAL replay was terminated by the user."; | ||
| public static final String EXCEPTION_INSERT_NODE_ARG_CONTAINS_NO_REPLAYABLE_DATA_5DA13453 = | ||
| "Insert node %s contains no replayable data."; | ||
| public static final String EXCEPTION_UNSUPPORTED_SNAPSHOT_DATA_TYPE_ARG_7A32D312 = | ||
| "Unsupported snapshot data type: %s"; | ||
| public static final String EXCEPTION_THE_WAL_FILE_IS_TRUNCATED_OR_CORRUPTED_6B0734C5 = | ||
| "The WAL file is truncated or corrupted."; | ||
| public static final String | ||
| EXCEPTION_PASSWORD_WAS_NOT_PROVIDED_AND_INTERACTIVE_INPUT_IS_UNAVAILABLE_40F42BCD = | ||
| "Password was not provided and interactive input is unavailable. Specify -pw/--password."; | ||
|
|
||
| private ImportWALMessages() {} | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.apache.iotdb.db.i18n; | ||
|
|
||
| /** WAL 导入工具的编译期国际化常量(中文)。 */ | ||
| public final class ImportWALMessages { | ||
|
|
||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The Chinese message class mirrors the English constants so |
||
| public static final String MESSAGE_IMPORT_WAL_5E42804E = "import-wal"; | ||
| public static final String | ||
| MESSAGE_PATH_OF_A_WAL_FILE_OR_A_DIRECTORY_CONTAINING_WAL_FILES_473D0554 = | ||
| "WAL 文件或包含 WAL 文件的目录路径。"; | ||
| public static final String MESSAGE_TARGET_IOTDB_HOST_DEFAULT_127_0_0_1_3729156F = | ||
| "目标 IoTDB 主机。默认:127.0.0.1。"; | ||
| public static final String MESSAGE_TARGET_IOTDB_RPC_PORT_DEFAULT_6667_FC0D345D = | ||
| "目标 IoTDB RPC 端口。默认:6667。"; | ||
| public static final String MESSAGE_TARGET_IOTDB_USERNAME_DEFAULT_ROOT_EB91453B = | ||
| "目标 IoTDB 用户名。默认:root。"; | ||
| public static final String | ||
| MESSAGE_TARGET_IOTDB_PASSWORD_PROMPTED_INTERACTIVELY_IF_OMITTED_29681961 = | ||
| "目标 IoTDB 密码。未提供时将交互式询问。"; | ||
| public static final String MESSAGE_PASSWORD_PROMPT_F2D0E794 = "密码:"; | ||
| public static final String MESSAGE_TARGET_DATABASE_FOR_TABLE_MODEL_WAL_ENTRIES_27BACD1C = | ||
| "表模型 WAL 条目的目标数据库。"; | ||
| public static final String | ||
| MESSAGE_WHEN_ALL_WAL_FILES_ARE_REPLAYED_SUCCESSFULLY_DO_OPERATION_ON_SOURCE_WAL_FILES_OPTIONAL_PARAMETERS_ARE_NONE_DEFAULT_AND_DELETE_41963A66 = | ||
| "所有 WAL 文件成功重放后,对源 WAL 文件执行操作。可选参数为 none(默认)和 delete。"; | ||
| public static final String | ||
| MESSAGE_NUMBER_OF_THREADS_USED_TO_REPLAY_WAL_DIRECTORIES_IN_PARALLEL_DEFAULT_1_6AEF4F50 = | ||
| "并行重放 WAL 目录所用的线程数。默认:1。"; | ||
| public static final String MESSAGE_PRINT_THIS_HELP_MESSAGE_E800AF7A = "打印帮助信息。"; | ||
| public static final String MESSAGE_ARGUMENT_ERROR_ARG_A9767F62 = "参数错误:%s"; | ||
| public static final String MESSAGE_WAL_IMPORT_FAILED_ARG_55C014BA = "WAL 导入失败:%s"; | ||
| public static final String EXCEPTION_SOURCE_PATH_DOES_NOT_EXIST_ARG_7C806CA2 = | ||
| "源路径不存在:%s"; | ||
| public static final String EXCEPTION_SOURCE_FILE_IS_NOT_A_WAL_FILE_ARG_14A43F76 = | ||
| "源文件不是 WAL 文件:%s"; | ||
| public static final String EXCEPTION_NO_WAL_FILES_FOUND_UNDER_ARG_45F7FA22 = | ||
| "路径下未找到 WAL 文件:%s"; | ||
| public static final String EXCEPTION_INVALID_PORT_ARG_A7CDD5AC = "无效端口:%s"; | ||
| public static final String | ||
| EXCEPTION_INVALID_THREAD_COUNT_ARG_EXPECTED_A_POSITIVE_INTEGER_F3AE2CFD = | ||
| "无效线程数:%s。应为正整数。"; | ||
| public static final String EXCEPTION_WAL_REPLAY_WAS_INTERRUPTED_770BA8AD = | ||
| "WAL 重放被中断。"; | ||
| public static final String | ||
| MESSAGE_REPLAYED_ARG_OPERATIONS_FROM_ARG_WAL_FILES_SKIPPED_ARG_ENTRIES_F0D37E3A = | ||
| "已重放 %d 个操作(来自 %d 个 WAL 文件);跳过 %d 个条目。"; | ||
| public static final String | ||
| MESSAGE_PROGRESS_ARG_COMPLETED_FILES_ARG_TOTAL_FILES_ARG_PROCESSED_BYTES_ARG_TOTAL_BYTES_ARG_PERCENT_ARG_ELAPSED_SECONDS_ARG_RATE_ARG_MB_PER_SECOND_F1C1356F = | ||
| "进度:已完成 %d/%d 个 WAL 文件,已处理 %d/%d 字节(%.1f%%),耗时 %.1f 秒,速率 %.1f MB/s。"; | ||
| public static final String | ||
| MESSAGE_IMPORT_DURATION_ARG_SECONDS_TOTAL_SIZE_ARG_BYTES_AVERAGE_RATE_ARG_MB_PER_SECOND_4B4EA58D = | ||
| "导入耗时:%.1f 秒;文件总大小:%d 字节;平均速率:%.1f MB/s。"; | ||
| public static final String MESSAGE_DELETED_ARG_SOURCE_WAL_FILES_C7A5AA1B = | ||
| "已删除 %d 个源 WAL 文件。"; | ||
| public static final String EXCEPTION_FAILED_TO_REPLAY_WAL_FILE_ARG_AT_OFFSET_ARG_ARG_FCFAF7F9 = | ||
| "重放 WAL 文件 %s 时失败,偏移量 %d:%s"; | ||
| public static final String EXCEPTION_FAILED_TO_DELETE_SOURCE_WAL_FILE_ARG_ARG_236AF580 = | ||
| "删除源 WAL 文件 %s 失败:%s"; | ||
| public static final String | ||
| EXCEPTION_UNSUPPORTED_ON_SUCCESS_VALUE_ARG_EXPECTED_NONE_OR_DELETE_F1C8EACE = | ||
| "不支持的 on_success 值:%s。应为 none 或 delete。"; | ||
| public static final String EXCEPTION_TABLE_MODEL_WAL_ENTRIES_REQUIRE_DB_DATABASE_F7597726 = | ||
| "表模型 WAL 条目要求指定 -db/--database。"; | ||
| public static final String EXCEPTION_UNSUPPORTED_WAL_OPERATION_ARG_ABD227A0 = | ||
| "不支持的 WAL 操作:%s"; | ||
| public static final String | ||
| MESSAGE_TREE_MODEL_DELETE_OPERATION_DETECTED_ARG_CHOOSE_E_EXECUTE_S_SKIP_A_EXECUTE_ALL_L_SKIP_ALL_Q_QUIT_11E39FD7 = | ||
| "检测到树模型删除操作:%s。请选择 e=执行、s=跳过、a=全部执行、l=全部跳过、q=终止重放:"; | ||
| public static final String | ||
| MESSAGE_UNSUPPORTED_WAL_OPERATION_ARG_CHOOSE_S_SKIP_L_SKIP_ALL_Q_QUIT_0A734E52 = | ||
| "不支持的 WAL 操作:%s。请选择 s=跳过、l=全部跳过、q=终止重放:"; | ||
| public static final String EXCEPTION_WAL_REPLAY_WAS_TERMINATED_BY_THE_USER_E0BD6197 = | ||
| "用户终止了 WAL 重放。"; | ||
| public static final String EXCEPTION_INSERT_NODE_ARG_CONTAINS_NO_REPLAYABLE_DATA_5DA13453 = | ||
| "Insert node %s 不包含可重放数据。"; | ||
| public static final String EXCEPTION_UNSUPPORTED_SNAPSHOT_DATA_TYPE_ARG_7A32D312 = | ||
| "Unsupported snapshot data type: %s"; | ||
| public static final String EXCEPTION_THE_WAL_FILE_IS_TRUNCATED_OR_CORRUPTED_6B0734C5 = | ||
| "WAL 文件被截断或已损坏。"; | ||
| public static final String | ||
| EXCEPTION_PASSWORD_WAS_NOT_PROVIDED_AND_INTERACTIVE_INPUT_IS_UNAVAILABLE_40F42BCD = | ||
| "未提供密码且当前环境不支持交互式输入,请指定 -pw/--password。"; | ||
|
|
||
| private ImportWALMessages() {} | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -72,6 +72,11 @@ public boolean hasNext() { | |
| return false; | ||
| } | ||
| try { | ||
| // An active WAL has no end marker until its writer closes. Reaching EOF exactly between | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. An active WAL may end cleanly at an entry boundary without an end marker. Checking available bytes before deserialization avoids misclassifying that boundary as corruption, while partial bytes still follow the existing corruption path. This is covered by the ImportWAL WAL reader tests. |
||
| // entries is therefore valid, while EOF during deserialization still marks a partial entry. | ||
| if (walInputStream.available() == 0) { | ||
| return false; | ||
| } | ||
| nextEntry = WALEntry.deserialize(logStream); | ||
| if (nextEntry.getType() == WALEntryType.WAL_FILE_INFO_END_MARKER) { | ||
| nextEntry = null; | ||
|
|
@@ -98,6 +103,10 @@ public long getWALCurrentReadOffset() throws IOException { | |
| return walInputStream.getFileCurrentPos(); | ||
| } | ||
|
|
||
| public boolean isFileCorrupted() { | ||
| return fileCorrupted; | ||
| } | ||
|
|
||
| /** | ||
| * Like {@link Iterator#next()}. | ||
| * | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Centralizing the import tool messages in the English locale keeps all user-facing CLI and replay errors localizable and gives the Chinese locale a stable key set to mirror.